From 34d159da598738e974d13711d5dc71e8fef991ff Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Tue, 2 Jun 2026 03:44:58 +0530 Subject: [PATCH 1/3] =?UTF-8?q?I=20always=20forgot=20to=20Push=20and=20won?= =?UTF-8?q?der=20where=20is=20all=20the=20changes=20=F0=9F=A5=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + AGENT.md | 20 +- Dockerfile | 9 +- README.md | 3 +- docker-compose.yml | 4 - input.txt.example | 32 +- pipeline-config.txt | 24 +- requirements-llm.txt | 4 + requirements-ml.txt | 21 - requirements.txt | 4 + src/website_profiling/analysis/__init__.py | 15 +- src/website_profiling/analysis/local.py | 349 ++++++++ src/website_profiling/analysis/text.py | 58 ++ src/website_profiling/cli.py | 13 +- src/website_profiling/common.py | 2 +- src/website_profiling/db/storage.py | 59 ++ src/website_profiling/llm/__init__.py | 4 + src/website_profiling/llm/base.py | 50 ++ src/website_profiling/llm/enrich.py | 298 +++++++ src/website_profiling/llm/prompts.py | 20 + .../llm/providers/__init__.py | 0 .../llm/providers/anthropic.py | 35 + src/website_profiling/llm/providers/gemini.py | 36 + src/website_profiling/llm/providers/ollama.py | 37 + src/website_profiling/llm/providers/openai.py | 42 + src/website_profiling/llm_config.py | 52 ++ src/website_profiling/ml/__init__.py | 1 - src/website_profiling/ml/enrich.py | 774 ------------------ src/website_profiling/reporting/builder.py | 27 +- src/website_profiling/reporting/categories.py | 20 +- src/website_profiling/tools/keywords.py | 32 +- tests/test_analysis.py | 44 + tests/test_llm_config.py | 33 + tests/test_llm_parse.py | 14 + web/app/api/llm-config/route.js | 67 ++ web/app/api/pipeline-config/route.js | 15 +- web/app/api/run/route.js | 38 +- web/src/components/PipelineRunnerFab.jsx | 176 +++- .../components/links/tabs/PageAnalysisTab.jsx | 9 - web/src/lib/llmConfigSchema.js | 117 +++ web/src/lib/pipelineConfigSchema.js | 28 +- web/src/server/llmConfig.js | 181 ++++ web/src/server/pipelineConfig.js | 12 +- web/src/strings.json | 7 +- web/src/views/Overview.jsx | 52 +- 45 files changed, 1834 insertions(+), 1005 deletions(-) create mode 100644 requirements-llm.txt delete mode 100644 requirements-ml.txt create mode 100644 src/website_profiling/analysis/local.py create mode 100644 src/website_profiling/analysis/text.py create mode 100644 src/website_profiling/llm/__init__.py create mode 100644 src/website_profiling/llm/base.py create mode 100644 src/website_profiling/llm/enrich.py create mode 100644 src/website_profiling/llm/prompts.py create mode 100644 src/website_profiling/llm/providers/__init__.py create mode 100644 src/website_profiling/llm/providers/anthropic.py create mode 100644 src/website_profiling/llm/providers/gemini.py create mode 100644 src/website_profiling/llm/providers/ollama.py create mode 100644 src/website_profiling/llm/providers/openai.py create mode 100644 src/website_profiling/llm_config.py delete mode 100644 src/website_profiling/ml/__init__.py delete mode 100644 src/website_profiling/ml/enrich.py create mode 100644 tests/test_analysis.py create mode 100644 tests/test_llm_config.py create mode 100644 tests/test_llm_parse.py create mode 100644 web/app/api/llm-config/route.js create mode 100644 web/src/lib/llmConfigSchema.js create mode 100644 web/src/server/llmConfig.js diff --git a/.gitignore b/.gitignore index 60c0f5d0..31d0df48 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ web/.env.local # Legacy local config file (use report.db / web UI instead) input.txt +*__pycache__* \ No newline at end of file diff --git a/AGENT.md b/AGENT.md index 12928e88..e8d1d7f4 100644 --- a/AGENT.md +++ b/AGENT.md @@ -2,12 +2,14 @@ **What it is:** `python -m src` from repo root (`src/__main__.py` -> package **`website_profiling`**). Config: stored in **`report.db`** (`pipeline_config` table, `key/value/is_unknown/updated_at`). A shadow **`pipeline-config.txt`** is auto-written next to `report.db` on every Save/Run. CLI loads DB first (`REPORT_DB_PATH` or `cwd/report.db`), then shadow file; `--config` overrides with a file. Reference keys: `input.txt.example` (not auto-loaded). +**LLM / AI:** Settings live in **`llm_config`** table in the same `report.db`. Configure only via web UI **AI** tab (`GET/PUT /api/llm-config`, localhost). Never in `pipeline-config.txt` or `--config`. + **Frontend:** **`web/`** (Next.js) -- server reads `report.db` via `/api/report/*`. **Key paths** -- `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `ml/enrich.py`, `tools/` -- `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.js`, `server/pipelineConfig.js` +- `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `analysis/`, `llm/`, `tools/` +- `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.js`, `server/pipelineConfig.js`, `server/llmConfig.js` **Run / APIs** @@ -16,8 +18,8 @@ - **`preserve_crawl_history`** (default true): append crawls; `false` recreates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs` - **`enrich_keywords_after_report`**: when omitted or `auto` (UI: Auto), follows `enable_google_search_console`; when set to Yes/No, explicit override - **`REPORT_DB_PATH`** env: DB path used by both Python and Next.js (Docker: `/data/report.db`; local default: `report.db` at repo root). Pipeline config lives in this DB. -- **`web/`:** `/api/report/*` (SQLite); `/api/run` spawns Python (localhost only); `/api/pipeline-config` GET/PUT for persistent settings; `PipelineRunnerFab` saves state to `report.db` (`pipeline_config` table) + shadow `pipeline-config.txt` before each run -- **Docker:** `Dockerfile` + `docker-compose.yml`; **`LIGHTHOUSE_CHROME_FLAGS`**; ML caches under `/data/cache/*` in compose +- **`web/`:** `/api/report/*` (SQLite); `/api/run` spawns Python (localhost only); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `PipelineRunnerFab` saves pipeline + LLM state before each run +- **Docker:** `Dockerfile` + `docker-compose.yml`; **`LIGHTHOUSE_CHROME_FLAGS`** **Where to edit** @@ -26,9 +28,11 @@ | Crawl | `crawl/crawler.py` | | Report | `reporting/builder.py`, `reporting/categories.py` | | DB schema | `db/storage.py` `init_schema` | -| ML | `ml/enrich.py`, `requirements-ml.txt` | +| Local analysis | `analysis/local.py`, `requirements.txt` | +| LLM enrichment | `llm/enrich.py`, `llm_config.py`, `requirements-llm.txt` | | Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` | -| UI config schema | `web/src/lib/pipelineConfigSchema.js` | -| UI config I/O | `web/src/server/pipelineConfig.js` | +| UI pipeline schema | `web/src/lib/pipelineConfigSchema.js` | +| UI LLM schema | `web/src/lib/llmConfigSchema.js` | +| UI config I/O | `web/src/server/pipelineConfig.js`, `web/src/server/llmConfig.js` | -Schema changes: edit `init_schema` only (no migration layer). ML stack: prefer Python **3.12** for spaCy/blis; **3.13** may fail pip builds. +Schema changes: edit `init_schema` only (no migration layer). diff --git a/Dockerfile b/Dockerfile index d860d475..ba18fb49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:20-bookworm-slim AS base -# Python venv + Chromium + build tools (ML stack may compile native wheels on some platforms) +# Python venv + Chromium + build tools RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ python3 \ @@ -38,15 +38,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHON=/opt/venv/bin/python \ CHROME_PATH=/usr/bin/chromium -# Python: base requirements, then ML/NLP (sentence-transformers, spaCy, KeyBERT, scikit-learn, …) +# Python: base requirements + optional LLM API clients COPY requirements.txt /app/requirements.txt -COPY requirements-ml.txt /app/requirements-ml.txt +COPY requirements-llm.txt /app/requirements-llm.txt RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m venv /opt/venv \ && /opt/venv/bin/pip install --upgrade pip \ && /opt/venv/bin/pip install -r /app/requirements.txt \ - && /opt/venv/bin/pip install -r /app/requirements-ml.txt \ - && /opt/venv/bin/python -m spacy download en_core_web_sm \ + && /opt/venv/bin/pip install -r /app/requirements-llm.txt \ && ln -sf /opt/venv/bin/python /usr/local/bin/python \ && ln -sf /opt/venv/bin/python /usr/local/bin/python3 diff --git a/README.md b/README.md index 0d8060f6..e2ff1237 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Activate `.venv`, then: pip install -r requirements.txt ``` -Optional ML: `pip install -r requirements-ml.txt` +Optional LLM enrichment: `pip install -r requirements-llm.txt` — configure in the web UI **AI** tab only (not via `pipeline-config.txt`). **2. Configure & run the pipeline** @@ -33,6 +33,7 @@ The easiest way is via the **web UI** (terminal icon, bottom-right corner at `ht - A shadow `pipeline-config.txt` is auto-written next to `report.db` on every Save/Run (safe to delete; regenerated automatically). - On first open, if the table is empty, the UI imports from shadow `pipeline-config.txt` (if present). - Click **Save settings** to persist, or **Run pipeline** to save + run immediately. +- **AI enrichment** (OpenAI, Gemini, Claude, Ollama): use the **AI** tab in the pipeline runner — settings are stored in `llm_config` inside `report.db` only (not in `pipeline-config.txt`). To run from the CLI instead: diff --git a/docker-compose.yml b/docker-compose.yml index da04b8d0..e97bc9b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,10 +11,6 @@ services: REPORT_DB_PATH: /data/report.db PYTHON: /opt/venv/bin/python NODE_ENV: production - HF_HOME: /data/cache/huggingface - TRANSFORMERS_CACHE: /data/cache/huggingface/hub - TORCH_HOME: /data/cache/torch - SENTENCE_TRANSFORMERS_HOME: /data/cache/sentence-transformers GOOGLE_SECRETS_PATH: /data/.secrets/google.json # Lighthouse in Docker: system Chromium + no-sandbox (Chrome requirement in containers) CHROME_PATH: /usr/bin/chromium diff --git a/input.txt.example b/input.txt.example index e2600edb..02955e2c 100644 --- a/input.txt.example +++ b/input.txt.example @@ -12,7 +12,7 @@ polite_delay = 0.2 ignore_robots = false allow_external = false store_outlinks = true -# Optional: store truncated body text for ML / UI (larger DB & report JSON) +# Optional: store truncated body text for analysis / AI (larger DB & report JSON) store_content_excerpt = true content_excerpt_max_chars = 4096 @@ -56,33 +56,15 @@ lighthouse_max_pages = 2 keyword_output_dir = . keyword_max_pages = 200 -# --- Optional ML (pip install -r requirements-ml.txt) --- -# Merged into report when run_report runs; re-run ML only: python -m src enrich +# --- Content analysis (local; pip install covers rapidfuzz, langdetect) --- enable_duplicate_detection = true -enable_anomaly_urls = true enable_language_detection = true -enable_ner_spacy = true -enable_semantic_similar_internal = true -enable_semantic_keywords = true +analysis_fuzzy_threshold = 92 +analysis_simhash_hamming = 0 +analysis_dup_max_pages = 2000 -ml_sentence_model = all-MiniLM-L6-v2 -ml_max_pages_st = 400 -ml_similar_top_k = 5 -ml_fuzzy_threshold = 92 -ml_simhash_hamming = 0 -ml_dup_max_pages = 2000 -ml_ner_max_pages = 80 -ml_semantic_keyword_max = 200 -ml_keyword_cluster_sim = 75 -# Fuzzy duplicate merge must also exceed this embedding cosine (0–100) when enabled -enable_embedding_duplicate_refine = true -ml_dup_embed_min_pct = 88 -# KeyBERT phrases per page (uses sentence-transformers model) -enable_keybert = true -ml_keybert_max_pages = 60 -ml_keybert_top_n = 8 -# Progress bars for long ST.encode runs -ml_verbose = true +# AI enrichment (OpenAI, Gemini, Claude, Ollama) is configured ONLY in the web UI +# (Pipeline runner → AI tab). Not available via this file or pipeline-config.txt. # --- Pipeline (python -m src) --- run_crawl = true diff --git a/pipeline-config.txt b/pipeline-config.txt index 2d309eea..a7d33b1f 100644 --- a/pipeline-config.txt +++ b/pipeline-config.txt @@ -52,28 +52,12 @@ lighthouse_max_pages = 2 keyword_output_dir = . keyword_max_pages = 200 -# --- ML --- +# --- Content analysis (local; no PyTorch) --- enable_duplicate_detection = true -enable_anomaly_urls = true enable_language_detection = true -enable_ner_spacy = true -enable_semantic_similar_internal = true -enable_semantic_keywords = true -ml_sentence_model = all-MiniLM-L6-v2 -ml_max_pages_st = 400 -ml_similar_top_k = 5 -ml_fuzzy_threshold = 92 -ml_simhash_hamming = 0 -ml_dup_max_pages = 2000 -ml_ner_max_pages = 80 -ml_semantic_keyword_max = 200 -ml_keyword_cluster_sim = 75 -enable_embedding_duplicate_refine = true -ml_dup_embed_min_pct = 88 -enable_keybert = true -ml_keybert_max_pages = 60 -ml_keybert_top_n = 8 -ml_verbose = true +analysis_fuzzy_threshold = 92 +analysis_simhash_hamming = 0 +analysis_dup_max_pages = 2000 # --- Pipeline --- run_crawl = true diff --git a/requirements-llm.txt b/requirements-llm.txt new file mode 100644 index 00000000..6800c042 --- /dev/null +++ b/requirements-llm.txt @@ -0,0 +1,4 @@ +# Optional LLM providers for AI enrichment (configure via web UI AI tab only) +httpx>=0.27.0 +openai>=1.0.0 +anthropic>=0.25.0 diff --git a/requirements-ml.txt b/requirements-ml.txt deleted file mode 100644 index 80aed7d8..00000000 --- a/requirements-ml.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Optional ML/NLP stack for WebsiteProfiling (pip install -r requirements-ml.txt) -# Enable features via pipeline config flags: enable_duplicate_detection, enable_anomaly_urls, etc. -# -# Python version: use 3.12.x (matches .github/workflows/deploy-pages.yml). -# On Python 3.13, pip may compile blis (spaCy → thinc) from source and fail with C/API errors. -# Fix: recreate the venv with Python 3.12, e.g. `python3.12 -m venv venv` then reinstall. - -rapidfuzz>=3.0.0 -scikit-learn>=1.3.0 -langdetect>=1.0.9 - -# Heavy: PyTorch + models (semantic similarity & keyword clustering) -sentence-transformers>=2.2.0 - -# Optional: KeyBERT salient phrases (enable_keybert in pipeline config) -keybert>=0.8.0 - -# NER — English model via official wheel (same as `python -m spacy download en_core_web_sm`) -spacy>=3.7.0 -# Optional CPU inference without importing full PyTorch in custom scripts (advanced) -# onnxruntime>=1.16.0 diff --git a/requirements.txt b/requirements.txt index 93b5ac10..1470c044 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,10 @@ tqdm>=4.64.0 networkx>=2.8.0 python-Wappalyzer>=0.3.1 +# Local content analysis (duplicates, language) +rapidfuzz>=3.0.0 +langdetect>=1.0.9 + # Google Search Console + GA4 integration (optional; required for `python -m src google`) google-auth>=2.0.0 google-auth-oauthlib>=1.0.0 diff --git a/src/website_profiling/analysis/__init__.py b/src/website_profiling/analysis/__init__.py index c6d5457b..52a8a063 100644 --- a/src/website_profiling/analysis/__init__.py +++ b/src/website_profiling/analysis/__init__.py @@ -1 +1,14 @@ -"""Page / content analysis.""" +"""Local content analysis (duplicates, language).""" +from .local import ( + merge_analysis_into_payload, + merge_bundles, + run_local_enrichment, +) +from .text import normalize_fingerprint_text + +__all__ = [ + "merge_analysis_into_payload", + "merge_bundles", + "normalize_fingerprint_text", + "run_local_enrichment", +] diff --git a/src/website_profiling/analysis/local.py b/src/website_profiling/analysis/local.py new file mode 100644 index 00000000..47a82d2d --- /dev/null +++ b/src/website_profiling/analysis/local.py @@ -0,0 +1,349 @@ +"""Local deterministic content analysis (no LLM).""" +from __future__ import annotations + +import hashlib +import re +from collections import Counter, defaultdict +from typing import Any + +import pandas as pd + +from .text import normalize_fingerprint_text + +LOCAL_INSTALL_HINT = "Install analysis dependencies: pip install rapidfuzz langdetect" + + +def _cfg_bool(cfg: dict[str, str] | None, key: str, default: bool = False) -> bool: + if not cfg: + return default + return str(cfg.get(key, default)).lower() in ("true", "1", "yes") + + +def _cfg_int(cfg: dict[str, str] | None, key: str, default: int) -> int: + if not cfg: + return default + raw = cfg.get(key) + if raw is None or str(raw).strip() == "": + # Legacy ml_* keys from old shadow files + legacy = { + "analysis_fuzzy_threshold": "ml_fuzzy_threshold", + "analysis_simhash_hamming": "ml_simhash_hamming", + "analysis_dup_max_pages": "ml_dup_max_pages", + }.get(key) + if legacy and cfg: + raw = cfg.get(legacy) + if raw is None or str(raw).strip() == "": + return default + try: + return int(str(raw).strip()) + except ValueError: + return default + + +def _tokenize_simhash(text: str) -> list[str]: + return re.findall(r"[a-z0-9]{3,}", text.lower()) + + +def _stable_token_hash(token: str) -> int: + return int.from_bytes(hashlib.md5(token.encode("utf-8")).digest()[:8], "little") + + +def simhash_64(text: str) -> int: + tokens = _tokenize_simhash(text) + if not tokens: + return 0 + vec = [0] * 64 + for tok in tokens: + h = _stable_token_hash(tok) + for i in range(64): + if (h >> i) & 1: + vec[i] += 1 + else: + vec[i] -= 1 + out = 0 + for i in range(64): + if vec[i] > 0: + out |= 1 << i + return out + + +def _hamming(a: int, b: int) -> int: + x = a ^ b + c = 0 + while x: + c += x & 1 + x >>= 1 + return c + + +def _import_rapidfuzz(): + try: + from rapidfuzz import fuzz + + return fuzz + except ImportError as e: + raise ImportError(f"{LOCAL_INSTALL_HINT}\n({e})") from e + + +def _import_langdetect(): + try: + from langdetect import LangDetectException, detect + + return detect, LangDetectException + except ImportError as e: + raise ImportError(f"{LOCAL_INSTALL_HINT}\n({e})") from e + + +def compute_duplicate_groups( + df: pd.DataFrame, + cfg: dict[str, str] | None, +) -> tuple[list[dict[str, Any]], dict[str, str]]: + if df.empty or not _cfg_bool(cfg, "enable_duplicate_detection", False): + return [], {} + + success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + if "content_type" in success.columns: + success = success[success["content_type"].fillna("").str.contains("text/html", case=False, na=False)] + max_pages = _cfg_int(cfg, "analysis_dup_max_pages", 2000) or 2000 + success = success.head(max_pages) + + url_to_fp: dict[str, str] = {} + url_to_sh: dict[str, int] = {} + for _, row in success.iterrows(): + u = str(row.get("url") or "").strip().rstrip("/") + if not u: + continue + fp = normalize_fingerprint_text(row) + if len(fp) < 20: + continue + url_to_fp[u] = fp + url_to_sh[u] = simhash_64(fp) + + bucket: dict[int, list[str]] = defaultdict(list) + for u, h in url_to_sh.items(): + bucket[h].append(u) + + fuzz = _import_rapidfuzz() + fuzzy_threshold = _cfg_int(cfg, "analysis_fuzzy_threshold", 92) or 92 + hamming_max = _cfg_int(cfg, "analysis_simhash_hamming", 0) or 0 + + parent: dict[str, str] = {} + + def find(x: str) -> str: + if x not in parent: + parent[x] = x + if parent[x] != x: + parent[x] = find(parent[x]) + return parent[x] + + def union(a: str, b: str) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + urls = list(url_to_fp.keys()) + for u in urls: + parent.setdefault(u, u) + + for _h, members in bucket.items(): + if len(members) < 2: + continue + base = members[0] + for m in members[1:]: + union(base, m) + + if hamming_max > 0 and len(urls) <= 800: + sh_list = [(u, url_to_sh[u]) for u in urls] + for i, (u1, h1) in enumerate(sh_list): + for u2, h2 in sh_list[i + 1 :]: + if _hamming(h1, h2) <= hamming_max: + union(u1, u2) + + if len(urls) <= 600: + for i, u1 in enumerate(urls): + fp1 = url_to_fp.get(u1, "") + for u2 in urls[i + 1 :]: + fp2 = url_to_fp.get(u2, "") + if fp1 and fp2 and fuzz.token_set_ratio(fp1, fp2) >= fuzzy_threshold: + union(u1, u2) + + clusters: dict[str, list[str]] = defaultdict(list) + for u in urls: + clusters[find(u)].append(u) + + groups_out: list[dict[str, Any]] = [] + url_to_gid: dict[str, str] = {} + gid = 0 + max_groups = 200 + for _root, members in clusters.items(): + if len(members) < 2: + continue + members = sorted(set(members)) + rep = members[0] + hashes = {url_to_sh.get(m) for m in members} + methods = ["simhash"] if len(hashes) == 1 else ["fuzzy"] + gkey = f"dup_{gid}" + gid += 1 + groups_out.append( + { + "id": gkey, + "representative_url": rep, + "member_urls": members[:100], + "member_count": len(members), + "methods": methods, + } + ) + for m in members: + url_to_gid[m] = gkey + if gid >= max_groups: + break + + return groups_out[:max_groups], url_to_gid + + +def compute_language_signals(df: pd.DataFrame, cfg: dict[str, str] | None) -> tuple[dict[str, str], dict[str, Any]]: + if df.empty or not _cfg_bool(cfg, "enable_language_detection", False): + return {}, {"counts": {}, "mixed_site": False} + + detect, LangDetectException = _import_langdetect() + by_url: dict[str, str] = {} + for _, row in df.iterrows(): + u = str(row.get("url") or "").strip().rstrip("/") + if not u: + continue + st = str(row.get("status") or "") + if not re.match(r"2\d{2}", st): + continue + text = normalize_fingerprint_text(row) + if len(text) < 30: + continue + try: + lang = detect(text[:2000]) + by_url[u] = lang + except LangDetectException: + continue + + counts = dict(Counter(by_url.values()).most_common(20)) + mixed = len(counts) > 1 + summary = {"counts": counts, "mixed_site": mixed, "detected_pages": len(by_url)} + return by_url, summary + + +def run_local_enrichment(df: pd.DataFrame, cfg: dict[str, str] | None) -> dict[str, Any]: + bundle: dict[str, Any] = { + "content_duplicates": [], + "url_duplicate_group_id": {}, + "language_by_url": {}, + "language_summary": {"counts": {}, "mixed_site": False}, + "spacy_by_url": {}, + "similar_internal_by_url": {}, + "ner_site_summary": {}, + "keyphrases_by_url": {}, + "ml_errors": [], + } + if df.empty: + return bundle + + try: + dups, url_gid = compute_duplicate_groups(df, cfg) + bundle["content_duplicates"] = dups + bundle["url_duplicate_group_id"] = url_gid + except ImportError as e: + bundle["ml_errors"].append(str(e)) + + try: + lang_map, lang_summary = compute_language_signals(df, cfg) + bundle["language_by_url"] = lang_map + bundle["language_summary"] = lang_summary + except ImportError as e: + bundle["ml_errors"].append(str(e)) + + return bundle + + +def merge_bundles(local: dict[str, Any], llm: dict[str, Any]) -> dict[str, Any]: + out = dict(local or {}) + llm = llm or {} + for key in ( + "content_duplicates", + "url_duplicate_group_id", + "language_by_url", + "language_summary", + "spacy_by_url", + "similar_internal_by_url", + "ner_site_summary", + "keyphrases_by_url", + ): + if key in llm and llm[key]: + if key in ("language_by_url", "spacy_by_url", "similar_internal_by_url", "keyphrases_by_url"): + merged = dict(out.get(key) or {}) + merged.update(llm[key]) + out[key] = merged + elif key == "url_duplicate_group_id": + merged = dict(out.get(key) or {}) + merged.update(llm[key]) + out[key] = merged + else: + out[key] = llm[key] + errs = list(out.get("ml_errors") or []) + list(llm.get("ml_errors") or []) + if errs: + out["ml_errors"] = errs + return out + + +def merge_analysis_into_payload(payload: dict[str, Any], bundle: dict[str, Any]) -> None: + """Mutate report payload with analysis / LLM enrichment fields.""" + payload["content_duplicates"] = bundle.get("content_duplicates") or [] + payload.pop("anomalies", None) + payload["language_summary"] = bundle.get("language_summary") or {} + ns = bundle.get("ner_site_summary") or {} + if ns: + payload["ner_site_summary"] = ns + else: + payload.pop("ner_site_summary", None) + err = bundle.get("ml_errors") or [] + if err: + payload["ml_errors"] = err + else: + payload.pop("ml_errors", None) + + dup_gid = bundle.get("url_duplicate_group_id") or {} + sim_map = bundle.get("similar_internal_by_url") or {} + lang_map = bundle.get("language_by_url") or {} + nlp_map = bundle.get("spacy_by_url") or {} + kp_map = bundle.get("keyphrases_by_url") or {} + + for rec in payload.get("links") or []: + if not isinstance(rec, dict): + continue + u = str(rec.get("url") or "").strip() + uk = u.rstrip("/") + rec.pop("duplicate_group_id", None) + rec.pop("similar_internal", None) + rec.pop("detected_language", None) + rec.pop("nlp_entities", None) + rec.pop("ml_anomaly", None) + rec.pop("keyphrases", None) + if uk in dup_gid: + rec["duplicate_group_id"] = dup_gid[uk] + nei = sim_map.get(uk) or sim_map.get(u) + if nei: + rec["similar_internal"] = list(nei) + if uk in lang_map: + rec["detected_language"] = lang_map[uk] + if uk in nlp_map: + rec["nlp_entities"] = nlp_map[uk] + if uk in kp_map: + rec["keyphrases"] = kp_map[uk] + pa = rec.get("page_analysis") + if isinstance(pa, dict): + sig = pa.get("signals") + if isinstance(sig, dict): + sig.pop("language", None) + sig.pop("nlp_entities", None) + if not sig: + pa.pop("signals", None) + if uk in lang_map: + pa.setdefault("signals", {})["language"] = lang_map[uk] + if uk in nlp_map: + pa.setdefault("signals", {})["nlp_entities"] = nlp_map[uk] diff --git a/src/website_profiling/analysis/text.py b/src/website_profiling/analysis/text.py new file mode 100644 index 00000000..511b9f54 --- /dev/null +++ b/src/website_profiling/analysis/text.py @@ -0,0 +1,58 @@ +"""Shared text helpers for content analysis and LLM enrichment.""" +from __future__ import annotations + +import json +import re + +import pandas as pd + + +def top_keywords_as_text(row: pd.Series, max_terms: int = 15) -> str: + if "top_keywords" not in row.index: + return "" + raw = row.get("top_keywords") + if raw is None or (isinstance(raw, float) and pd.isna(raw)): + return "" + s = str(raw).strip() + if not s or s == "[]": + return "" + try: + arr = json.loads(s) + if not isinstance(arr, list): + return "" + words: list[str] = [] + for item in arr[:max_terms]: + if isinstance(item, dict) and item.get("word"): + words.append(str(item["word"])) + return " ".join(words) + except json.JSONDecodeError: + return "" + + +def normalize_fingerprint_text(row: pd.Series) -> str: + """Concatenate on-page text signals for duplicates, language, and LLM context.""" + parts: list[str] = [] + for col in ( + "title", + "h1", + "meta_description", + "heading_sequence", + "og_title", + "og_description", + "twitter_title", + "content_excerpt", + ): + if col not in row.index: + continue + v = row.get(col) + if v is None or (isinstance(v, float) and pd.isna(v)): + continue + s = str(v).strip() + if s: + parts.append(s) + kw_extra = top_keywords_as_text(row) + if kw_extra: + parts.append(kw_extra) + t = " ".join(parts).lower() + t = re.sub(r"\s+", " ", t) + return t[:12000] diff --git a/src/website_profiling/cli.py b/src/website_profiling/cli.py index f3850673..7cb26ce5 100644 --- a/src/website_profiling/cli.py +++ b/src/website_profiling/cli.py @@ -259,9 +259,11 @@ def path(key: str, default: str) -> str: if not db_path: print("enrich requires sqlite_db in config.", file=sys.stderr) sys.exit(1) - print("WebsiteProfiling: ML enrich only (updates latest report payload)...", flush=True) + print("WebsiteProfiling: enrich only (updates latest report payload)...", flush=True) from .db import db_session, get_latest_crawl_run_id, init_schema, read_crawl, read_report_payload, write_report_payload - from .ml.enrich import merge_ml_into_payload, run_ml_enrichment + from .analysis import merge_analysis_into_payload, merge_bundles, run_local_enrichment + from .llm.enrich import run_llm_enrichment + from .llm_config import load_llm_config_from_db, llm_is_enabled with db_session(db_path) as conn: init_schema(conn) @@ -271,8 +273,11 @@ def path(key: str, default: str) -> str: if not payload: print("No report_payload in DB. Run report first.", file=sys.stderr) sys.exit(1) - ml_bundle = run_ml_enrichment(df, cfg) - merge_ml_into_payload(payload, ml_bundle) + local_bundle = run_local_enrichment(df, cfg) + llm_cfg = load_llm_config_from_db(db_path) + llm_bundle = run_llm_enrichment(df, llm_cfg, db_path=db_path) if llm_is_enabled(llm_cfg) else {} + bundle = merge_bundles(local_bundle, llm_bundle) + merge_analysis_into_payload(payload, bundle) write_report_payload(conn, payload) print("Enrich done. New report_payload row written.", flush=True) sys.exit(0) diff --git a/src/website_profiling/common.py b/src/website_profiling/common.py index 36720e4c..ac75d7ee 100644 --- a/src/website_profiling/common.py +++ b/src/website_profiling/common.py @@ -230,7 +230,7 @@ def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict: """Extract content analytics: word count, reading level, content-to-HTML ratio, top keywords. excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized - plain-text excerpt (truncated) in ``content_excerpt`` for ML / UI. + plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI. """ import re from collections import Counter diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py index 171fe69b..29c1779e 100644 --- a/src/website_profiling/db/storage.py +++ b/src/website_profiling/db/storage.py @@ -415,6 +415,19 @@ def init_schema(conn: sqlite3.Connection) -> None: is_unknown INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL ); + + CREATE TABLE IF NOT EXISTS llm_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + is_secret INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS llm_cache ( + cache_key TEXT PRIMARY KEY, + response_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); """) conn.commit() @@ -474,6 +487,52 @@ def write_pipeline_config( raise +def read_llm_config(conn: sqlite3.Connection) -> dict[str, str]: + """Return {key: value} from llm_config table (includes secrets).""" + try: + cur = conn.execute("SELECT key, value FROM llm_config ORDER BY key") + return {str(row["key"]): str(row["value"]) for row in cur.fetchall()} + except Exception: + return {} + + +def write_llm_config(conn: sqlite3.Connection, entries: dict[str, str], secret_keys: set[str] | None = None) -> None: + """Replace all llm_config rows. secret_keys marks keys stored with is_secret=1.""" + now = time.strftime("%Y-%m-%d %H:%M:%S") + secret_keys = secret_keys or set() + conn.execute("BEGIN") + try: + conn.execute("DELETE FROM llm_config") + for k, v in entries.items(): + is_secret = 1 if k in secret_keys else 0 + conn.execute( + "INSERT INTO llm_config (key, value, is_secret, updated_at) VALUES (?, ?, ?, ?)", + (str(k), str(v), is_secret, now), + ) + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + +def read_llm_cache(conn: sqlite3.Connection, cache_key: str) -> Optional[str]: + try: + cur = conn.execute("SELECT response_json FROM llm_cache WHERE cache_key = ?", (cache_key,)) + row = cur.fetchone() + return str(row["response_json"]) if row else None + except Exception: + return None + + +def write_llm_cache(conn: sqlite3.Connection, cache_key: str, response_json: str) -> None: + now = time.strftime("%Y-%m-%d %H:%M:%S") + conn.execute( + "INSERT OR REPLACE INTO llm_cache (cache_key, response_json, created_at) VALUES (?, ?, ?)", + (cache_key, response_json, now), + ) + conn.commit() + + def _crawl_results_has_run_id(conn: sqlite3.Connection) -> bool: """True if crawl_results exists and includes crawl_run_id (append-by-run crawls).""" try: diff --git a/src/website_profiling/llm/__init__.py b/src/website_profiling/llm/__init__.py new file mode 100644 index 00000000..59a56d38 --- /dev/null +++ b/src/website_profiling/llm/__init__.py @@ -0,0 +1,4 @@ +from .enrich import cluster_keywords_llm, run_llm_enrichment +from .base import get_llm_client + +__all__ = ["cluster_keywords_llm", "get_llm_client", "run_llm_enrichment"] diff --git a/src/website_profiling/llm/base.py b/src/website_profiling/llm/base.py new file mode 100644 index 00000000..b27dd5da --- /dev/null +++ b/src/website_profiling/llm/base.py @@ -0,0 +1,50 @@ +"""LLM provider abstraction for content enrichment.""" +from __future__ import annotations + +import json +import re +from typing import Any, Protocol + + +class LLMClient(Protocol): + def complete_json(self, system: str, user: str) -> dict[str, Any]: ... + + +def parse_json_response(text: str) -> dict[str, Any]: + text = (text or "").strip() + if not text: + return {} + try: + data = json.loads(text) + return data if isinstance(data, dict) else {"data": data} + except json.JSONDecodeError: + pass + m = re.search(r"\{[\s\S]*\}", text) + if m: + try: + data = json.loads(m.group(0)) + return data if isinstance(data, dict) else {"data": data} + except json.JSONDecodeError: + pass + return {} + + +def get_llm_client(cfg: dict[str, str]) -> LLMClient: + provider = (cfg.get("llm_provider") or "none").strip().lower() + if provider == "openai": + from .providers.openai import OpenAIClient + + return OpenAIClient(cfg) + if provider == "anthropic": + from .providers.anthropic import AnthropicClient + + return AnthropicClient(cfg) + if provider == "gemini": + from .providers.gemini import GeminiClient + + return GeminiClient(cfg) + if provider == "ollama": + from .providers.ollama import OllamaClient + + return OllamaClient(cfg) + raise ValueError(f"Unknown LLM provider: {provider}") diff --git a/src/website_profiling/llm/enrich.py b/src/website_profiling/llm/enrich.py new file mode 100644 index 00000000..f5d8efea --- /dev/null +++ b/src/website_profiling/llm/enrich.py @@ -0,0 +1,298 @@ +"""LLM-backed content enrichment (UI-configured via llm_config table).""" +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections import Counter +from typing import Any, Optional + +import pandas as pd + +from ..analysis.text import normalize_fingerprint_text +from ..llm_config import llm_is_enabled +from .base import get_llm_client +from .prompts import ( + KEYPHRASES_SYSTEM, + KEYWORD_CLUSTER_SYSTEM, + NER_SYSTEM, + PROMPT_VERSION, + SIMILAR_SYSTEM, +) + +LLM_INSTALL_HINT = "Install LLM dependencies: pip install -r requirements-llm.txt" + + +def _cfg_bool(cfg: dict[str, str] | None, key: str, default: bool = False) -> bool: + if not cfg: + return default + return str(cfg.get(key, default)).lower() in ("true", "1", "yes") + + +def _cfg_int(cfg: dict[str, str] | None, key: str, default: int) -> int: + if not cfg: + return default + raw = cfg.get(key) + if raw is None or str(raw).strip() == "": + return default + try: + return int(str(raw).strip()) + except ValueError: + return default + + +def _html_success_df(df: pd.DataFrame, max_pages: int) -> pd.DataFrame: + success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + if "content_type" in success.columns: + success = success[success["content_type"].fillna("").str.contains("text/html", case=False, na=False)] + return success.head(max_pages) + + +def _page_batch_items(df: pd.DataFrame, max_pages: int) -> list[dict[str, str]]: + items: list[dict[str, str]] = [] + for _, row in _html_success_df(df, max_pages).iterrows(): + u = str(row.get("url") or "").strip().rstrip("/") + text = normalize_fingerprint_text(row) + if not u or len(text) < 40: + continue + items.append({"url": u, "text": text[:4000]}) + return items + + +def _cache_key(task: str, model: str, payload: str) -> str: + h = hashlib.sha256(f"{PROMPT_VERSION}:{task}:{model}:{payload}".encode()).hexdigest() + return h + + +def _read_cache(db_path: Optional[str], key: str) -> Optional[dict[str, Any]]: + if not db_path or not os.path.isfile(db_path): + return None + try: + from ..db import db_session, init_schema + from ..db.storage import read_llm_cache + + with db_session(db_path) as conn: + init_schema(conn) + raw = read_llm_cache(conn, key) + if raw: + return json.loads(raw) + except Exception: + pass + return None + + +def _write_cache(db_path: Optional[str], key: str, data: dict[str, Any]) -> None: + if not db_path: + return + try: + from ..db import db_session, init_schema + from ..db.storage import write_llm_cache + + with db_session(db_path) as conn: + init_schema(conn) + write_llm_cache(conn, key, json.dumps(data)) + except Exception: + pass + + +def _call_cached( + client: Any, + task: str, + system: str, + user_payload: dict[str, Any], + cfg: dict[str, str], + db_path: Optional[str], +) -> dict[str, Any]: + model = (cfg.get("llm_model") or cfg.get("llm_provider") or "").strip() + payload_str = json.dumps(user_payload, sort_keys=True) + ck = _cache_key(task, model, payload_str) + cached = _read_cache(db_path, ck) + if cached is not None: + return cached + result = client.complete_json(system, json.dumps(user_payload)) + _write_cache(db_path, ck, result) + return result + + +def aggregate_ner_site_summary(spacy_by_url: dict[str, dict[str, Any]]) -> dict[str, Any]: + label_totals: Counter[str] = Counter() + total_entities = 0 + for _u, info in (spacy_by_url or {}).items(): + if not isinstance(info, dict): + continue + total_entities += int(info.get("entity_count") or 0) + for pair in info.get("top_entity_labels") or []: + if isinstance(pair, (list, tuple)) and len(pair) >= 2: + label_totals[str(pair[0])] += int(pair[1]) + return { + "label_counts": dict(label_totals.most_common(40)), + "pages_with_ner": len(spacy_by_url or {}), + "total_entities": total_entities, + } + + +def _run_ner( + client: Any, + items: list[dict[str, str]], + cfg: dict[str, str], + db_path: Optional[str], +) -> dict[str, dict[str, Any]]: + batch_size = max(1, _cfg_int(cfg, "llm_batch_size", 5)) + out: dict[str, dict[str, Any]] = {} + for i in range(0, len(items), batch_size): + batch = items[i : i + batch_size] + data = _call_cached(client, "ner", NER_SYSTEM, {"pages": batch}, cfg, db_path) + for p in data.get("pages") or []: + u = str(p.get("url") or "").strip().rstrip("/") + if not u: + continue + labels = p.get("top_entity_labels") or [] + out[u] = { + "entity_count": int(p.get("entity_count") or 0), + "top_entity_labels": labels, + } + return out + + +def _run_keyphrases( + client: Any, + items: list[dict[str, str]], + cfg: dict[str, str], + db_path: Optional[str], +) -> dict[str, dict[str, Any]]: + batch_size = max(1, _cfg_int(cfg, "llm_batch_size", 5)) + out: dict[str, dict[str, Any]] = {} + for i in range(0, len(items), batch_size): + batch = items[i : i + batch_size] + data = _call_cached(client, "keyphrases", KEYPHRASES_SYSTEM, {"pages": batch}, cfg, db_path) + for p in data.get("pages") or []: + u = str(p.get("url") or "").strip().rstrip("/") + if not u: + continue + phrases = p.get("phrases") or [] + pairs = [[str(x[0]), float(x[1])] for x in phrases if isinstance(x, (list, tuple)) and len(x) >= 2] + out[u] = {"phrases": pairs} + return out + + +def _run_similar_internal( + client: Any, + items: list[dict[str, str]], + cfg: dict[str, str], + db_path: Optional[str], +) -> dict[str, list[dict[str, Any]]]: + top_k = min(_cfg_int(cfg, "llm_similar_top_k", 5) or 5, 15) + all_urls = [x["url"] for x in items] + out: dict[str, list[dict[str, Any]]] = {} + batch_size = max(1, min(_cfg_int(cfg, "llm_batch_size", 5), 3)) + for i in range(0, len(items), batch_size): + batch = items[i : i + batch_size] + payload = { + "pages": batch, + "candidate_urls": all_urls[:80], + "top_k": top_k, + } + data = _call_cached(client, "similar", SIMILAR_SYSTEM, payload, cfg, db_path) + for p in data.get("pages") or []: + u = str(p.get("url") or "").strip().rstrip("/") + if not u: + continue + sim = [] + for s in (p.get("similar") or [])[:top_k]: + if isinstance(s, dict) and s.get("url"): + sim.append({"url": str(s["url"]), "score": round(float(s.get("score") or 0), 4)}) + if sim: + out[u] = sim + return out + + +def cluster_keywords_llm( + keywords: list[str], + cfg: dict[str, str] | None, + db_path: Optional[str] = None, +) -> list[dict[str, Any]]: + if not keywords or not cfg or not llm_is_enabled(cfg): + return [] + if not _cfg_bool(cfg, "llm_enable_keyword_clusters", False): + return [] + kws = keywords[:200] + if len(kws) < 2: + return [] + try: + client = get_llm_client(cfg) + data = _call_cached( + client, + "kw_clusters", + KEYWORD_CLUSTER_SYSTEM, + {"keywords": kws}, + cfg, + db_path, + ) + clusters = data.get("clusters") or [] + out: list[dict[str, Any]] = [] + for c in clusters: + if not isinstance(c, dict): + continue + words = c.get("keywords") or [] + if len(words) < 2: + continue + out.append( + { + "top_keyword": str(c.get("top_keyword") or words[0]), + "keywords": sorted(str(w) for w in words), + "cluster_score": round(float(c.get("cluster_score") or 0.9), 4), + } + ) + out.sort(key=lambda x: -x["cluster_score"]) + return out + except Exception as e: + raise RuntimeError(str(e)) from e + + +def run_llm_enrichment( + df: pd.DataFrame, + cfg: dict[str, str] | None, + db_path: Optional[str] = None, +) -> dict[str, Any]: + bundle: dict[str, Any] = { + "spacy_by_url": {}, + "similar_internal_by_url": {}, + "ner_site_summary": {}, + "keyphrases_by_url": {}, + "ml_errors": [], + } + if df.empty or not cfg or not llm_is_enabled(cfg): + return bundle + + max_pages = _cfg_int(cfg, "llm_max_pages", 60) or 60 + items = _page_batch_items(df, max_pages) + if not items: + return bundle + + try: + client = get_llm_client(cfg) + except Exception as e: + bundle["ml_errors"].append(str(e)) + return bundle + + if _cfg_bool(cfg, "llm_enable_ner", True): + try: + bundle["spacy_by_url"] = _run_ner(client, items, cfg, db_path) + except Exception as e: + bundle["ml_errors"].append(f"LLM NER: {e}") + + if _cfg_bool(cfg, "llm_enable_keyphrases", True): + try: + bundle["keyphrases_by_url"] = _run_keyphrases(client, items, cfg, db_path) + except Exception as e: + bundle["ml_errors"].append(f"LLM keyphrases: {e}") + + if _cfg_bool(cfg, "llm_enable_similar_internal", True): + try: + bundle["similar_internal_by_url"] = _run_similar_internal(client, items, cfg, db_path) + except Exception as e: + bundle["ml_errors"].append(f"LLM similar pages: {e}") + + bundle["ner_site_summary"] = aggregate_ner_site_summary(bundle.get("spacy_by_url") or {}) + return bundle diff --git a/src/website_profiling/llm/prompts.py b/src/website_profiling/llm/prompts.py new file mode 100644 index 00000000..cec4af83 --- /dev/null +++ b/src/website_profiling/llm/prompts.py @@ -0,0 +1,20 @@ +"""Versioned prompts for LLM enrichment tasks.""" +from __future__ import annotations + +PROMPT_VERSION = "v1" + +NER_SYSTEM = """You extract named entities from web page text for SEO analysis. +Return JSON: {"pages": [{"url": "...", "entity_count": N, "top_entity_labels": [["ORG", 2], ["PERSON", 1]]}]} +Use standard NER labels (ORG, PERSON, GPE, PRODUCT, etc.). Count occurrences per label.""" + +KEYPHRASES_SYSTEM = """You extract SEO keyphrases from web page content. +Return JSON: {"pages": [{"url": "...", "phrases": [["phrase text", 0.95], ...]}]} +Provide 3-8 phrases per page with scores 0-1.""" + +SIMILAR_SYSTEM = """You find semantically similar internal pages for SEO deduplication review. +Return JSON: {"pages": [{"url": "...", "similar": [{"url": "...", "score": 0.87}, ...]}]} +Scores 0-1; only include URLs from the provided candidate list.""" + +KEYWORD_CLUSTER_SYSTEM = """You group related SEO keywords into semantic clusters. +Return JSON: {"clusters": [{"top_keyword": "...", "keywords": ["a","b"], "cluster_score": 0.9}]} +Only merge clearly related terms; omit singletons.""" diff --git a/src/website_profiling/llm/providers/__init__.py b/src/website_profiling/llm/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/website_profiling/llm/providers/anthropic.py b/src/website_profiling/llm/providers/anthropic.py new file mode 100644 index 00000000..ce7e8467 --- /dev/null +++ b/src/website_profiling/llm/providers/anthropic.py @@ -0,0 +1,35 @@ +"""Anthropic Messages API.""" +from __future__ import annotations + +from typing import Any + +from ..base import parse_json_response + + +class AnthropicClient: + def __init__(self, cfg: dict[str, str]) -> None: + self._cfg = cfg + self._model = (cfg.get("llm_model") or "claude-3-5-haiku-latest").strip() + self._timeout = float(cfg.get("llm_timeout_s") or 120) + self._api_key = (cfg.get("llm_api_key") or "").strip() + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + if not self._api_key: + raise RuntimeError("Anthropic API key missing. Set it in the AI tab or ANTHROPIC_API_KEY.") + try: + import anthropic + except ImportError as e: + raise ImportError("pip install anthropic (or requirements-llm.txt)") from e + + client = anthropic.Anthropic(api_key=self._api_key, timeout=self._timeout) + msg = client.messages.create( + model=self._model, + max_tokens=4096, + system=system + "\nRespond with valid JSON only.", + messages=[{"role": "user", "content": user}], + ) + parts = [] + for block in msg.content: + if getattr(block, "type", None) == "text": + parts.append(block.text) + return parse_json_response("\n".join(parts)) diff --git a/src/website_profiling/llm/providers/gemini.py b/src/website_profiling/llm/providers/gemini.py new file mode 100644 index 00000000..08b839b1 --- /dev/null +++ b/src/website_profiling/llm/providers/gemini.py @@ -0,0 +1,36 @@ +"""Google Gemini generateContent API.""" +from __future__ import annotations + +from typing import Any + +from ..base import parse_json_response + + +class GeminiClient: + def __init__(self, cfg: dict[str, str]) -> None: + self._model = (cfg.get("llm_model") or "gemini-2.0-flash").strip() + self._timeout = float(cfg.get("llm_timeout_s") or 120) + self._api_key = (cfg.get("llm_api_key") or "").strip() + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + if not self._api_key: + raise RuntimeError("Gemini API key missing. Set it in the AI tab or GEMINI_API_KEY.") + try: + import httpx + except ImportError as e: + raise ImportError("pip install httpx (or requirements-llm.txt)") from e + + url = f"https://generativelanguage.googleapis.com/v1beta/models/{self._model}:generateContent" + payload = { + "contents": [{"parts": [{"text": f"{system}\n\n{user}\n\nRespond with valid JSON only."}]}], + "generationConfig": {"responseMimeType": "application/json", "temperature": 0.2}, + } + with httpx.Client(timeout=self._timeout) as client: + r = client.post(url, params={"key": self._api_key}, json=payload) + r.raise_for_status() + data = r.json() + text = "" + for cand in data.get("candidates") or []: + for part in (cand.get("content") or {}).get("parts") or []: + text += part.get("text") or "" + return parse_json_response(text) diff --git a/src/website_profiling/llm/providers/ollama.py b/src/website_profiling/llm/providers/ollama.py new file mode 100644 index 00000000..47c77cef --- /dev/null +++ b/src/website_profiling/llm/providers/ollama.py @@ -0,0 +1,37 @@ +"""Ollama local chat API.""" +from __future__ import annotations + +import json +from typing import Any + +from ..base import parse_json_response + + +class OllamaClient: + def __init__(self, cfg: dict[str, str]) -> None: + self._model = (cfg.get("llm_model") or "llama3.2").strip() + self._timeout = float(cfg.get("llm_timeout_s") or 120) + self._base = (cfg.get("llm_base_url") or "http://127.0.0.1:11434").strip().rstrip("/") + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + try: + import httpx + except ImportError as e: + raise ImportError("pip install httpx (or requirements-llm.txt)") from e + + payload = { + "model": self._model, + "stream": False, + "format": "json", + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + } + url = f"{self._base}/api/chat" + with httpx.Client(timeout=self._timeout) as client: + r = client.post(url, json=payload) + r.raise_for_status() + data = r.json() + content = (data.get("message") or {}).get("content") or "" + return parse_json_response(content if isinstance(content, str) else json.dumps(content)) diff --git a/src/website_profiling/llm/providers/openai.py b/src/website_profiling/llm/providers/openai.py new file mode 100644 index 00000000..273b7031 --- /dev/null +++ b/src/website_profiling/llm/providers/openai.py @@ -0,0 +1,42 @@ +"""OpenAI-compatible chat completions with JSON output.""" +from __future__ import annotations + +import json +from typing import Any + +from ..base import parse_json_response + + +class OpenAIClient: + def __init__(self, cfg: dict[str, str]) -> None: + self._cfg = cfg + self._model = (cfg.get("llm_model") or "gpt-4o-mini").strip() + self._timeout = float(cfg.get("llm_timeout_s") or 120) + self._api_key = (cfg.get("llm_api_key") or "").strip() + self._base = (cfg.get("llm_base_url") or "https://api.openai.com/v1").strip().rstrip("/") + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + if not self._api_key: + raise RuntimeError("OpenAI API key missing. Set it in the AI tab or OPENAI_API_KEY.") + try: + import httpx + except ImportError as e: + raise ImportError("pip install httpx (or requirements-llm.txt)") from e + + payload = { + "model": self._model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "response_format": {"type": "json_object"}, + "temperature": 0.2, + } + headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"} + url = f"{self._base}/chat/completions" + with httpx.Client(timeout=self._timeout) as client: + r = client.post(url, headers=headers, json=payload) + r.raise_for_status() + data = r.json() + content = data["choices"][0]["message"]["content"] + return parse_json_response(content if isinstance(content, str) else json.dumps(content)) diff --git a/src/website_profiling/llm_config.py b/src/website_profiling/llm_config.py new file mode 100644 index 00000000..ed82b34f --- /dev/null +++ b/src/website_profiling/llm_config.py @@ -0,0 +1,52 @@ +""" +Load LLM settings from report.db llm_config table only (UI-managed). +Not read from pipeline-config.txt or --config files. +""" +from __future__ import annotations + +import os +from typing import Optional + +_ENV_KEY_BY_PROVIDER = { + "openai": "OPENAI_API_KEY", + "gemini": "GEMINI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", +} + + +def load_llm_config_from_db(db_path: str) -> dict[str, str]: + if not db_path or not os.path.isfile(db_path): + return {} + try: + from .db import db_session, init_schema + from .db.storage import read_llm_config + + with db_session(db_path) as conn: + init_schema(conn) + cfg = read_llm_config(conn) + except Exception: + return {} + + if not cfg: + return {} + + provider = (cfg.get("llm_provider") or "none").strip().lower() + if provider and provider != "none": + if not (cfg.get("llm_api_key") or "").strip(): + env_var = _ENV_KEY_BY_PROVIDER.get(provider) + if env_var: + env_val = (os.environ.get(env_var) or "").strip() + if env_val: + cfg = dict(cfg) + cfg["llm_api_key"] = env_val + cfg["_llm_api_key_source"] = "env" + return cfg + + +def llm_is_enabled(cfg: dict[str, str]) -> bool: + if not cfg: + return False + if str(cfg.get("llm_enabled", "")).lower() not in ("true", "1", "yes"): + return False + provider = (cfg.get("llm_provider") or "none").strip().lower() + return provider not in ("", "none") diff --git a/src/website_profiling/ml/__init__.py b/src/website_profiling/ml/__init__.py deleted file mode 100644 index 3dc18aca..00000000 --- a/src/website_profiling/ml/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Optional ML enrichment.""" diff --git a/src/website_profiling/ml/enrich.py b/src/website_profiling/ml/enrich.py deleted file mode 100644 index 92b71b76..00000000 --- a/src/website_profiling/ml/enrich.py +++ /dev/null @@ -1,774 +0,0 @@ -""" -Optional ML/NLP enrichment for crawl reports. All features are gated by config flags. - -Install extras: pip install -r requirements-ml.txt -""" -from __future__ import annotations - -import hashlib -import json -import os -import re -from collections import Counter, defaultdict -from typing import Any, Optional - -import pandas as pd - -ML_INSTALL_HINT = "Install optional ML dependencies: pip install -r requirements-ml.txt" - - -def _cfg_bool(cfg: dict[str, str] | None, key: str, default: bool = False) -> bool: - if not cfg: - return default - return str(cfg.get(key, default)).lower() in ("true", "1", "yes") - - -def _cfg_int(cfg: dict[str, str] | None, key: str, default: int) -> int: - if not cfg: - return default - raw = cfg.get(key) - if raw is None or str(raw).strip() == "": - return default - try: - return int(str(raw).strip()) - except ValueError: - return default - - -def _top_keywords_as_text(row: pd.Series, max_terms: int = 15) -> str: - if "top_keywords" not in row.index: - return "" - raw = row.get("top_keywords") - if raw is None or (isinstance(raw, float) and pd.isna(raw)): - return "" - s = str(raw).strip() - if not s or s == "[]": - return "" - try: - arr = json.loads(s) - if not isinstance(arr, list): - return "" - words: list[str] = [] - for item in arr[:max_terms]: - if isinstance(item, dict) and item.get("word"): - words.append(str(item["word"])) - return " ".join(words) - except json.JSONDecodeError: - return "" - - -def _normalize_fingerprint_text(row: pd.Series) -> str: - """Concatenate all available on-page text signals for ML (duplicates, ST, langdetect, spaCy).""" - parts: list[str] = [] - for col in ( - "title", - "h1", - "meta_description", - "heading_sequence", - "og_title", - "og_description", - "twitter_title", - "content_excerpt", - ): - if col not in row.index: - continue - v = row.get(col) - if v is None or (isinstance(v, float) and pd.isna(v)): - continue - s = str(v).strip() - if s: - parts.append(s) - kw_extra = _top_keywords_as_text(row) - if kw_extra: - parts.append(kw_extra) - t = " ".join(parts).lower() - t = re.sub(r"\s+", " ", t) - return t[:12000] - - -def _tokenize_simhash(text: str) -> list[str]: - return re.findall(r"[a-z0-9]{3,}", text.lower()) - - -def _stable_token_hash(token: str) -> int: - return int.from_bytes(hashlib.md5(token.encode("utf-8")).digest()[:8], "little") - - -def simhash_64(text: str) -> int: - """64-bit SimHash for near-duplicate detection (exact bucket grouping; optional Hamming merge).""" - tokens = _tokenize_simhash(text) - if not tokens: - return 0 - vec = [0] * 64 - for tok in tokens: - h = _stable_token_hash(tok) - for i in range(64): - if (h >> i) & 1: - vec[i] += 1 - else: - vec[i] -= 1 - out = 0 - for i in range(64): - if vec[i] > 0: - out |= 1 << i - return out - - -def _hamming(a: int, b: int) -> int: - x = a ^ b - c = 0 - while x: - c += x & 1 - x >>= 1 - return c - - -def _import_rapidfuzz(): - try: - from rapidfuzz import fuzz - - return fuzz - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - -def _import_sklearn(): - try: - from sklearn.ensemble import IsolationForest - from sklearn.preprocessing import StandardScaler - - return IsolationForest, StandardScaler - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - -def _import_langdetect(): - try: - from langdetect import LangDetectException, detect - - return detect, LangDetectException - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - -def _import_sentence_transformers(): - # Before importing HF stack: hide safetensors "LOAD REPORT" / weight-key chatter on stderr. - os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") - try: - from sentence_transformers import SentenceTransformer - - return SentenceTransformer - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - -def _import_spacy(): - try: - import spacy - - return spacy - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - -def compute_duplicate_groups( - df: pd.DataFrame, - cfg: dict[str, str] | None, -) -> tuple[list[dict[str, Any]], dict[str, str]]: - """ - SimHash exact groups + optional rapidfuzz merge (high token_set_ratio). - Returns (groups for payload, url -> group_id). - """ - if df.empty or not _cfg_bool(cfg, "enable_duplicate_detection", False): - return [], {} - - success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - if "content_type" in success.columns: - success = success[success["content_type"].fillna("").str.contains("text/html", case=False, na=False)] - max_pages = _cfg_int(cfg, "ml_dup_max_pages", 2000) or 2000 - success = success.head(max_pages) - - url_to_fp: dict[str, str] = {} - url_to_sh: dict[str, int] = {} - for _, row in success.iterrows(): - u = str(row.get("url") or "").strip().rstrip("/") - if not u: - continue - fp = _normalize_fingerprint_text(row) - if len(fp) < 20: - continue - url_to_fp[u] = fp - url_to_sh[u] = simhash_64(fp) - - # Exact SimHash buckets - bucket: dict[int, list[str]] = defaultdict(list) - for u, h in url_to_sh.items(): - bucket[h].append(u) - - fuzz = _import_rapidfuzz() - fuzzy_threshold = _cfg_int(cfg, "ml_fuzzy_threshold", 92) or 92 - hamming_max = _cfg_int(cfg, "ml_simhash_hamming", 0) or 0 - - # Union-find - parent: dict[str, str] = {} - - def find(x: str) -> str: - if x not in parent: - parent[x] = x - if parent[x] != x: - parent[x] = find(parent[x]) - return parent[x] - - def union(a: str, b: str) -> None: - ra, rb = find(a), find(b) - if ra != rb: - parent[rb] = ra - - urls = list(url_to_fp.keys()) - for u in urls: - parent.setdefault(u, u) - - # Merge exact simhash - for h, members in bucket.items(): - if len(members) < 2: - continue - base = members[0] - for m in members[1:]: - union(base, m) - - # Hamming-close simhash (optional, O(n^2) capped) - if hamming_max > 0 and len(urls) <= 800: - sh_list = [(u, url_to_sh[u]) for u in urls] - for i, (u1, h1) in enumerate(sh_list): - for u2, h2 in sh_list[i + 1 :]: - if _hamming(h1, h2) <= hamming_max: - union(u1, u2) - - # Optional: sentence-transformer cosine on fingerprint text — only merge fuzzy candidates above this similarity - embed_norm: dict[str, Any] = {} - if _cfg_bool(cfg, "enable_embedding_duplicate_refine", False) and len(urls) <= 600 and len(urls) >= 2: - try: - import numpy as np - - ST = _import_sentence_transformers() - model_name = (cfg or {}).get("ml_sentence_model", "all-MiniLM-L6-v2").strip() or "all-MiniLM-L6-v2" - model = ST(model_name) - texts = [url_to_fp[u][:4000] for u in urls] - verbose = _cfg_bool(cfg, "ml_verbose", False) - emb = model.encode( - texts, - show_progress_bar=verbose, - batch_size=32, - convert_to_numpy=True, - ) - norms = np.linalg.norm(emb, axis=1, keepdims=True) - norms[norms == 0] = 1e-12 - e = emb / norms - for i, u in enumerate(urls): - embed_norm[u] = e[i] - except ImportError: - embed_norm = {} - - # Fuzzy title fingerprint merge (pairwise cap) - if len(urls) <= 600: - import numpy as np - - min_embed = float(_cfg_int(cfg, "ml_dup_embed_min_pct", 88) or 88) / 100.0 - for i, u1 in enumerate(urls): - fp1 = url_to_fp.get(u1, "") - for u2 in urls[i + 1 :]: - fp2 = url_to_fp.get(u2, "") - if not fp1 or not fp2: - continue - if fuzz.token_set_ratio(fp1, fp2) >= fuzzy_threshold: - if embed_norm: - v1 = embed_norm.get(u1) - v2 = embed_norm.get(u2) - if v1 is not None and v2 is not None and float(np.dot(v1, v2)) >= min_embed: - union(u1, u2) - else: - union(u1, u2) - - clusters: dict[str, list[str]] = defaultdict(list) - for u in urls: - clusters[find(u)].append(u) - - groups_out: list[dict[str, Any]] = [] - url_to_gid: dict[str, str] = {} - gid = 0 - max_groups = 200 - for root, members in clusters.items(): - if len(members) < 2: - continue - members = sorted(set(members)) - rep = members[0] - methods = [] - hashes = {url_to_sh.get(m) for m in members} - if len(hashes) == 1: - methods.append("simhash") - if len(members) > 1 and len(hashes) > 1: - methods.append("fuzzy") - if not methods: - methods.append("simhash") - gkey = f"dup_{gid}" - gid += 1 - groups_out.append( - { - "id": gkey, - "representative_url": rep, - "member_urls": members[:100], - "member_count": len(members), - "methods": methods, - } - ) - for m in members: - url_to_gid[m] = gkey - if gid >= max_groups: - break - - return groups_out[:max_groups], url_to_gid - - -def compute_anomalies(df: pd.DataFrame, cfg: dict[str, str] | None) -> list[dict[str, Any]]: - if df.empty or not _cfg_bool(cfg, "enable_anomaly_urls", False): - return [] - - IsolationForest, StandardScaler = _import_sklearn() - rows: list[dict[str, Any]] = [] - feat_rows: list[list[float]] = [] - - def _pa_int(row: pd.Series, key: str) -> int: - if "page_analysis" not in row.index: - return 0 - raw = row.get("page_analysis") - if raw is None or (isinstance(raw, float) and pd.isna(raw)): - return 0 - try: - obj = json.loads(str(raw)) if isinstance(raw, str) else raw - if isinstance(obj, dict): - return int(obj.get(key) or 0) - except (json.JSONDecodeError, TypeError, ValueError): - pass - return 0 - - for _, row in df.iterrows(): - u = str(row.get("url") or "").strip().rstrip("/") - if not u: - continue - st = str(row.get("status") or "") - ok = bool(re.match(r"2\d{2}", st)) - wc = float(pd.to_numeric(row.get("word_count"), errors="coerce") or 0) - cl = float(pd.to_numeric(row.get("content_length"), errors="coerce") or 0) - rt = float(pd.to_numeric(row.get("response_time_ms"), errors="coerce") or 0) - ol = float(pd.to_numeric(row.get("outlinks"), errors="coerce") or 0) - rl = float(pd.to_numeric(row.get("reading_level"), errors="coerce") or 0) - chratio = float(pd.to_numeric(row.get("content_html_ratio"), errors="coerce") or 0) - h1c = float(pd.to_numeric(row.get("h1_count"), errors="coerce") or 0) - mdlen = float(pd.to_numeric(row.get("meta_description_len"), errors="coerce") or 0) - il = float(_pa_int(row, "internal_link_count")) - el = float(_pa_int(row, "external_link_count")) - feat_rows.append([wc, cl, rt, ol, rl, chratio, h1c, mdlen, il, el, 1.0 if ok else 0.0]) - rows.append({"url": u, "status": st}) - - if len(feat_rows) < 10: - return [] - - scaler = StandardScaler() - X = scaler.fit_transform(feat_rows) - iso = IsolationForest(random_state=42, contamination="auto", n_estimators=128) - pred = iso.fit_predict(X) - scores = iso.decision_function(X) - - out: list[dict[str, Any]] = [] - for i, p in enumerate(pred): - if p != -1: - continue - r = rows[i] - f = feat_rows[i] - reasons = [] - if f[2] > 3000: - reasons.append("high_response_time_ms") - if f[0] < 50 and f[1] > 500: - reasons.append("low_word_count_high_html") - if f[3] > 200: - reasons.append("very_high_outlinks") - if f[8] == 0 and r["status"].startswith("2"): - reasons.append("zero_internal_links_in_analysis") - out.append( - { - "url": r["url"], - "anomaly_score": round(float(scores[i]), 4), - "reasons": reasons or ["multivariate_outlier"], - } - ) - out.sort(key=lambda x: x["anomaly_score"]) - return out[:150] - - -def compute_language_signals(df: pd.DataFrame, cfg: dict[str, str] | None) -> tuple[dict[str, str], dict[str, Any]]: - if df.empty or not _cfg_bool(cfg, "enable_language_detection", False): - return {}, {"counts": {}, "mixed_site": False} - - detect, LangDetectException = _import_langdetect() - by_url: dict[str, str] = {} - for _, row in df.iterrows(): - u = str(row.get("url") or "").strip().rstrip("/") - if not u: - continue - st = str(row.get("status") or "") - if not re.match(r"2\d{2}", st): - continue - text = _normalize_fingerprint_text(row) - if len(text) < 30: - continue - try: - lang = detect(text[:2000]) - by_url[u] = lang - except LangDetectException: - continue - - counts = dict(Counter(by_url.values()).most_common(20)) - mixed = len(counts) > 1 - summary = {"counts": counts, "mixed_site": mixed, "detected_pages": len(by_url)} - return by_url, summary - - -def compute_spacy_signals(df: pd.DataFrame, cfg: dict[str, str] | None) -> dict[str, dict[str, Any]]: - if df.empty or not _cfg_bool(cfg, "enable_ner_spacy", False): - return {} - - spacy = _import_spacy() - try: - nlp = spacy.load("en_core_web_sm") - except OSError: - raise ImportError( - "spaCy English model missing. Install ML deps (includes en-core-web-sm): " - "pip install -r requirements-ml.txt — or: python -m spacy download en_core_web_sm" - ) from None - - max_pages = _cfg_int(cfg, "ml_ner_max_pages", 80) or 80 - success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - out: dict[str, dict[str, Any]] = {} - n = 0 - for _, row in success.iterrows(): - if n >= max_pages: - break - u = str(row.get("url") or "").strip().rstrip("/") - text = _normalize_fingerprint_text(row) - if len(text) < 40: - continue - doc = nlp(text[:50000]) - labels = [e.label_ for e in doc.ents] - lc = Counter(labels) - out[u] = { - "entity_count": len(doc.ents), - "top_entity_labels": [list(x) for x in lc.most_common(8)], - } - n += 1 - return out - - -def aggregate_ner_site_summary(spacy_by_url: dict[str, dict[str, Any]]) -> dict[str, Any]: - """Roll up spaCy NER label counts across pages for site-level charts.""" - label_totals: Counter[str] = Counter() - total_entities = 0 - for _u, info in (spacy_by_url or {}).items(): - if not isinstance(info, dict): - continue - total_entities += int(info.get("entity_count") or 0) - for pair in info.get("top_entity_labels") or []: - if isinstance(pair, (list, tuple)) and len(pair) >= 2: - label_totals[str(pair[0])] += int(pair[1]) - elif isinstance(pair, (list, tuple)) and len(pair) == 1: - label_totals[str(pair[0])] += 1 - return { - "label_counts": dict(label_totals.most_common(40)), - "pages_with_ner": len(spacy_by_url or {}), - "total_entities": total_entities, - } - - -def compute_similar_internal( - df: pd.DataFrame, - cfg: dict[str, str] | None, -) -> dict[str, list[dict[str, Any]]]: - if df.empty or not _cfg_bool(cfg, "enable_semantic_similar_internal", False): - return {} - - ST = _import_sentence_transformers() - model_name = (cfg or {}).get("ml_sentence_model", "all-MiniLM-L6-v2").strip() or "all-MiniLM-L6-v2" - model = ST(model_name) - - max_pages = _cfg_int(cfg, "ml_max_pages_st", 400) or 400 - top_k = min(_cfg_int(cfg, "ml_similar_top_k", 5) or 5, 15) - - success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - if "content_type" in success.columns: - success = success[success["content_type"].fillna("").str.contains("text/html", case=False, na=False)] - - urls: list[str] = [] - texts: list[str] = [] - for _, row in success.head(max_pages).iterrows(): - u = str(row.get("url") or "").strip().rstrip("/") - t = _normalize_fingerprint_text(row) - if not u or len(t) < 15: - continue - urls.append(u) - texts.append(t[:2000]) - - if len(urls) < 2: - return {} - - verbose = _cfg_bool(cfg, "ml_verbose", False) - emb = model.encode(texts, show_progress_bar=verbose, batch_size=32, convert_to_numpy=True) - # cosine similarity via normalized vectors - import numpy as np - - norms = np.linalg.norm(emb, axis=1, keepdims=True) - norms[norms == 0] = 1e-12 - e = emb / norms - sim = e @ e.T - - result: dict[str, list[dict[str, Any]]] = {} - n = len(urls) - for i in range(n): - scores = [(sim[i, j], j) for j in range(n) if j != i] - scores.sort(reverse=True) - result[urls[i]] = [ - {"url": urls[j], "score": round(float(s), 4)} for s, j in scores[:top_k] - ] - return result - - -def compute_keyphrases_by_url( - df: pd.DataFrame, - cfg: dict[str, str] | None, -) -> dict[str, dict[str, Any]]: - """KeyBERT keyphrases per URL (uses same SentenceTransformer as semantic features).""" - if df.empty or not _cfg_bool(cfg, "enable_keybert", False): - return {} - try: - from keybert import KeyBERT - except ImportError as e: - raise ImportError(f"{ML_INSTALL_HINT}\n({e})") from e - - ST = _import_sentence_transformers() - model_name = (cfg or {}).get("ml_sentence_model", "all-MiniLM-L6-v2").strip() or "all-MiniLM-L6-v2" - st_model = ST(model_name) - kw_model = KeyBERT(model=st_model) - max_pages = _cfg_int(cfg, "ml_keybert_max_pages", 60) or 60 - top_n = _cfg_int(cfg, "ml_keybert_top_n", 8) or 8 - - success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - if "content_type" in success.columns: - success = success[success["content_type"].fillna("").str.contains("text/html", case=False, na=False)] - - out: dict[str, dict[str, Any]] = {} - n = 0 - for _, row in success.iterrows(): - if n >= max_pages: - break - u = str(row.get("url") or "").strip().rstrip("/") - text = _normalize_fingerprint_text(row) - if len(text) < 40: - continue - try: - kws = kw_model.extract_keywords( - text[:12000], - keyphrase_ngram_range=(1, 2), - stop_words="english", - top_n=top_n, - use_mmr=True, - diversity=0.5, - ) - except Exception: - continue - pairs = [[str(k[0]), float(k[1])] for k in kws] if kws else [] - out[u] = {"phrases": pairs} - n += 1 - return out - - -def merge_ml_into_payload(payload: dict[str, Any], ml_bundle: dict[str, Any]) -> None: - """Mutate report payload dict in place with ML fields and per-link merge.""" - payload["content_duplicates"] = ml_bundle.get("content_duplicates") or [] - payload["anomalies"] = ml_bundle.get("anomalies") or [] - payload["language_summary"] = ml_bundle.get("language_summary") or {} - ns = ml_bundle.get("ner_site_summary") or {} - if ns: - payload["ner_site_summary"] = ns - else: - payload.pop("ner_site_summary", None) - err = ml_bundle.get("ml_errors") or [] - if err: - payload["ml_errors"] = err - else: - payload.pop("ml_errors", None) - - dup_gid = ml_bundle.get("url_duplicate_group_id") or {} - sim_map = ml_bundle.get("similar_internal_by_url") or {} - lang_map = ml_bundle.get("language_by_url") or {} - spacy_map = ml_bundle.get("spacy_by_url") or {} - kp_map = ml_bundle.get("keyphrases_by_url") or {} - anomalies_list = ml_bundle.get("anomalies") or [] - anomaly_by_url = {str(a.get("url") or "").strip().rstrip("/"): a for a in anomalies_list if a.get("url")} - - for rec in payload.get("links") or []: - if not isinstance(rec, dict): - continue - u = str(rec.get("url") or "").strip() - uk = u.rstrip("/") - rec.pop("duplicate_group_id", None) - rec.pop("similar_internal", None) - rec.pop("detected_language", None) - rec.pop("nlp_entities", None) - rec.pop("ml_anomaly", None) - rec.pop("keyphrases", None) - if uk in dup_gid: - rec["duplicate_group_id"] = dup_gid[uk] - nei = sim_map.get(uk) or sim_map.get(u) - if nei: - rec["similar_internal"] = list(nei) - if uk in lang_map: - rec["detected_language"] = lang_map[uk] - if uk in spacy_map: - rec["nlp_entities"] = spacy_map[uk] - if uk in anomaly_by_url: - rec["ml_anomaly"] = anomaly_by_url[uk] - if uk in kp_map: - rec["keyphrases"] = kp_map[uk] - pa = rec.get("page_analysis") - if isinstance(pa, dict): - sig = pa.get("signals") - if isinstance(sig, dict): - sig.pop("language", None) - sig.pop("nlp_entities", None) - if not sig: - pa.pop("signals", None) - if uk in lang_map: - pa.setdefault("signals", {})["language"] = lang_map[uk] - if uk in spacy_map: - pa.setdefault("signals", {})["nlp_entities"] = spacy_map[uk] - - -def run_ml_enrichment(df: pd.DataFrame, cfg: dict[str, str] | None) -> dict[str, Any]: - """ - Run all enabled enrichment steps. Returns a dict with keys for merging into report payload / per-URL maps. - """ - bundle: dict[str, Any] = { - "content_duplicates": [], - "url_duplicate_group_id": {}, - "anomalies": [], - "language_by_url": {}, - "language_summary": {"counts": {}, "mixed_site": False}, - "spacy_by_url": {}, - "similar_internal_by_url": {}, - "ner_site_summary": {}, - "keyphrases_by_url": {}, - } - - if df.empty: - return bundle - - try: - dups, url_gid = compute_duplicate_groups(df, cfg) - bundle["content_duplicates"] = dups - bundle["url_duplicate_group_id"] = url_gid - except ImportError as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - try: - bundle["anomalies"] = compute_anomalies(df, cfg) - except ImportError as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - try: - lang_map, lang_summary = compute_language_signals(df, cfg) - bundle["language_by_url"] = lang_map - bundle["language_summary"] = lang_summary - except ImportError as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - try: - bundle["spacy_by_url"] = compute_spacy_signals(df, cfg) - except (ImportError, OSError) as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - bundle["ner_site_summary"] = aggregate_ner_site_summary(bundle.get("spacy_by_url") or {}) - - try: - bundle["similar_internal_by_url"] = compute_similar_internal(df, cfg) - except ImportError as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - try: - bundle["keyphrases_by_url"] = compute_keyphrases_by_url(df, cfg) - except ImportError as e: - bundle["ml_errors"] = bundle.get("ml_errors", []) + [str(e)] - - return bundle - - -def cluster_keywords_semantic( - keywords: list[str], - cfg: dict[str, str] | None, -) -> list[dict[str, Any]]: - """Cluster keyword strings by embedding similarity (cosine).""" - if not keywords or not _cfg_bool(cfg, "enable_semantic_keywords", False): - return [] - - ST = _import_sentence_transformers() - model_name = (cfg or {}).get("ml_sentence_model", "all-MiniLM-L6-v2").strip() or "all-MiniLM-L6-v2" - model = ST(model_name) - max_kw = _cfg_int(cfg, "ml_semantic_keyword_max", 200) or 200 - kws = keywords[:max_kw] - if len(kws) < 2: - return [] - - import numpy as np - - verbose = _cfg_bool(cfg, "ml_verbose", False) - emb = model.encode(kws, show_progress_bar=verbose, batch_size=64, convert_to_numpy=True) - norms = np.linalg.norm(emb, axis=1, keepdims=True) - norms[norms == 0] = 1e-12 - e = emb / norms - sim = e @ e.T - - threshold = float(_cfg_int(cfg, "ml_keyword_cluster_sim", 75) or 75) / 100.0 - parent = {i: i for i in range(len(kws))} - - def find(x: int) -> int: - if parent[x] != x: - parent[x] = find(parent[x]) - return parent[x] - - def union(a: int, b: int) -> None: - ra, rb = find(a), find(b) - if ra != rb: - parent[rb] = ra - - for i in range(len(kws)): - for j in range(i + 1, len(kws)): - if sim[i, j] >= threshold: - union(i, j) - - clusters: dict[int, list[int]] = defaultdict(list) - for i in range(len(kws)): - clusters[find(i)].append(i) - - out: list[dict[str, Any]] = [] - for _, idxs in clusters.items(): - if len(idxs) < 2: - continue - words = [kws[i] for i in idxs] - out.append( - { - "top_keyword": words[0], - "keywords": sorted(words), - "cluster_score": round(float(np.mean([sim[idxs[0], j] for j in idxs[1:]])), 4) - if len(idxs) > 1 - else 1.0, - } - ) - out.sort(key=lambda x: -x["cluster_score"]) - return out diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index 1a0025bc..0408a239 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -27,7 +27,9 @@ ) from ..tools.keywords import cluster_keywords, extract_candidates_from_df, score_keywords from ..config import get_bool, get_int -from ..ml.enrich import cluster_keywords_semantic, run_ml_enrichment +from ..analysis import merge_bundles, run_local_enrichment +from ..llm.enrich import cluster_keywords_llm, run_llm_enrichment +from ..llm_config import load_llm_config_from_db, llm_is_enabled from .categories import build_categories from ..security_scanner import run_security_scan @@ -971,8 +973,11 @@ def run_simple_report( with open(security_findings_output, "w", encoding="utf-8") as fh: json.dump(security_findings, fh, indent=2, default=str) - print(" ML enrichment (optional)...", flush=True) - ml_bundle = run_ml_enrichment(df, config) + print(" Content analysis (local + optional LLM)...", flush=True) + local_bundle = run_local_enrichment(df, config) + llm_cfg = load_llm_config_from_db(db_path) if db_path else {} + llm_bundle = run_llm_enrichment(df, llm_cfg, db_path=db_path) if llm_is_enabled(llm_cfg) else {} + ml_bundle = merge_bundles(local_bundle, llm_bundle) print(" Building report categories...", flush=True) if not db_path: @@ -1087,8 +1092,6 @@ def run_simple_report( sim_map = ml_bundle.get("similar_internal_by_url") or {} lang_map = ml_bundle.get("language_by_url") or {} spacy_map = ml_bundle.get("spacy_by_url") or {} - anomalies_list = ml_bundle.get("anomalies") or [] - anomaly_by_url = {str(a.get("url") or "").strip().rstrip("/"): a for a in anomalies_list if a.get("url")} kp_map = ml_bundle.get("keyphrases_by_url") or {} # Full links list: every crawled URL with url, status, inlinks, title, content_length, depth @@ -1248,8 +1251,6 @@ def _bool_col(col): rec["detected_language"] = lang_map[uk] if uk in spacy_map: rec["nlp_entities"] = spacy_map[uk] - if uk in anomaly_by_url: - rec["ml_anomaly"] = anomaly_by_url[uk] if uk in kp_map: rec["keyphrases"] = kp_map[uk] @@ -1326,11 +1327,14 @@ def _bool_col(col): print(" Building content analytics...", flush=True) content_analytics = _build_content_analytics(df) semantic_keyword_clusters: list[dict[str, Any]] = [] - if get_bool(config or {}, "enable_semantic_keywords", False): + llm_cfg_for_clusters = load_llm_config_from_db(db_path) if db_path else {} + if db_path and llm_is_enabled(llm_cfg_for_clusters): try: - words = [x["word"] for x in (content_analytics.get("top_keywords_site") or []) if x.get("word")] - semantic_keyword_clusters = cluster_keywords_semantic(words, config or {}) - except ImportError as e: + llm_cfg = llm_cfg_for_clusters + if str(llm_cfg.get("llm_enable_keyword_clusters", "")).lower() in ("true", "1", "yes"): + words = [x["word"] for x in (content_analytics.get("top_keywords_site") or []) if x.get("word")] + semantic_keyword_clusters = cluster_keywords_llm(words, llm_cfg, db_path=db_path) + except Exception as e: ml_bundle.setdefault("ml_errors", []).append(str(e)) outbound_max = get_int(config or {}, "outbound_domain_max_rows", 200) or 200 outbound_link_domains = _build_outbound_link_domains(df, start_url or "", outbound_max) @@ -1376,7 +1380,6 @@ def _bool_col(col): "response_time_stats": response_time_stats, "depth_distribution": depth_distribution, "content_duplicates": ml_bundle.get("content_duplicates") or [], - "anomalies": ml_bundle.get("anomalies") or [], "language_summary": ml_bundle.get("language_summary") or {}, "ner_site_summary": ml_bundle.get("ner_site_summary") or {}, "semantic_keyword_clusters": semantic_keyword_clusters, diff --git a/src/website_profiling/reporting/categories.py b/src/website_profiling/reporting/categories.py index 63a5bcef..56d9880b 100644 --- a/src/website_profiling/reporting/categories.py +++ b/src/website_profiling/reporting/categories.py @@ -625,7 +625,7 @@ def category_security( def category_intelligence(ml_bundle: Optional[dict] = None) -> dict: - """Content intelligence: duplicate clusters, anomalies, language mix from optional ML enrichment.""" + """Content intelligence: duplicate clusters and language mix from local + optional AI enrichment.""" issues: list[dict] = [] deductions: list[tuple[int, bool]] = [] ml_bundle = ml_bundle or {} @@ -648,22 +648,6 @@ def category_intelligence(ml_bundle: Optional[dict] = None) -> dict: )) deductions.append((8, True)) - anomalies = ml_bundle.get("anomalies") or [] - if len(anomalies) >= 5: - issues.append(_issue( - f"Unusual pages (multivariate outlier): {len(anomalies)} URL(s) flagged.", - priority="Medium", - recommendation="Review anomalies for crawl noise, soft-404s, or template bugs.", - )) - deductions.append((min(15, 5 + len(anomalies) // 10), True)) - elif anomalies: - issues.append(_issue( - f"{len(anomalies)} URL(s) look statistically unusual vs the rest of the crawl.", - priority="Low", - recommendation="Spot-check flagged URLs in Link Explorer (ml_anomaly).", - )) - deductions.append((3, True)) - lang = ml_bundle.get("language_summary") or {} if lang.get("mixed_site") and (lang.get("detected_pages") or 0) >= 10: counts = lang.get("counts") or {} @@ -702,7 +686,7 @@ def build_categories( summary_seo should have: issues["broken"], issues["redirects"]. security_findings: optional list from security scanner (finding_type, severity, url, message, recommendation). lighthouse_summary: optional dict from lighthouse_runner (median_metrics, top_failures); when set, Core Web Vitals uses real data. - ml_bundle: optional dict from ml_enrich.run_ml_enrichment (duplicates, anomalies, language_summary, etc.) for Content intelligence category. + ml_bundle: optional dict from analysis + LLM enrichment (duplicates, language_summary, etc.) for Content intelligence category. """ issues_broken = summary_seo.get("issues", {}).get("broken", []) issues_redirects = summary_seo.get("issues", {}).get("redirects", []) diff --git a/src/website_profiling/tools/keywords.py b/src/website_profiling/tools/keywords.py index 23c3967b..f53402f6 100644 --- a/src/website_profiling/tools/keywords.py +++ b/src/website_profiling/tools/keywords.py @@ -257,16 +257,28 @@ def run_keyword_pipeline( clusters = cluster_keywords(scored) semantic_clusters: list[dict[str, Any]] = [] - from ..config import get_bool - - if get_bool(config, "enable_semantic_keywords", False): - try: - from ..ml.enrich import cluster_keywords_semantic - - top_kw = [s["keyword"] for s in scored[:200] if s.get("keyword")] - semantic_clusters = cluster_keywords_semantic(top_kw, config) - except ImportError as e: - print(f"Semantic keywords skipped: {e}", file=sys.stderr) + llm_cfg: dict[str, str] = {} + db_raw = (config.get("sqlite_db") or "").strip() + if db_raw: + db_path = db_raw if os.path.isabs(db_raw) else os.path.join(output_dir, db_raw) + env_db = (os.environ.get("REPORT_DB_PATH") or "").strip() + if env_db: + db_path = os.path.abspath(env_db) + from ..llm_config import load_llm_config_from_db, llm_is_enabled + + llm_cfg = load_llm_config_from_db(db_path) + if llm_is_enabled(llm_cfg) and str(llm_cfg.get("llm_enable_keyword_clusters", "")).lower() in ( + "true", + "1", + "yes", + ): + try: + from ..llm.enrich import cluster_keywords_llm + + top_kw = [s["keyword"] for s in scored[:200] if s.get("keyword")] + semantic_clusters = cluster_keywords_llm(top_kw, llm_cfg, db_path=db_path) + except Exception as e: + print(f"Semantic keywords skipped: {e}", file=sys.stderr) ts = datetime.now(timezone.utc).isoformat() top_path = os.path.join(output_dir, "top_keywords.csv") diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 00000000..6227ecf7 --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,44 @@ +"""Tests for local content analysis.""" +from __future__ import annotations + +import pandas as pd + +from website_profiling.analysis.local import compute_duplicate_groups, simhash_64 + + +def test_simhash_identical_text_same_hash(): + t = "hello world " * 10 + assert simhash_64(t) == simhash_64(t) + + +def test_duplicate_groups_fuzzy_merge(): + df = pd.DataFrame( + [ + { + "url": "https://example.com/a", + "status": "200", + "content_type": "text/html", + "title": "Best SEO Tools Guide", + "meta_description": "A guide to SEO tools for marketers", + "h1": "SEO Tools", + "content_excerpt": " ".join(["seo tools"] * 50), + }, + { + "url": "https://example.com/b", + "status": "200", + "content_type": "text/html", + "title": "Best SEO Tools Guide", + "meta_description": "A guide to SEO tools for marketers", + "h1": "SEO Tools", + "content_excerpt": " ".join(["seo tools"] * 50), + }, + ] + ) + cfg = { + "enable_duplicate_detection": "true", + "analysis_fuzzy_threshold": "90", + "analysis_dup_max_pages": "100", + } + groups, url_gid = compute_duplicate_groups(df, cfg) + assert len(groups) >= 1 + assert url_gid.get("https://example.com/a") == url_gid.get("https://example.com/b") diff --git a/tests/test_llm_config.py b/tests/test_llm_config.py new file mode 100644 index 00000000..cf34a8bc --- /dev/null +++ b/tests/test_llm_config.py @@ -0,0 +1,33 @@ +"""Tests for LLM config loading.""" +from __future__ import annotations + +import os +import tempfile + +from website_profiling.db import db_session, init_schema +from website_profiling.db.storage import write_llm_config +from website_profiling.llm_config import llm_is_enabled, load_llm_config_from_db + + +def test_load_llm_config_from_db(): + with tempfile.TemporaryDirectory() as tmp: + db_path = os.path.join(tmp, "report.db") + with db_session(db_path) as conn: + init_schema(conn) + write_llm_config( + conn, + { + "llm_enabled": "true", + "llm_provider": "ollama", + "llm_model": "llama3.2", + }, + secret_keys=set(), + ) + cfg = load_llm_config_from_db(db_path) + assert cfg.get("llm_provider") == "ollama" + assert llm_is_enabled(cfg) + + +def test_llm_disabled_by_default(): + assert not llm_is_enabled({}) + assert not llm_is_enabled({"llm_enabled": "false", "llm_provider": "openai"}) diff --git a/tests/test_llm_parse.py b/tests/test_llm_parse.py new file mode 100644 index 00000000..c274d89a --- /dev/null +++ b/tests/test_llm_parse.py @@ -0,0 +1,14 @@ +"""Tests for LLM JSON parsing.""" +from __future__ import annotations + +from website_profiling.llm.base import parse_json_response + + +def test_parse_json_response_plain(): + assert parse_json_response('{"pages": []}') == {"pages": []} + + +def test_parse_json_response_markdown_fence(): + text = 'Here is JSON:\n{"clusters": [{"top_keyword": "seo"}]}\n' + out = parse_json_response(text) + assert "clusters" in out diff --git a/web/app/api/llm-config/route.js b/web/app/api/llm-config/route.js new file mode 100644 index 00000000..15aaabb6 --- /dev/null +++ b/web/app/api/llm-config/route.js @@ -0,0 +1,67 @@ +import { NextResponse } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { loadLlmConfig, saveLlmConfig } from '@/server/llmConfig'; +import { ALL_LLM_SCHEMA_KEYS, getLlmFieldByKey } from '@/lib/llmConfigSchema'; + +export const runtime = 'nodejs'; + +/** + * GET /api/llm-config — LLM settings from report.db only (secrets masked). + */ +export async function GET(request) { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + try { + const result = await loadLlmConfig(); + return NextResponse.json(result); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +} + +/** + * PUT /api/llm-config — Body: { state: Record } + */ +export async function PUT(request) { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + let body; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const { state: rawState } = body; + if (!rawState || typeof rawState !== 'object') { + return NextResponse.json({ error: 'Missing state object' }, { status: 400 }); + } + + const state = {}; + for (const [key, rawValue] of Object.entries(rawState)) { + if (!ALL_LLM_SCHEMA_KEYS.has(key)) continue; + if (key.endsWith('_masked')) continue; + const field = getLlmFieldByKey(key); + if (!field) continue; + + if (field.type === 'bool') { + state[key] = rawValue === true || rawValue === 'true'; + } else { + state[key] = rawValue == null ? '' : String(rawValue); + if (rawState[`${key}_masked`] === true) { + state[`${key}_masked`] = true; + } + } + } + + try { + const dbPath = await saveLlmConfig(state); + return NextResponse.json({ ok: true, dbPath }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +} diff --git a/web/app/api/pipeline-config/route.js b/web/app/api/pipeline-config/route.js index 8df1799a..52a50755 100644 --- a/web/app/api/pipeline-config/route.js +++ b/web/app/api/pipeline-config/route.js @@ -47,9 +47,11 @@ export async function PUT(request) { return NextResponse.json({ error: 'Missing state object' }, { status: 400 }); } - // Coerce each key to its declared type; ignore keys not in schema + // Coerce each key to its declared type; ignore keys not in schema. + // LLM keys are UI-only (llm_config table) — never persist via pipeline-config. const state = {}; for (const [key, rawValue] of Object.entries(rawState)) { + if (key.startsWith('llm_')) continue; if (!ALL_SCHEMA_KEYS.has(key)) continue; const field = getFieldByKey(key); if (!field) continue; @@ -66,9 +68,16 @@ export async function PUT(request) { } } - // Validate unknownKeys shape + // Validate unknownKeys shape; drop llm_* (UI-only) and legacy ml_* keys const safeUnknownKeys = Array.isArray(unknownKeys) - ? unknownKeys.filter((u) => u && typeof u.key === 'string' && typeof u.value === 'string') + ? unknownKeys.filter( + (u) => + u && + typeof u.key === 'string' && + typeof u.value === 'string' && + !u.key.startsWith('llm_') && + !u.key.startsWith('ml_'), + ) : []; try { diff --git a/web/app/api/run/route.js b/web/app/api/run/route.js index d58dd85b..8a4d390a 100644 --- a/web/app/api/run/route.js +++ b/web/app/api/run/route.js @@ -1,6 +1,8 @@ import { NextResponse } from 'next/server'; import { startPipelineJob } from '@/server/pipelineJobs'; import { loadPipelineConfig, savePipelineConfig } from '@/server/pipelineConfig'; +import { saveLlmConfig } from '@/server/llmConfig'; +import { ALL_LLM_SCHEMA_KEYS, getLlmFieldByKey } from '@/lib/llmConfigSchema'; import { ALL_SCHEMA_KEYS, getFieldByKey, validatePipelineRun } from '@/lib/pipelineConfigSchema'; export const runtime = 'nodejs'; @@ -32,7 +34,7 @@ export async function POST(request) { body = {}; } - const { command = null, state: rawState, unknownKeys = [], python, repoRoot } = body; + const { command = null, state: rawState, unknownKeys = [], llmState: rawLlmState, python, repoRoot } = body; let resolvedState = rawState; let resolvedUnknownKeys = unknownKeys; @@ -56,6 +58,7 @@ export async function POST(request) { // Coerce state per field type const state = {}; for (const [key, rawValue] of Object.entries(resolvedState)) { + if (key.startsWith('llm_')) continue; if (!ALL_SCHEMA_KEYS.has(key)) continue; const field = getFieldByKey(key); if (!field) continue; @@ -73,7 +76,14 @@ export async function POST(request) { } const safeUnknownKeys = Array.isArray(resolvedUnknownKeys) - ? resolvedUnknownKeys.filter((u) => u && typeof u.key === 'string' && typeof u.value === 'string') + ? resolvedUnknownKeys.filter( + (u) => + u && + typeof u.key === 'string' && + typeof u.value === 'string' && + !u.key.startsWith('llm_') && + !u.key.startsWith('ml_'), + ) : []; const validationErrors = validatePipelineRun({ state, command: command || null }); @@ -88,6 +98,30 @@ export async function POST(request) { return NextResponse.json({ error: `Failed to save config: ${msg}` }, { status: 500 }); } + if (rawLlmState && typeof rawLlmState === 'object') { + const llmCoerced = {}; + for (const [key, rawValue] of Object.entries(rawLlmState)) { + if (!ALL_LLM_SCHEMA_KEYS.has(key)) continue; + if (key.endsWith('_masked')) continue; + const field = getLlmFieldByKey(key); + if (!field) continue; + if (field.type === 'bool') { + llmCoerced[key] = rawValue === true || rawValue === 'true'; + } else { + llmCoerced[key] = rawValue == null ? '' : String(rawValue); + if (rawLlmState[`${key}_masked`] === true) { + llmCoerced[`${key}_masked`] = true; + } + } + } + try { + await saveLlmConfig(llmCoerced); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: `Failed to save LLM config: ${msg}` }, { status: 500 }); + } + } + try { // No config path needed — Python reads from DB via REPORT_DB_PATH env. const id = startPipelineJob(command, null, { python, repoRoot }); diff --git a/web/src/components/PipelineRunnerFab.jsx b/web/src/components/PipelineRunnerFab.jsx index 45601309..f286d6d5 100644 --- a/web/src/components/PipelineRunnerFab.jsx +++ b/web/src/components/PipelineRunnerFab.jsx @@ -11,6 +11,10 @@ import { buildInitialPipelineConfigState, validatePipelineRun, } from '@/lib/pipelineConfigSchema'; +import { + LLM_CONFIG_SECTIONS, + buildInitialLlmConfigState, +} from '@/lib/llmConfigSchema'; const COMMANDS = [ { value: '', label: 'Full pipeline (per form config)' }, @@ -20,15 +24,16 @@ const COMMANDS = [ { value: 'lighthouse', label: 'lighthouse' }, { value: 'keywords', label: 'keywords' }, { value: 'warnings', label: 'warnings' }, - { value: 'enrich', label: 'enrich' }, + { value: 'enrich', label: 'enrich (analysis + AI)' }, { value: 'google', label: 'google (fetch GSC & GA4)' }, { value: 'keywords --enrich-google', label: 'keywords --enrich-google (Keywords Explorer)' }, ]; const TAB_RUN = 'run'; +const TAB_AI = 'ai'; const TAB_OTHER = 'other'; -// Build tab list: de-dupe section ids (safety), then append run +// Build tab list: de-dupe section ids (safety), AI tab, then Run function buildMainTabs(unknownKeys) { const seen = new Set(); const tabs = []; @@ -38,6 +43,7 @@ function buildMainTabs(unknownKeys) { tabs.push({ id: s.id, label: s.label }); } } + tabs.push({ id: TAB_AI, label: 'AI' }); tabs.push({ id: TAB_RUN, label: 'Run' }); if (unknownKeys.length > 0) { tabs.push({ id: TAB_OTHER, label: 'Other' }); @@ -55,6 +61,54 @@ function ConfigField({ field: f, value, disabled, onChange }) {

{f.help}

) : null; + if (f.type === 'select') { + const strVal = value == null ? String(f.defaultValue ?? '') : String(value); + return ( +
+ + + {helpEl} +
+ ); + } + + if (f.type === 'secret') { + const strVal = value == null ? '' : String(value); + const placeholder = strVal.startsWith('••••') ? strVal : 'Paste API key (optional if env var set)'; + return ( +
+ + onChange(e.target.value)} + className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground font-mono" + /> + {helpEl} +
+ ); + } + if (f.type === 'bool') { const checked = value === true; return ( @@ -158,6 +212,8 @@ export default function PipelineRunnerFab() { const [minimized, setMinimized] = useState(false); const [activeTab, setActiveTab] = useState(PIPELINE_CONFIG_SECTIONS[0].id); const [configState, setConfigState] = useState(buildInitialPipelineConfigState); + const [llmConfigState, setLlmConfigState] = useState(buildInitialLlmConfigState); + const [llmConfigMasked, setLlmConfigMasked] = useState({}); const [unknownKeys, setUnknownKeys] = useState([]); const [configPath, setConfigPath] = useState(''); const [configSource, setConfigSource] = useState(null); // 'store'|'legacy'|'defaults' @@ -241,17 +297,33 @@ export default function PipelineRunnerFab() { setLoading(true); setLoadError(''); try { - const res = await fetch(apiUrl('/pipeline-config')); - const data = await res.json().catch(() => ({})); - if (!res.ok) throw new Error(data.error || res.statusText); + const [pipeRes, llmRes] = await Promise.all([ + fetch(apiUrl('/pipeline-config')), + fetch(apiUrl('/llm-config')), + ]); + const data = await pipeRes.json().catch(() => ({})); + const llmData = await llmRes.json().catch(() => ({})); + if (!pipeRes.ok) throw new Error(data.error || pipeRes.statusText); setConfigState(data.state || buildInitialPipelineConfigState()); setUnknownKeys(Array.isArray(data.unknownKeys) ? data.unknownKeys : []); setConfigPath(data.dbPath || data.configPath || ''); setConfigSource(data.source || 'defaults'); + if (llmRes.ok && llmData.state) { + setLlmConfigState(llmData.state); + const masked = {}; + for (const [k, v] of Object.entries(llmData.state)) { + if (k.endsWith('_masked')) masked[k] = v; + } + setLlmConfigMasked(masked); + } else { + setLlmConfigState(buildInitialLlmConfigState()); + setLlmConfigMasked({}); + } setLegacyBannerDismissed(false); } catch (e) { setLoadError(e instanceof Error ? e.message : String(e)); setConfigState(buildInitialPipelineConfigState()); + setLlmConfigState(buildInitialLlmConfigState()); } finally { setLoading(false); } @@ -269,6 +341,44 @@ export default function PipelineRunnerFab() { } }, [showModal, minimized]); + const setLlmField = useCallback((key, v) => { + setLlmConfigState((prev) => ({ ...prev, [key]: v })); + if (key === 'llm_api_key') { + setLlmConfigMasked((prev) => { + const next = { ...prev }; + delete next.llm_api_key_masked; + return next; + }); + } + }, []); + + const buildLlmPayload = useCallback(() => ({ ...llmConfigState, ...llmConfigMasked }), [llmConfigState, llmConfigMasked]); + + const saveLlmSettings = useCallback(async () => { + setSaving(true); + setSaveMsg(''); + try { + const res = await fetch(apiUrl('/llm-config'), { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ state: buildLlmPayload() }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || res.statusText); + setSaveMsg('AI settings saved.'); + setTimeout(() => setSaveMsg(''), 3000); + const reload = await fetch(apiUrl('/llm-config')); + const reloaded = await reload.json().catch(() => ({})); + if (reload.ok && reloaded.state) { + setLlmConfigState(reloaded.state); + } + } catch (e) { + setSaveMsg(`Save failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setSaving(false); + } + }, [buildLlmPayload]); + const setField = useCallback((key, v) => { setConfigState((prev) => ({ ...prev, [key]: v })); }, []); @@ -278,7 +388,7 @@ export default function PipelineRunnerFab() { setSaveMsg(''); }, []); - // Save settings to report.db pipeline_config table + shadow file (PUT) + // Save pipeline + AI settings to report.db (PUT) const saveSettings = useCallback(async () => { setSaving(true); setSaveMsg(''); @@ -290,7 +400,14 @@ export default function PipelineRunnerFab() { }); const data = await res.json().catch(() => ({})); if (!res.ok) throw new Error(data.error || res.statusText); - setConfigPath(data.configPath || configPath); + const llmRes = await fetch(apiUrl('/llm-config'), { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ state: buildLlmPayload() }), + }); + const llmData = await llmRes.json().catch(() => ({})); + if (!llmRes.ok) throw new Error(llmData.error || llmRes.statusText); + setConfigPath(data.configPath || data.dbPath || configPath); setConfigSource('store'); setSaveMsg('Saved.'); setTimeout(() => setSaveMsg(''), 3000); @@ -299,7 +416,7 @@ export default function PipelineRunnerFab() { } finally { setSaving(false); } - }, [configState, unknownKeys, configPath]); + }, [configState, unknownKeys, configPath, buildLlmPayload]); // Run pipeline (save first, then spawn) const run = useCallback(async () => { @@ -321,6 +438,7 @@ export default function PipelineRunnerFab() { command: command || null, state: configState, unknownKeys, + llmState: buildLlmPayload(), python: pythonExe.trim() || undefined, repoRoot: repoRoot.trim() || undefined, }), @@ -338,7 +456,7 @@ export default function PipelineRunnerFab() { setLog(e instanceof Error ? e.message : String(e)); setBusy(false); } - }, [command, configState, unknownKeys, pythonExe, repoRoot, stopPoll, watchJob]); + }, [command, configState, unknownKeys, buildLlmPayload, pythonExe, repoRoot, stopPoll, watchJob]); const openFab = () => { if (minimized && !showModal) { @@ -618,6 +736,46 @@ export default function PipelineRunnerFab() { + ) : activeTab === TAB_AI ? ( +
+

+ AI enrichment settings are stored only in{' '} + report.db (llm_config) — never in + pipeline-config.txt. Configure provider and tasks here before running report. +

+ {LLM_CONFIG_SECTIONS.map((section) => ( +
+

+ {section.label} +

+
+ {section.fields.map((f) => ( + setLlmField(f.key, v)} + /> + ))} +
+
+ ))} + +
) : activeTab === TAB_OTHER ? (
0 || - link.ml_anomaly || link.detected_language || link.keyphrases?.phrases?.length > 0 || pa?.signals?.language || @@ -356,14 +355,6 @@ export default function PageAnalysisTab({ link }) {
)} - {link.ml_anomaly && ( -
-
{p.anomalyIsolation}
-
- {p.anomalyScorePrefix} {link.ml_anomaly.anomaly_score} — {(link.ml_anomaly.reasons || []).join(', ')} -
-
- )} {similarRows.length > 0 && (
diff --git a/web/src/lib/llmConfigSchema.js b/web/src/lib/llmConfigSchema.js new file mode 100644 index 00000000..6e61f7dd --- /dev/null +++ b/web/src/lib/llmConfigSchema.js @@ -0,0 +1,117 @@ +/** + * LLM / AI enrichment settings — stored only in report.db (llm_config table). + * Not part of pipeline-config.txt or CLI --config files. + */ + +export const LLM_CONFIG_SECTIONS = [ + { + id: 'llm_provider', + label: 'Provider', + fields: [ + { + key: 'llm_enabled', + label: 'Enable AI enrichment', + type: 'bool', + defaultValue: false, + help: 'Uses the provider below during report generation. Configure API keys here or via environment variables.', + }, + { + key: 'llm_provider', + label: 'Provider', + type: 'select', + defaultValue: 'none', + options: [ + { value: 'none', label: 'None' }, + { value: 'openai', label: 'OpenAI' }, + { value: 'gemini', label: 'Google Gemini' }, + { value: 'anthropic', label: 'Anthropic Claude' }, + { value: 'ollama', label: 'Ollama (local)' }, + ], + }, + { + key: 'llm_model', + label: 'Model', + type: 'text', + defaultValue: '', + placeholder: 'e.g. gpt-4o-mini, gemini-2.0-flash, claude-3-5-haiku-latest, llama3.2', + help: 'Leave blank to use provider default.', + }, + { + key: 'llm_base_url', + label: 'Base URL (Ollama / custom)', + type: 'text', + defaultValue: 'http://127.0.0.1:11434', + help: 'Ollama API root. Ignored for cloud providers unless using an OpenAI-compatible proxy.', + }, + { + key: 'llm_api_key', + label: 'API key', + type: 'secret', + defaultValue: '', + help: 'Optional if set via OPENAI_API_KEY, GEMINI_API_KEY, or ANTHROPIC_API_KEY. Never written to pipeline-config.txt.', + }, + ], + }, + { + id: 'llm_tasks', + label: 'Tasks', + fields: [ + { key: 'llm_enable_ner', label: 'Named entities (NER)', type: 'bool', defaultValue: true }, + { key: 'llm_enable_keyphrases', label: 'Keyphrases', type: 'bool', defaultValue: true }, + { key: 'llm_enable_similar_internal', label: 'Similar internal pages', type: 'bool', defaultValue: true }, + { key: 'llm_enable_keyword_clusters', label: 'Semantic keyword clusters', type: 'bool', defaultValue: true }, + ], + }, + { + id: 'llm_limits', + label: 'Limits', + fields: [ + { key: 'llm_max_pages', label: 'Max pages (LLM tasks)', type: 'number', defaultValue: '60' }, + { key: 'llm_batch_size', label: 'Pages per API batch', type: 'number', defaultValue: '5' }, + { key: 'llm_timeout_s', label: 'Request timeout (s)', type: 'number', defaultValue: '120' }, + { key: 'llm_similar_top_k', label: 'Similar pages top K', type: 'number', defaultValue: '5' }, + ], + }, +]; + +export const ALL_LLM_SCHEMA_KEYS = new Set( + LLM_CONFIG_SECTIONS.flatMap((s) => s.fields.map((f) => f.key)), +); + +const SECRET_KEYS = new Set(['llm_api_key']); + +export function isLlmSecretKey(key) { + return SECRET_KEYS.has(key); +} + +export function getLlmFieldByKey(key) { + for (const section of LLM_CONFIG_SECTIONS) { + const f = section.fields.find((x) => x.key === key); + if (f) return f; + } + return null; +} + +export function buildInitialLlmConfigState() { + const out = {}; + for (const section of LLM_CONFIG_SECTIONS) { + for (const f of section.fields) { + if (f.type === 'bool') { + out[f.key] = f.defaultValue; + } else { + out[f.key] = String(f.defaultValue ?? ''); + } + } + } + return out; +} + +/** Mask stored API key for GET responses. */ +export function maskLlmSecretForClient(key, value) { + if (!isLlmSecretKey(key) || !value || String(value).trim() === '') { + return ''; + } + const s = String(value); + if (s.length <= 4) return '••••'; + return `••••${s.slice(-4)}`; +} diff --git a/web/src/lib/pipelineConfigSchema.js b/web/src/lib/pipelineConfigSchema.js index cf523308..6c204bbd 100644 --- a/web/src/lib/pipelineConfigSchema.js +++ b/web/src/lib/pipelineConfigSchema.js @@ -122,30 +122,14 @@ export const PIPELINE_CONFIG_SECTIONS = [ ], }, { - id: 'ml', - label: 'ML', + id: 'analysis', + label: 'Content analysis', fields: [ - { key: 'enable_duplicate_detection', label: 'Duplicate detection', type: 'bool', defaultValue: true }, - { key: 'enable_anomaly_urls', label: 'Anomaly URLs', type: 'bool', defaultValue: true }, + { key: 'enable_duplicate_detection', label: 'Duplicate detection (SimHash/fuzzy)', type: 'bool', defaultValue: true }, { key: 'enable_language_detection', label: 'Language detection', type: 'bool', defaultValue: true }, - { key: 'enable_ner_spacy', label: 'NER (spaCy)', type: 'bool', defaultValue: true }, - { key: 'enable_semantic_similar_internal', label: 'Semantic similar internal', type: 'bool', defaultValue: true }, - { key: 'enable_semantic_keywords', label: 'Semantic keywords', type: 'bool', defaultValue: true }, - { key: 'ml_sentence_model', label: 'Sentence model', type: 'text', defaultValue: 'all-MiniLM-L6-v2' }, - { key: 'ml_max_pages_st', label: 'Max pages (sentence-transformers)', type: 'number', defaultValue: '400' }, - { key: 'ml_similar_top_k', label: 'Similar top K', type: 'number', defaultValue: '5' }, - { key: 'ml_fuzzy_threshold', label: 'Fuzzy threshold', type: 'number', defaultValue: '92' }, - { key: 'ml_simhash_hamming', label: 'Simhash Hamming', type: 'number', defaultValue: '0' }, - { key: 'ml_dup_max_pages', label: 'Dup max pages', type: 'number', defaultValue: '2000' }, - { key: 'ml_ner_max_pages', label: 'NER max pages', type: 'number', defaultValue: '80' }, - { key: 'ml_semantic_keyword_max', label: 'Semantic keyword max', type: 'number', defaultValue: '200' }, - { key: 'ml_keyword_cluster_sim', label: 'Keyword cluster similarity', type: 'number', defaultValue: '75' }, - { key: 'enable_embedding_duplicate_refine', label: 'Embedding duplicate refine', type: 'bool', defaultValue: true }, - { key: 'ml_dup_embed_min_pct', label: 'Dup embed min %', type: 'number', defaultValue: '88' }, - { key: 'enable_keybert', label: 'KeyBERT', type: 'bool', defaultValue: true }, - { key: 'ml_keybert_max_pages', label: 'KeyBERT max pages', type: 'number', defaultValue: '60' }, - { key: 'ml_keybert_top_n', label: 'KeyBERT top N', type: 'number', defaultValue: '8' }, - { key: 'ml_verbose', label: 'ML verbose progress', type: 'bool', defaultValue: true }, + { key: 'analysis_fuzzy_threshold', label: 'Fuzzy threshold', type: 'number', defaultValue: '92' }, + { key: 'analysis_simhash_hamming', label: 'Simhash Hamming', type: 'number', defaultValue: '0' }, + { key: 'analysis_dup_max_pages', label: 'Dup max pages', type: 'number', defaultValue: '2000' }, ], }, { diff --git a/web/src/server/llmConfig.js b/web/src/server/llmConfig.js new file mode 100644 index 00000000..6a7cd42d --- /dev/null +++ b/web/src/server/llmConfig.js @@ -0,0 +1,181 @@ +/** + * LLM config stored only in report.db (llm_config table). No shadow file. + */ +import initSqlJs from 'sql.js'; +import fs from 'fs'; +import path from 'path'; +import { + LLM_CONFIG_SECTIONS, + ALL_LLM_SCHEMA_KEYS, + getLlmFieldByKey, + buildInitialLlmConfigState, + maskLlmSecretForClient, + isLlmSecretKey, +} from '@/lib/llmConfigSchema'; +import { getReportDbPath } from '@/server/pipelineConfig'; + +const LLM_CONFIG_DDL = ` + CREATE TABLE IF NOT EXISTS llm_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + is_secret INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL + ); +`; + +const MASK_SENTINEL = '__MASKED__'; + +async function openDb() { + const SQL = await initSqlJs(); + const dbPath = getReportDbPath(); + let buf; + if (fs.existsSync(dbPath)) { + buf = fs.readFileSync(dbPath); + } + const db = buf ? new SQL.Database(new Uint8Array(buf)) : new SQL.Database(); + db.run(LLM_CONFIG_DDL); + return db; +} + +function persistDb(db) { + const dbPath = getReportDbPath(); + const dir = path.dirname(dbPath); + fs.mkdirSync(dir, { recursive: true }); + const buf = db.export(); + const tmp = dbPath + '.tmp'; + fs.writeFileSync(tmp, Buffer.from(buf)); + fs.renameSync(tmp, dbPath); +} + +function readLlmConfigFromDb(db) { + const known = {}; + try { + const res = db.exec('SELECT key, value, is_secret FROM llm_config ORDER BY key'); + if (!res.length || !res[0].values) return known; + for (const row of res[0].values) { + known[String(row[0])] = String(row[1]); + } + } catch { + /* empty */ + } + return known; +} + +function applyLlmDefaults(parsedMap) { + const state = buildInitialLlmConfigState(); + for (const [key, rawValue] of Object.entries(parsedMap)) { + if (!ALL_LLM_SCHEMA_KEYS.has(key)) continue; + const field = getLlmFieldByKey(key); + if (!field) continue; + if (field.type === 'bool') { + state[key] = ['true', '1', 'yes'].includes(String(rawValue).toLowerCase()); + } else { + state[key] = String(rawValue ?? ''); + } + } + return state; +} + +/** Client-safe state (secrets masked). */ +export function maskLlmStateForClient(state) { + const out = { ...state }; + for (const key of Object.keys(out)) { + if (isLlmSecretKey(key)) { + out[key] = maskLlmSecretForClient(key, out[key]); + if (out[key]) out[`${key}_masked`] = true; + } + } + return out; +} + +/** + * @returns {Promise<{ state: Record, dbPath: string, source: 'store'|'defaults' }>} + */ +export async function loadLlmConfig() { + const dbPath = getReportDbPath(); + let db; + try { + db = await openDb(); + const known = readLlmConfigFromDb(db); + if (Object.keys(known).length > 0) { + const state = applyLlmDefaults(known); + return { state: maskLlmStateForClient(state), dbPath, source: 'store' }; + } + return { state: maskLlmStateForClient(buildInitialLlmConfigState()), dbPath, source: 'defaults' }; + } finally { + try { + db?.close(); + } catch { + /* ignore */ + } + } +} + +/** + * @param {Record} state + * @param {{ preserveSecrets?: boolean }} [options] + */ +export async function saveLlmConfig(state, { preserveSecrets = true } = {}) { + const dbPath = getReportDbPath(); + let db; + try { + db = await openDb(); + const existing = preserveSecrets ? readLlmConfigFromDb(db) : {}; + + const entries = {}; + const secretKeys = new Set(); + for (const section of LLM_CONFIG_SECTIONS) { + for (const f of section.fields) { + const v = state[f.key]; + if (v === undefined) continue; + if (f.type === 'bool') { + entries[f.key] = v === true ? 'true' : 'false'; + } else if (isLlmSecretKey(f.key)) { + const raw = v == null ? '' : String(v).trim(); + const isMasked = + raw === '' || + raw === MASK_SENTINEL || + raw.startsWith('••••') || + state[`${f.key}_masked`] === true; + if (isMasked && existing[f.key]) { + entries[f.key] = existing[f.key]; + } else if (raw && !raw.startsWith('••••')) { + entries[f.key] = raw; + } else { + entries[f.key] = ''; + } + if (entries[f.key]) secretKeys.add(f.key); + } else { + entries[f.key] = v == null ? '' : String(v); + } + } + } + + const now = new Date().toISOString().slice(0, 19).replace('T', ' '); + db.run('BEGIN'); + try { + db.run('DELETE FROM llm_config'); + const insertStmt = db.prepare( + 'INSERT INTO llm_config (key, value, is_secret, updated_at) VALUES (?, ?, ?, ?)' + ); + for (const [k, v] of Object.entries(entries)) { + insertStmt.run([k, v, secretKeys.has(k) ? 1 : 0, now]); + } + insertStmt.free(); + db.run('COMMIT'); + } catch (e) { + db.run('ROLLBACK'); + throw e; + } + persistDb(db); + } finally { + try { + db?.close(); + } catch { + /* ignore */ + } + } + return dbPath; +} + +export { MASK_SENTINEL }; diff --git a/web/src/server/pipelineConfig.js b/web/src/server/pipelineConfig.js index 3541532d..15c71032 100644 --- a/web/src/server/pipelineConfig.js +++ b/web/src/server/pipelineConfig.js @@ -214,6 +214,14 @@ function writeShadowFile(state, unknownKeys) { // ─── DB read helpers ────────────────────────────────────────────────────────── +function isLegacyOrLlmKey(key) { + return key.startsWith('llm_') || key.startsWith('ml_'); +} + +function filterUnknownKeys(list) { + return (list || []).filter((u) => u && !isLegacyOrLlmKey(u.key)); +} + /** * Read all rows from the pipeline_config table. * @param {import('sql.js').Database} db @@ -264,7 +272,7 @@ export async function loadPipelineConfig() { if (Object.keys(known).length > 0 || unknown.length > 0) { const { state, unknownKeys: schemaUnknown } = applySchemaDefaults(known); // Merge DB unknown rows with any schema-unknown keys from known map - const allUnknown = [...unknown, ...schemaUnknown]; + const allUnknown = filterUnknownKeys([...unknown, ...schemaUnknown]); return { state, unknownKeys: allUnknown, source: 'store', dbPath }; } @@ -276,7 +284,7 @@ export async function loadPipelineConfig() { const parsed = parseInputTxt(raw); if (Object.keys(parsed).length > 0) { const { state, unknownKeys } = applySchemaDefaults(parsed); - return { state, unknownKeys, source: 'legacy', dbPath }; + return { state, unknownKeys: filterUnknownKeys(unknownKeys), source: 'legacy', dbPath }; } } catch { // fall through diff --git a/web/src/strings.json b/web/src/strings.json index 736e2ff0..1d39df1f 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -1046,7 +1046,7 @@ } }, "overview": { - "mlErrors": "ML enrichment reported {count} error{plural}", + "mlErrors": "AI enrichment reported {count} error{plural}", "dashboard": "Dashboard", "subtitleSiteHealth": "Site health summary for", "crawlDone": "Crawl completed.", @@ -1435,12 +1435,13 @@ "pageResourcesCaption": "Page resources (same as summary counts)", "lhCategoryCaption": "Lighthouse category scores (0–100)", "scoreAxis": "Score", - "intelligenceMl": "Intelligence (Python ML)", + "intelligenceMl": "AI enrichment", + "keyphrasesKeybert": "Keyphrases", "duplicateCluster": "Duplicate cluster", "detectedLanguage": "Detected language", "anomalyIsolation": "Anomaly (IsolationForest)", "anomalyScorePrefix": "score", - "similarInternalCaption": "Precomputed similar internal URLs (cosine vs site pages)", + "similarInternalCaption": "Similar internal URLs (AI-ranked)", "socialPreviews": "Social previews", "facebookOg": "Facebook / Open Graph", "twitterX": "Twitter / X", diff --git a/web/src/views/Overview.jsx b/web/src/views/Overview.jsx index de594d92..b48dc2c3 100644 --- a/web/src/views/Overview.jsx +++ b/web/src/views/Overview.jsx @@ -173,7 +173,6 @@ export default function Overview({ searchQuery = '' }) { const { data, reportDiff, startUrlByRunId } = useReport(); const searchParams = useSearchParams(); const [mlErrOpen, setMlErrOpen] = useState(false); - const [anomOpen, setAnomOpen] = useState(false); const vo = strings.views.overview; const sj = strings.common; const q = (searchQuery || '').toLowerCase().trim(); @@ -745,7 +744,6 @@ export default function Overview({ searchQuery = '' }) { )} {(data.content_duplicates?.length > 0 || - data.anomalies?.length > 0 || (data.language_summary?.counts && Object.keys(data.language_summary.counts).length > 0) || (data.semantic_keyword_clusters?.length > 0) || (data.ner_site_summary?.label_counts && Object.keys(data.ner_site_summary.label_counts).length > 0)) && ( @@ -754,17 +752,12 @@ export default function Overview({ searchQuery = '' }) { {vo.contentIntelligence} -
+
{vo.duplicateGroups}
{data.content_duplicates?.length ?? 0}
{vo.nearDuplicateGroups}
 - -
{vo.anomalies}
-
{data.anomalies?.length ?? 0}
-
{vo.outliers}
-
{vo.parentTopics}
{data.semantic_keyword_clusters?.length ?? 0}
@@ -783,7 +776,7 @@ export default function Overview({ searchQuery = '' }) { : vo.entitiesSitewide}
- +
{vo.languagesSampled}
{Object.entries(data.language_summary?.counts || {}) @@ -805,47 +798,6 @@ export default function Overview({ searchQuery = '' }) { )}
- - {data.anomalies?.length > 0 && ( - - - {anomOpen && ( -
- - - - {vo.thUrl} - {vo.thScore} - {vo.thReasons} - - - - {data.anomalies.map((a, idx) => ( - - - - {a.url} - - - {a.anomaly_score ?? sj.emDash} - - {(a.reasons || []).join(', ') || sj.emDash} - - - ))} - -
-
- )} - - )}
)} From 2fd451e687943436ae7a702f756a1467c8e50421 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Tue, 2 Jun 2026 10:45:26 +0530 Subject: [PATCH 2/3] After weeks of changes --- .gitignore | 7 +- AGENT.md | 26 +- Dockerfile | 23 +- Docs.md | 2 +- README.md | 74 +- alembic.ini | 42 + alembic/env.py | 50 + alembic/script.py.mako | 26 + alembic/versions/001_initial_schema.py | 173 +++ .../versions/002_perf_columns_and_indexes.py | 66 + data/pipeline-config.txt | 78 ++ docker-compose.yml | 23 +- docker-entrypoint.sh | 5 + input.txt.example | 15 +- pipeline-config.txt | 21 +- requirements.txt | 5 + src/website_profiling/analysis/page.py | 2 +- src/website_profiling/cli.py | 254 ++-- src/website_profiling/config.py | 21 +- src/website_profiling/crawl/crawler.py | 120 +- src/website_profiling/db/__init__.py | 5 +- src/website_profiling/db/storage.py | 1192 ++++++++--------- .../integrations/google/auth.py | 10 +- .../integrations/google/fetch.py | 6 +- .../integrations/google/keyword_enrich.py | 13 +- .../integrations/google/keyword_store.py | 92 +- .../integrations/google/store.py | 46 +- .../integrations/google/suggest.py | 77 +- src/website_profiling/lighthouse/runner.py | 140 +- src/website_profiling/llm/enrich.py | 137 +- src/website_profiling/llm_config.py | 11 +- src/website_profiling/reporting/builder.py | 181 +-- src/website_profiling/reporting/categories.py | 2 +- src/website_profiling/tools/keywords.py | 146 +- src/website_profiling/tools/plot.py | 72 +- src/website_profiling/tools/warnings.py | 95 +- tests/test_llm_config.py | 44 +- tests/test_storage_bulk.py | 53 + tests/test_suggest_cache_concurrent.py | 55 + web/app/(reports)/[slug]/page.tsx | 18 + web/app/(reports)/layout.tsx | 6 + web/app/(reports)/not-found.tsx | 28 + web/app/[slug]/page.jsx | 8 - .../google/auth/{route.js => route.ts} | 32 +- .../api/integrations/google/callback/route.js | 92 -- .../api/integrations/google/callback/route.ts | 144 ++ .../google/credentials/{route.js => route.ts} | 16 +- .../credentials/upload/{route.js => route.ts} | 31 +- .../google/disconnect/{route.js => route.ts} | 10 +- .../google/keywords/by-page/route.js | 71 - .../google/keywords/by-page/route.ts | 85 ++ .../keywords/expand/{route.js => route.ts} | 46 +- .../google/keywords/history/batch/route.js | 105 -- .../google/keywords/history/batch/route.ts | 88 ++ .../google/keywords/history/route.js | 66 - .../google/keywords/history/route.ts | 48 + .../integrations/google/page-data/route.js | 59 - .../integrations/google/page-data/route.ts | 63 + .../google/properties/{route.js => route.ts} | 40 +- .../api/integrations/google/status/route.js | 41 - .../api/integrations/google/status/route.ts | 31 + .../google/test/{route.js => route.ts} | 29 +- web/app/api/jobs/[id]/{route.js => route.ts} | 12 +- web/app/api/llm-config/{route.js => route.ts} | 25 +- .../pipeline-config/{route.js => route.ts} | 52 +- web/app/api/report-db/route.js | 29 - web/app/api/report/crawl-payload/route.ts | 21 + .../api/report/meta/{route.js => route.ts} | 7 +- .../api/report/payload/{route.js => route.ts} | 9 +- web/app/api/report/portfolio/route.js | 44 - web/app/api/report/portfolio/route.ts | 61 + web/app/api/run/{route.js => route.ts} | 46 +- ...ent-providers.jsx => client-providers.tsx} | 12 +- web/app/{layout.jsx => layout.tsx} | 3 +- web/app/page.jsx | 7 - web/app/page.tsx | 23 + web/app/pipeline/page.tsx | 7 + web/global.d.ts | 21 + web/jsconfig.json | 8 - web/next.config.mjs | 2 - web/package-lock.json | 359 +++-- web/package.json | 15 +- web/src/ReportShell.jsx | 464 ------- web/src/ReportShell.tsx | 263 ++++ web/src/components/AppShell.tsx | 280 ++++ web/src/components/{Badge.jsx => Badge.tsx} | 15 +- web/src/components/{Button.jsx => Button.tsx} | 14 +- web/src/components/{Card.jsx => Card.tsx} | 15 +- .../components/GoogleIntegrationsPanel.tsx | 744 ++++++++++ web/src/components/IntegrationsModal.jsx | 731 ---------- web/src/components/IntegrationsModal.tsx | 43 + web/src/components/PageHeader.jsx | 11 - web/src/components/PageHeader.tsx | 19 + .../{PageLayout.jsx => PageLayout.tsx} | 12 +- web/src/components/PipelineRunnerFab.jsx | 871 ------------ web/src/components/ReportCompareControls.tsx | 108 ++ web/src/components/ReportSelector.jsx | 87 -- web/src/components/ReportSelector.tsx | 33 + ...llSkeleton.jsx => ReportShellSkeleton.tsx} | 13 +- .../components/{Skeleton.jsx => Skeleton.tsx} | 2 +- web/src/components/{Table.jsx => Table.tsx} | 35 +- .../{ThemeToggle.jsx => ThemeToggle.tsx} | 5 +- web/src/components/compare/CompareCharts.tsx | 419 ++++++ .../components/compare/CompareDeltaBadge.tsx | 40 + .../components/compare/CompareTabPanels.tsx | 523 ++++++++ ...oogleChartCard.jsx => GoogleChartCard.tsx} | 10 +- ...ableToolbar.jsx => GoogleTableToolbar.tsx} | 16 +- ...iesChart.jsx => GoogleTimeSeriesChart.tsx} | 14 +- ...{GoogleViewTabs.jsx => GoogleViewTabs.tsx} | 18 +- ...edTable.jsx => SortablePaginatedTable.tsx} | 25 +- .../{SummaryCard.jsx => SummaryCard.tsx} | 14 +- ...geDoughnut.jsx => UrlCoverageDoughnut.tsx} | 13 +- ...GapListsPanel.jsx => UrlGapListsPanel.tsx} | 102 +- .../google/{tableUtils.js => tableUtils.ts} | 36 +- web/src/components/index.js | 16 - web/src/components/index.ts | 16 + .../{KeywordCharts.jsx => KeywordCharts.tsx} | 23 +- .../{KeywordPanels.jsx => KeywordPanels.tsx} | 53 +- ...bleColumns.jsx => KeywordTableColumns.tsx} | 119 +- ...wordTableUtils.js => keywordTableUtils.ts} | 47 +- ...iagnosticGroup.jsx => DiagnosticGroup.tsx} | 26 +- ...{DiagnosticItem.jsx => DiagnosticItem.tsx} | 11 +- ...itExpandable.jsx => LhAuditExpandable.tsx} | 9 +- ...{LhDetailsTable.jsx => LhDetailsTable.tsx} | 166 ++- ...{MultiPageTable.jsx => MultiPageTable.tsx} | 45 +- .../{QuickWinCard.jsx => QuickWinCard.tsx} | 16 +- .../{ScoreRing.jsx => ScoreRing.tsx} | 7 +- .../{ThresholdBar.jsx => ThresholdBar.tsx} | 9 +- .../lighthouse/{index.js => index.ts} | 0 .../links/{CharBar.jsx => CharBar.tsx} | 8 +- .../links/{CopyBtn.jsx => CopyBtn.tsx} | 9 +- .../{HeadingPills.jsx => HeadingPills.tsx} | 13 +- .../links/{InlineRing.jsx => InlineRing.tsx} | 7 +- .../{InspectorTabs.jsx => InspectorTabs.tsx} | 19 +- .../links/{MiniBar.jsx => MiniBar.tsx} | 9 +- .../links/{OGPreview.jsx => OGPreview.tsx} | 11 +- .../links/{RowTooltip.jsx => RowTooltip.tsx} | 13 +- .../{SecHeaderRow.jsx => SecHeaderRow.tsx} | 8 +- .../links/{SortTh.jsx => SortTh.tsx} | 11 +- .../components/links/{index.js => index.ts} | 0 .../tabs/{ContentTab.jsx => ContentTab.tsx} | 72 +- .../tabs/{IssuesTab.jsx => IssuesTab.tsx} | 34 +- .../tabs/{OverviewTab.jsx => OverviewTab.tsx} | 9 +- ...ageAnalysisTab.jsx => PageAnalysisTab.tsx} | 94 +- .../{SeoSocialTab.jsx => SeoSocialTab.tsx} | 13 +- .../{TechnicalTab.jsx => TechnicalTab.tsx} | 26 +- web/src/components/pipeline/ConfigField.tsx | 386 ++++++ .../components/pipeline/PipelineLogViewer.tsx | 430 ++++++ .../components/pipeline/PipelineRunPanel.tsx | 374 ++++++ .../components/pipeline/PipelineRunnerFab.tsx | 88 ++ .../pipeline/PipelineSettingsPanel.tsx | 361 +++++ .../pipeline/PipelineSettingsSectionTabs.tsx | 43 + web/src/components/pipeline/PipelineShell.tsx | 215 +++ .../pipeline/PipelineWizardProgress.tsx | 82 ++ .../components/pipeline/pipelinePresets.ts | 60 + .../pipeline/pipelineSettingsGroups.ts | 51 + web/src/components/pipeline/pipelineUi.tsx | 132 ++ .../{GscCharts.jsx => GscCharts.tsx} | 73 +- .../{gscTableUtils.js => gscTableUtils.ts} | 15 +- .../{PathTreeTable.jsx => PathTreeTable.tsx} | 50 +- .../traffic/{Ga4Charts.jsx => Ga4Charts.tsx} | 78 +- .../{ga4TableUtils.js => ga4TableUtils.ts} | 14 +- web/src/context/PipelineContext.tsx | 473 +++++++ web/src/context/ReportContext.jsx | 225 ---- web/src/context/ReportContext.tsx | 379 ++++++ .../{ThemeProvider.jsx => ThemeProvider.tsx} | 18 +- web/src/context/reportContext.js | 3 - web/src/context/reportContextTypes.ts | 27 + web/src/context/themeContext.js | 3 - web/src/context/themeContext.ts | 11 + .../context/{themeUtils.js => themeUtils.ts} | 14 +- web/src/context/useReport.js | 8 - web/src/context/useReport.ts | 13 + web/src/context/{useTheme.js => useTheme.ts} | 5 +- web/src/lib/appNav.ts | 92 ++ ...uditSqlExamples.js => auditSqlExamples.ts} | 24 +- web/src/lib/{badges.js => badges.ts} | 4 +- web/src/lib/compareChartData.ts | 100 ++ web/src/lib/{domainSlug.js => domainSlug.ts} | 68 +- web/src/lib/formatPipelineLog.ts | 231 ++++ web/src/lib/{gscMetrics.js => gscMetrics.ts} | 6 +- web/src/lib/homePortfolio.js | 92 -- web/src/lib/homePortfolio.ts | 158 +++ ...{llmConfigSchema.js => llmConfigSchema.ts} | 20 +- web/src/lib/loadReportDb.js | 194 --- web/src/lib/loadReportDb.ts | 289 ++++ ...onfigSchema.js => pipelineConfigSchema.ts} | 292 ++-- web/src/lib/pipelineDebug.ts | 16 + web/src/lib/pipelineJobEvents.js | 83 -- web/src/lib/pipelineJobEvents.ts | 117 ++ web/src/lib/pipelineReturn.ts | 47 + web/src/lib/{publicBase.js => publicBase.ts} | 19 +- web/src/lib/reportCompare.ts | 291 ++++ web/src/lib/reportCompareExtras.ts | 544 ++++++++ web/src/lib/{reportDiff.js => reportDiff.ts} | 22 +- web/src/lib/{reportNav.js => reportNav.ts} | 9 +- web/src/lib/reportTimestamps.ts | 16 + ...eStructureTree.js => siteStructureTree.ts} | 162 +-- web/src/lib/{strings.js => strings.ts} | 9 +- web/src/{patchConsole.js => patchConsole.ts} | 13 +- web/src/routes.js | 40 - web/src/routes.ts | 52 + web/src/server/db.ts | 40 + .../{googleSecrets.js => googleSecrets.ts} | 44 +- web/src/server/llmConfig.js | 181 --- web/src/server/llmConfig.ts | 130 ++ web/src/server/{localOnly.js => localOnly.ts} | 12 +- web/src/server/oauthReturn.ts | 56 + web/src/server/pipelineConfig.js | 361 ----- web/src/server/pipelineConfig.ts | 252 ++++ .../{pipelineJobs.js => pipelineJobs.ts} | 108 +- web/src/server/pipelineSpawnEnv.js | 16 - web/src/server/pipelineSpawnEnv.ts | 18 + web/src/server/reportDb.ts | 28 + web/src/server/reportSqlite.js | 48 - web/src/server/resolvePython.ts | 87 ++ web/src/strings.json | 287 +++- web/src/types/api.ts | 186 +++ web/src/types/chart.ts | 25 + web/src/types/components.ts | 180 +++ web/src/types/index.ts | 146 ++ web/src/types/report.ts | 766 +++++++++++ web/src/types/strings.ts | 11 + ...{chartJsDefaults.js => chartJsDefaults.ts} | 32 +- web/src/utils/chartOptions.ts | 21 + .../{chartPalette.js => chartPalette.ts} | 33 +- web/src/utils/errorMessage.ts | 6 + ...{lighthouseUtils.js => lighthouseUtils.ts} | 15 +- web/src/utils/{linkUtils.js => linkUtils.ts} | 60 +- web/src/views/{Charts.jsx => Charts.tsx} | 147 +- web/src/views/CompareReports.tsx | 579 ++++++++ web/src/views/{Content.jsx => Content.tsx} | 18 +- ...tentAnalytics.jsx => ContentAnalytics.tsx} | 198 ++- web/src/views/{Gallery.jsx => Gallery.tsx} | 69 +- web/src/views/{Home.jsx => Home.tsx} | 72 +- web/src/views/{Issues.jsx => Issues.tsx} | 59 +- ...wordsExplorer.jsx => KeywordsExplorer.tsx} | 63 +- .../views/{Lighthouse.jsx => Lighthouse.tsx} | 87 +- web/src/views/{Links.jsx => Links.tsx} | 136 +- web/src/views/{Network.jsx => Network.tsx} | 100 +- web/src/views/{Overview.jsx => Overview.tsx} | 175 +-- web/src/views/Pipeline.tsx | 106 ++ .../views/{Redirects.jsx => Redirects.tsx} | 14 +- ...hPerformance.jsx => SearchPerformance.tsx} | 85 +- web/src/views/{Security.jsx => Security.tsx} | 44 +- .../{SiteStructure.jsx => SiteStructure.tsx} | 24 +- .../views/{TechStack.jsx => TechStack.tsx} | 40 +- web/src/views/{Traffic.jsx => Traffic.tsx} | 76 +- web/tsconfig.json | 23 + web/tsconfig.strict.json | 6 + 250 files changed, 16070 insertions(+), 7085 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_initial_schema.py create mode 100644 alembic/versions/002_perf_columns_and_indexes.py create mode 100644 data/pipeline-config.txt create mode 100644 docker-entrypoint.sh create mode 100644 tests/test_storage_bulk.py create mode 100644 tests/test_suggest_cache_concurrent.py create mode 100644 web/app/(reports)/[slug]/page.tsx create mode 100644 web/app/(reports)/layout.tsx create mode 100644 web/app/(reports)/not-found.tsx delete mode 100644 web/app/[slug]/page.jsx rename web/app/api/integrations/google/auth/{route.js => route.ts} (62%) delete mode 100644 web/app/api/integrations/google/callback/route.js create mode 100644 web/app/api/integrations/google/callback/route.ts rename web/app/api/integrations/google/credentials/{route.js => route.ts} (77%) rename web/app/api/integrations/google/credentials/upload/{route.js => route.ts} (55%) rename web/app/api/integrations/google/disconnect/{route.js => route.ts} (62%) delete mode 100644 web/app/api/integrations/google/keywords/by-page/route.js create mode 100644 web/app/api/integrations/google/keywords/by-page/route.ts rename web/app/api/integrations/google/keywords/expand/{route.js => route.ts} (60%) delete mode 100644 web/app/api/integrations/google/keywords/history/batch/route.js create mode 100644 web/app/api/integrations/google/keywords/history/batch/route.ts delete mode 100644 web/app/api/integrations/google/keywords/history/route.js create mode 100644 web/app/api/integrations/google/keywords/history/route.ts delete mode 100644 web/app/api/integrations/google/page-data/route.js create mode 100644 web/app/api/integrations/google/page-data/route.ts rename web/app/api/integrations/google/properties/{route.js => route.ts} (55%) delete mode 100644 web/app/api/integrations/google/status/route.js create mode 100644 web/app/api/integrations/google/status/route.ts rename web/app/api/integrations/google/test/{route.js => route.ts} (53%) rename web/app/api/jobs/[id]/{route.js => route.ts} (64%) rename web/app/api/llm-config/{route.js => route.ts} (72%) rename web/app/api/pipeline-config/{route.js => route.ts} (63%) delete mode 100644 web/app/api/report-db/route.js create mode 100644 web/app/api/report/crawl-payload/route.ts rename web/app/api/report/meta/{route.js => route.ts} (64%) rename web/app/api/report/payload/{route.js => route.ts} (74%) delete mode 100644 web/app/api/report/portfolio/route.js create mode 100644 web/app/api/report/portfolio/route.ts rename web/app/api/run/{route.js => route.ts} (76%) rename web/app/{client-providers.jsx => client-providers.tsx} (51%) rename web/app/{layout.jsx => layout.tsx} (90%) delete mode 100644 web/app/page.jsx create mode 100644 web/app/page.tsx create mode 100644 web/app/pipeline/page.tsx create mode 100644 web/global.d.ts delete mode 100644 web/jsconfig.json delete mode 100644 web/src/ReportShell.jsx create mode 100644 web/src/ReportShell.tsx create mode 100644 web/src/components/AppShell.tsx rename web/src/components/{Badge.jsx => Badge.tsx} (80%) rename web/src/components/{Button.jsx => Button.tsx} (73%) rename web/src/components/{Card.jsx => Card.tsx} (67%) create mode 100644 web/src/components/GoogleIntegrationsPanel.tsx delete mode 100644 web/src/components/IntegrationsModal.jsx create mode 100644 web/src/components/IntegrationsModal.tsx delete mode 100644 web/src/components/PageHeader.jsx create mode 100644 web/src/components/PageHeader.tsx rename web/src/components/{PageLayout.jsx => PageLayout.tsx} (60%) delete mode 100644 web/src/components/PipelineRunnerFab.jsx create mode 100644 web/src/components/ReportCompareControls.tsx delete mode 100644 web/src/components/ReportSelector.jsx create mode 100644 web/src/components/ReportSelector.tsx rename web/src/components/{ReportShellSkeleton.jsx => ReportShellSkeleton.tsx} (93%) rename web/src/components/{Skeleton.jsx => Skeleton.tsx} (94%) rename web/src/components/{Table.jsx => Table.tsx} (53%) rename web/src/components/{ThemeToggle.jsx => ThemeToggle.tsx} (87%) create mode 100644 web/src/components/compare/CompareCharts.tsx create mode 100644 web/src/components/compare/CompareDeltaBadge.tsx create mode 100644 web/src/components/compare/CompareTabPanels.tsx rename web/src/components/google/{GoogleChartCard.jsx => GoogleChartCard.tsx} (64%) rename web/src/components/google/{GoogleTableToolbar.jsx => GoogleTableToolbar.tsx} (75%) rename web/src/components/google/{GoogleTimeSeriesChart.jsx => GoogleTimeSeriesChart.tsx} (92%) rename web/src/components/google/{GoogleViewTabs.jsx => GoogleViewTabs.tsx} (76%) rename web/src/components/google/{SortablePaginatedTable.jsx => SortablePaginatedTable.tsx} (88%) rename web/src/components/google/{SummaryCard.jsx => SummaryCard.tsx} (53%) rename web/src/components/google/{UrlCoverageDoughnut.jsx => UrlCoverageDoughnut.tsx} (84%) rename web/src/components/google/{UrlGapListsPanel.jsx => UrlGapListsPanel.tsx} (63%) rename web/src/components/google/{tableUtils.js => tableUtils.ts} (68%) delete mode 100644 web/src/components/index.js create mode 100644 web/src/components/index.ts rename web/src/components/keywordsExplorer/{KeywordCharts.jsx => KeywordCharts.tsx} (83%) rename web/src/components/keywordsExplorer/{KeywordPanels.jsx => KeywordPanels.tsx} (85%) rename web/src/components/keywordsExplorer/{KeywordTableColumns.jsx => KeywordTableColumns.tsx} (52%) rename web/src/components/keywordsExplorer/{keywordTableUtils.js => keywordTableUtils.ts} (76%) rename web/src/components/lighthouse/{DiagnosticGroup.jsx => DiagnosticGroup.tsx} (71%) rename web/src/components/lighthouse/{DiagnosticItem.jsx => DiagnosticItem.tsx} (92%) rename web/src/components/lighthouse/{LhAuditExpandable.jsx => LhAuditExpandable.tsx} (85%) rename web/src/components/lighthouse/{LhDetailsTable.jsx => LhDetailsTable.tsx} (62%) rename web/src/components/lighthouse/{MultiPageTable.jsx => MultiPageTable.tsx} (81%) rename web/src/components/lighthouse/{QuickWinCard.jsx => QuickWinCard.tsx} (82%) rename web/src/components/lighthouse/{ScoreRing.jsx => ScoreRing.tsx} (87%) rename web/src/components/lighthouse/{ThresholdBar.jsx => ThresholdBar.tsx} (92%) rename web/src/components/lighthouse/{index.js => index.ts} (100%) rename web/src/components/links/{CharBar.jsx => CharBar.tsx} (73%) rename web/src/components/links/{CopyBtn.jsx => CopyBtn.tsx} (80%) rename web/src/components/links/{HeadingPills.jsx => HeadingPills.tsx} (83%) rename web/src/components/links/{InlineRing.jsx => InlineRing.tsx} (81%) rename web/src/components/links/{InspectorTabs.jsx => InspectorTabs.tsx} (84%) rename web/src/components/links/{MiniBar.jsx => MiniBar.tsx} (80%) rename web/src/components/links/{OGPreview.jsx => OGPreview.tsx} (81%) rename web/src/components/links/{RowTooltip.jsx => RowTooltip.tsx} (79%) rename web/src/components/links/{SecHeaderRow.jsx => SecHeaderRow.tsx} (93%) rename web/src/components/links/{SortTh.jsx => SortTh.tsx} (76%) rename web/src/components/links/{index.js => index.ts} (100%) rename web/src/components/links/tabs/{ContentTab.jsx => ContentTab.tsx} (90%) rename web/src/components/links/tabs/{IssuesTab.jsx => IssuesTab.tsx} (88%) rename web/src/components/links/tabs/{OverviewTab.jsx => OverviewTab.tsx} (95%) rename web/src/components/links/tabs/{PageAnalysisTab.jsx => PageAnalysisTab.tsx} (88%) rename web/src/components/links/tabs/{SeoSocialTab.jsx => SeoSocialTab.tsx} (94%) rename web/src/components/links/tabs/{TechnicalTab.jsx => TechnicalTab.tsx} (88%) create mode 100644 web/src/components/pipeline/ConfigField.tsx create mode 100644 web/src/components/pipeline/PipelineLogViewer.tsx create mode 100644 web/src/components/pipeline/PipelineRunPanel.tsx create mode 100644 web/src/components/pipeline/PipelineRunnerFab.tsx create mode 100644 web/src/components/pipeline/PipelineSettingsPanel.tsx create mode 100644 web/src/components/pipeline/PipelineSettingsSectionTabs.tsx create mode 100644 web/src/components/pipeline/PipelineShell.tsx create mode 100644 web/src/components/pipeline/PipelineWizardProgress.tsx create mode 100644 web/src/components/pipeline/pipelinePresets.ts create mode 100644 web/src/components/pipeline/pipelineSettingsGroups.ts create mode 100644 web/src/components/pipeline/pipelineUi.tsx rename web/src/components/searchPerformance/{GscCharts.jsx => GscCharts.tsx} (80%) rename web/src/components/searchPerformance/{gscTableUtils.js => gscTableUtils.ts} (66%) rename web/src/components/siteStructure/{PathTreeTable.jsx => PathTreeTable.tsx} (86%) rename web/src/components/traffic/{Ga4Charts.jsx => Ga4Charts.tsx} (78%) rename web/src/components/traffic/{ga4TableUtils.js => ga4TableUtils.ts} (71%) create mode 100644 web/src/context/PipelineContext.tsx delete mode 100644 web/src/context/ReportContext.jsx create mode 100644 web/src/context/ReportContext.tsx rename web/src/context/{ThemeProvider.jsx => ThemeProvider.tsx} (71%) delete mode 100644 web/src/context/reportContext.js create mode 100644 web/src/context/reportContextTypes.ts delete mode 100644 web/src/context/themeContext.js create mode 100644 web/src/context/themeContext.ts rename web/src/context/{themeUtils.js => themeUtils.ts} (67%) delete mode 100644 web/src/context/useReport.js create mode 100644 web/src/context/useReport.ts rename web/src/context/{useTheme.js => useTheme.ts} (53%) create mode 100644 web/src/lib/appNav.ts rename web/src/lib/{auditSqlExamples.js => auditSqlExamples.ts} (80%) rename web/src/lib/{badges.js => badges.ts} (81%) create mode 100644 web/src/lib/compareChartData.ts rename web/src/lib/{domainSlug.js => domainSlug.ts} (61%) create mode 100644 web/src/lib/formatPipelineLog.ts rename web/src/lib/{gscMetrics.js => gscMetrics.ts} (67%) delete mode 100644 web/src/lib/homePortfolio.js create mode 100644 web/src/lib/homePortfolio.ts rename web/src/lib/{llmConfigSchema.js => llmConfigSchema.ts} (79%) delete mode 100644 web/src/lib/loadReportDb.js create mode 100644 web/src/lib/loadReportDb.ts rename web/src/lib/{pipelineConfigSchema.js => pipelineConfigSchema.ts} (62%) create mode 100644 web/src/lib/pipelineDebug.ts delete mode 100644 web/src/lib/pipelineJobEvents.js create mode 100644 web/src/lib/pipelineJobEvents.ts create mode 100644 web/src/lib/pipelineReturn.ts rename web/src/lib/{publicBase.js => publicBase.ts} (57%) create mode 100644 web/src/lib/reportCompare.ts create mode 100644 web/src/lib/reportCompareExtras.ts rename web/src/lib/{reportDiff.js => reportDiff.ts} (68%) rename web/src/lib/{reportNav.js => reportNav.ts} (56%) create mode 100644 web/src/lib/reportTimestamps.ts rename web/src/lib/{siteStructureTree.js => siteStructureTree.ts} (63%) rename web/src/lib/{strings.js => strings.ts} (67%) rename web/src/{patchConsole.js => patchConsole.ts} (61%) delete mode 100644 web/src/routes.js create mode 100644 web/src/routes.ts create mode 100644 web/src/server/db.ts rename web/src/server/{googleSecrets.js => googleSecrets.ts} (66%) delete mode 100644 web/src/server/llmConfig.js create mode 100644 web/src/server/llmConfig.ts rename web/src/server/{localOnly.js => localOnly.ts} (54%) create mode 100644 web/src/server/oauthReturn.ts delete mode 100644 web/src/server/pipelineConfig.js create mode 100644 web/src/server/pipelineConfig.ts rename web/src/server/{pipelineJobs.js => pipelineJobs.ts} (55%) delete mode 100644 web/src/server/pipelineSpawnEnv.js create mode 100644 web/src/server/pipelineSpawnEnv.ts create mode 100644 web/src/server/reportDb.ts delete mode 100644 web/src/server/reportSqlite.js create mode 100644 web/src/server/resolvePython.ts create mode 100644 web/src/types/api.ts create mode 100644 web/src/types/chart.ts create mode 100644 web/src/types/components.ts create mode 100644 web/src/types/index.ts create mode 100644 web/src/types/report.ts create mode 100644 web/src/types/strings.ts rename web/src/utils/{chartJsDefaults.js => chartJsDefaults.ts} (68%) create mode 100644 web/src/utils/chartOptions.ts rename web/src/utils/{chartPalette.js => chartPalette.ts} (59%) create mode 100644 web/src/utils/errorMessage.ts rename web/src/utils/{lighthouseUtils.js => lighthouseUtils.ts} (78%) rename web/src/utils/{linkUtils.js => linkUtils.ts} (71%) rename web/src/views/{Charts.jsx => Charts.tsx} (81%) create mode 100644 web/src/views/CompareReports.tsx rename web/src/views/{Content.jsx => Content.tsx} (96%) rename web/src/views/{ContentAnalytics.jsx => ContentAnalytics.tsx} (89%) rename web/src/views/{Gallery.jsx => Gallery.tsx} (90%) rename web/src/views/{Home.jsx => Home.tsx} (83%) rename web/src/views/{Issues.jsx => Issues.tsx} (88%) rename web/src/views/{KeywordsExplorer.jsx => KeywordsExplorer.tsx} (88%) rename web/src/views/{Lighthouse.jsx => Lighthouse.tsx} (85%) rename web/src/views/{Links.jsx => Links.tsx} (83%) rename web/src/views/{Network.jsx => Network.tsx} (68%) rename web/src/views/{Overview.jsx => Overview.tsx} (88%) create mode 100644 web/src/views/Pipeline.tsx rename web/src/views/{Redirects.jsx => Redirects.tsx} (90%) rename web/src/views/{SearchPerformance.jsx => SearchPerformance.tsx} (86%) rename web/src/views/{Security.jsx => Security.tsx} (88%) rename web/src/views/{SiteStructure.jsx => SiteStructure.tsx} (88%) rename web/src/views/{TechStack.jsx => TechStack.tsx} (82%) rename web/src/views/{Traffic.jsx => Traffic.tsx} (84%) create mode 100644 web/tsconfig.json create mode 100644 web/tsconfig.strict.json diff --git a/.gitignore b/.gitignore index 31d0df48..c67da0e2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,13 +2,12 @@ # Next.js UI: generated pipeline configs from the runner modal (repo root; must match Python cwd for paths) .website-profiling-ui-*.txt -# Legacy: configs under subfolder broke sqlite_db = report.db vs Next (kept for ignore if present) +# Legacy UI pipeline configs under subfolder (kept for ignore if present) .web-pipeline/ -# WebsiteProfiling generated outputs +# Legacy SQLite outputs (if present locally) report.db report.db.* - report.db-journal nodes.json @@ -16,6 +15,6 @@ nodes.json .secrets/ web/.env.local -# Legacy local config file (use report.db / web UI instead) +# Legacy local config file (use PostgreSQL pipeline_config / web UI instead) input.txt *__pycache__* \ No newline at end of file diff --git a/AGENT.md b/AGENT.md index e8d1d7f4..a3d0eee9 100644 --- a/AGENT.md +++ b/AGENT.md @@ -1,25 +1,27 @@ # Agent instructions -- WebsiteProfiling -**What it is:** `python -m src` from repo root (`src/__main__.py` -> package **`website_profiling`**). Config: stored in **`report.db`** (`pipeline_config` table, `key/value/is_unknown/updated_at`). A shadow **`pipeline-config.txt`** is auto-written next to `report.db` on every Save/Run. CLI loads DB first (`REPORT_DB_PATH` or `cwd/report.db`), then shadow file; `--config` overrides with a file. Reference keys: `input.txt.example` (not auto-loaded). +**What it is:** `python -m src` from repo root (`src/__main__.py` -> package **`website_profiling`**). Config: stored in **PostgreSQL** (`pipeline_config` table, `key/value/is_unknown/updated_at`). A shadow **`pipeline-config.txt`** is auto-written to `DATA_DIR` on every Save/Run. CLI loads DB first (`DATABASE_URL`), then shadow file; `--config` overrides with a file. Reference keys: `input.txt.example` (not auto-loaded). -**LLM / AI:** Settings live in **`llm_config`** table in the same `report.db`. Configure only via web UI **AI** tab (`GET/PUT /api/llm-config`, localhost). Never in `pipeline-config.txt` or `--config`. +**LLM / AI:** Settings live in **`llm_config`** table in PostgreSQL. Configure only via web UI **AI** tab (`GET/PUT /api/llm-config`, localhost). Never in `pipeline-config.txt` or `--config`. -**Frontend:** **`web/`** (Next.js) -- server reads `report.db` via `/api/report/*`. +**Frontend:** **`web/`** (Next.js) -- server reads PostgreSQL via `/api/report/*`. **Key paths** - `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `analysis/`, `llm/`, `tools/` -- `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.js`, `server/pipelineConfig.js`, `server/llmConfig.js` +- `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.js`, `server/pipelineConfig.js`, `server/llmConfig.js`, `server/db.js` +- `alembic/` -- schema migrations **Run / APIs** -- Pipeline: `python -m src` — reads config from `report.db` (`pipeline_config`); shadow `pipeline-config.txt` if table empty. CLI override: `python -m src --config path` +- Pipeline: `python -m src` — reads config from PostgreSQL (`pipeline_config`); shadow `pipeline-config.txt` if table empty. CLI override: `python -m src --config path` - Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` -- **`preserve_crawl_history`** (default true): append crawls; `false` recreates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs` -- **`enrich_keywords_after_report`**: when omitted or `auto` (UI: Auto), follows `enable_google_search_console`; when set to Yes/No, explicit override -- **`REPORT_DB_PATH`** env: DB path used by both Python and Next.js (Docker: `/data/report.db`; local default: `report.db` at repo root). Pipeline config lives in this DB. -- **`web/`:** `/api/report/*` (SQLite); `/api/run` spawns Python (localhost only); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `PipelineRunnerFab` saves pipeline + LLM state before each run -- **Docker:** `Dockerfile` + `docker-compose.yml`; **`LIGHTHOUSE_CHROME_FLAGS`** +- **`preserve_crawl_history`** (default true): append crawls; `false` truncates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs` +- **`DATABASE_URL`** env: PostgreSQL connection string (required). **`DATA_DIR`**: secrets + shadow config (Docker: `/data`). +- **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. +- **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. +- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `PipelineRunnerFab` saves pipeline + LLM state before each run +- **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`LIGHTHOUSE_CHROME_FLAGS`** **Where to edit** @@ -27,7 +29,7 @@ |------|--------| | Crawl | `crawl/crawler.py` | | Report | `reporting/builder.py`, `reporting/categories.py` | -| DB schema | `db/storage.py` `init_schema` | +| DB schema | `alembic/versions/` | | Local analysis | `analysis/local.py`, `requirements.txt` | | LLM enrichment | `llm/enrich.py`, `llm_config.py`, `requirements-llm.txt` | | Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` | @@ -35,4 +37,4 @@ | UI LLM schema | `web/src/lib/llmConfigSchema.js` | | UI config I/O | `web/src/server/pipelineConfig.js`, `web/src/server/llmConfig.js` | -Schema changes: edit `init_schema` only (no migration layer). +Schema changes: add Alembic migration (`alembic revision`). diff --git a/Dockerfile b/Dockerfile index ba18fb49..b6f0dd9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,13 +34,17 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ NEXT_TELEMETRY_DISABLED=1 \ WEBSITE_PROFILING_ROOT=/app \ - REPORT_DB_PATH=/data/report.db \ + DATABASE_URL=postgres://profiling:profiling@postgres:5432/website_profiling \ + DATA_DIR=/data \ PYTHON=/opt/venv/bin/python \ - CHROME_PATH=/usr/bin/chromium + CHROME_PATH=/usr/bin/chromium \ + LIGHTHOUSE_PATH=/usr/local/bin/lighthouse # Python: base requirements + optional LLM API clients COPY requirements.txt /app/requirements.txt COPY requirements-llm.txt /app/requirements-llm.txt +COPY alembic.ini /app/alembic.ini +COPY alembic /app/alembic RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m venv /opt/venv \ && /opt/venv/bin/pip install --upgrade pip \ @@ -49,6 +53,11 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && ln -sf /opt/venv/bin/python /usr/local/bin/python \ && ln -sf /opt/venv/bin/python /usr/local/bin/python3 +# Pre-install Lighthouse CLI (avoid flaky parallel `npx -y lighthouse` at runtime). +RUN --mount=type=cache,target=/root/.npm \ + npm install -g lighthouse@12.6.0 \ + && lighthouse --version + WORKDIR /app # Next.js install + build (layer cache) @@ -59,15 +68,17 @@ RUN --mount=type=cache,target=/root/.npm \ # Application source COPY src /app/src COPY web /app/web +COPY alembic /app/alembic +COPY alembic.ini /app/alembic.ini +COPY docker-entrypoint.sh /app/docker-entrypoint.sh RUN cd /app/web && npm run build && npm prune --omit=dev ENV NODE_ENV=production -# Persisted DB directory (bind mount or volume in compose) -RUN mkdir -p /data +# Persisted data directory (secrets + shadow config) +RUN mkdir -p /data && chmod +x /app/docker-entrypoint.sh EXPOSE 3000 -# Listen on all interfaces so the container is reachable from the host -CMD ["sh", "-c", "cd /app/web && npm run start -- -H 0.0.0.0 -p 3000"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/Docs.md b/Docs.md index c0e10879..a6aadf46 100644 --- a/Docs.md +++ b/Docs.md @@ -1,6 +1,6 @@ # Crawl overview -**What it does:** Starts from `start_url` in **pipeline config** (`report.db` / web UI), fetches HTML with HTTP GET, parses `` with BeautifulSoup (static HTML only—no JS execution), normalizes links, filters (same-site, robots, depth, excludes), and queues until `max_pages` or the queue is empty. +**What it does:** Starts from `start_url` in **pipeline config** (PostgreSQL / web UI), fetches HTML with HTTP GET, parses `` with BeautifulSoup (static HTML only—no JS execution), normalizes links, filters (same-site, robots, depth, excludes), and queues until `max_pages` or the queue is empty. **Main code:** `src/website_profiling/crawl/crawler.py`, `src/website_profiling/common.py`. diff --git a/README.md b/README.md index e2ff1237..7ae7d3ac 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,31 @@ docker compose up --build Open **http://localhost:3000/home**. Use **`http://localhost:3000`** +Docker Compose starts **PostgreSQL** and the web app. Data persists in Docker volumes (`pg-data` for the database, `profiling-data` for secrets and shadow config). + ## Run locally -**1. Python** (repo root) +**1. PostgreSQL** + +```bash +docker run -d --name wp-pg \ + -e POSTGRES_PASSWORD=dev \ + -e POSTGRES_DB=website_profiling \ + -p 5432:5432 postgres:16-alpine + +export DATABASE_URL=postgres://postgres:dev@localhost:5432/website_profiling +export DATA_DIR=$(pwd)/data +mkdir -p "$DATA_DIR" +``` + +Apply schema: + +```bash +pip install -r requirements.txt +alembic upgrade head +``` + +**2. Python** (repo root) ```bash python -m venv .venv @@ -24,37 +46,61 @@ Activate `.venv`, then: pip install -r requirements.txt ``` -Optional LLM enrichment: `pip install -r requirements-llm.txt` — configure in the web UI **AI** tab only (not via `pipeline-config.txt`). +Optional LLM enrichment: `pip install -r requirements-llm.txt` — configure in the web UI **AI** tab only. -**2. Configure & run the pipeline** +**3. Configure & run the pipeline** The easiest way is via the **web UI** (terminal icon, bottom-right corner at `http://localhost:3000`): -- Settings are stored in `report.db` (`pipeline_config` table) — the same database used for crawl data. Back up the whole pipeline by copying one file. -- A shadow `pipeline-config.txt` is auto-written next to `report.db` on every Save/Run (safe to delete; regenerated automatically). +- Settings are stored in PostgreSQL (`pipeline_config` table). +- A shadow `pipeline-config.txt` is auto-written to `DATA_DIR` on every Save/Run. - On first open, if the table is empty, the UI imports from shadow `pipeline-config.txt` (if present). -- Click **Save settings** to persist, or **Run pipeline** to save + run immediately. -- **AI enrichment** (OpenAI, Gemini, Claude, Ollama): use the **AI** tab in the pipeline runner — settings are stored in `llm_config` inside `report.db` only (not in `pipeline-config.txt`). +- **AI enrichment**: use the **AI** tab — settings live in `llm_config` in PostgreSQL only. -To run from the CLI instead: +To run from the CLI: ```bash +export DATABASE_URL=postgres://postgres:dev@localhost:5432/website_profiling python -m src ``` -Python reads settings from `report.db` (`pipeline_config` table) by default — use the web UI to configure, or set `REPORT_DB_PATH` to point at your database (Docker sets this automatically). If the table is empty, the CLI falls back to shadow `pipeline-config.txt` next to `report.db`. Override with `--config path` for a custom key=value file. Steps: `crawl`, `report`, `plot`, `lighthouse`, `keywords`, `warnings`, `enrich` as extra args. - -> **Reference:** `input.txt.example` shows all config keys in the legacy file format (optional; not loaded automatically). - -**3. Next.js UI** (`web/`) +**4. Next.js UI** (`web/`) ```bash cd web npm install +export DATABASE_URL=postgres://postgres:dev@localhost:5432/website_profiling +export DATA_DIR=../data npm run dev ``` Open **http://localhost:3000/home**. +If pipeline runs fail with `spawn python ENOENT`, macOS often has no `python` on PATH (only `python3`). The server auto-resolves `.venv/bin/python` or `python3`; you can also set `export PYTHON="$(pwd)/.venv/bin/python"` before `npm run dev`, or set **Python executable** under Pipeline → Advanced. + +### PostgreSQL performance tuning + +Optional environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `DB_POOL_MIN` | 2 | Python pipeline minimum pool connections | +| `DB_POOL_MAX` | 20 | Python pipeline maximum pool connections | +| `PGPOOL_MAX` | 20 | Next.js `pg` pool size | + +Pipeline config (web UI or shadow file): + +- **`crawl_stream_to_db`** — batch-write crawl rows during fetch (auto-enabled when `max_pages > 100`). +- **`lighthouse_concurrency`** — parallel Lighthouse URL audits (default 2). +- **`llm_concurrency`** (AI tab) — parallel LLM API batches (default 2). + +Benchmark crawl writes: `python scripts/bench_crawl_write.py -n 1000` (requires `DATABASE_URL`). + +### Backup + +```bash +pg_dump -Fc "$DATABASE_URL" -f backup.dump +``` + --- ## Google Search Console + GA4 Integration @@ -81,7 +127,7 @@ Pull real search and traffic data into your reports. ### CLI usage ```bash -# Fetch GSC + GA4 data and store in report.db (uses pipeline_config in report.db) +# Fetch GSC + GA4 data and store in PostgreSQL python -m src google # Validate credentials only (does not store data) diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..cd64a16f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,42 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os + +sqlalchemy.url = driver://user:pass@localhost/dbname + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..5e36511b --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,50 @@ +"""Alembic environment — uses DATABASE_URL from the environment.""" +from __future__ import annotations + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = None + + +def get_url() -> str: + url = (os.environ.get("DATABASE_URL") or "").strip() + if not url: + raise RuntimeError("DATABASE_URL is required for Alembic migrations") + if url.startswith("postgres://"): + return "postgresql+psycopg://" + url[len("postgres://") :] + if url.startswith("postgresql://") and "+psycopg" not in url: + return "postgresql+psycopg://" + url[len("postgresql://") :] + return url + + +def run_migrations_offline() -> None: + context.configure( + url=get_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = create_engine(get_url(), poolclass=pool.NullPool) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..04ea8276 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial_schema.py b/alembic/versions/001_initial_schema.py new file mode 100644 index 00000000..665a1b5e --- /dev/null +++ b/alembic/versions/001_initial_schema.py @@ -0,0 +1,173 @@ +"""Initial PostgreSQL schema for WebsiteProfiling. + +Revision ID: 001 +Revises: +Create Date: 2026-06-02 +""" +from __future__ import annotations + +from alembic import op + +revision = "001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE crawl_runs ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + start_url TEXT + ); + + CREATE TABLE crawl_results ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + url TEXT NOT NULL, + data JSONB NOT NULL, + UNIQUE (crawl_run_id, url) + ); + CREATE INDEX idx_crawl_results_run ON crawl_results(crawl_run_id); + + CREATE TABLE edges ( + crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + from_url TEXT NOT NULL, + to_url TEXT NOT NULL, + PRIMARY KEY (crawl_run_id, from_url, to_url) + ); + + CREATE TABLE nodes ( + crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + url TEXT NOT NULL, + count INTEGER NOT NULL, + PRIMARY KEY (crawl_run_id, url) + ); + + CREATE TABLE lighthouse_summary ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE lighthouse_runs ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + url TEXT NOT NULL, + strategy TEXT NOT NULL, + run_index INTEGER NOT NULL, + data JSONB NOT NULL + ); + + CREATE TABLE lighthouse_page_summaries ( + url TEXT PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE report_payload ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + generated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE google_data ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE keyword_data ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE keyword_history ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + keyword TEXT NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + position DOUBLE PRECISION, + clicks INTEGER, + impressions INTEGER, + ctr DOUBLE PRECISION + ); + CREATE INDEX idx_kw_history_keyword ON keyword_history(keyword); + + CREATE TABLE keyword_suggest_cache ( + cache_key TEXT PRIMARY KEY, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + data JSONB NOT NULL + ); + + CREATE TABLE lh_audits ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + run_id BIGINT NOT NULL REFERENCES lighthouse_runs(id) ON DELETE CASCADE, + audit_id TEXT NOT NULL, + category_id TEXT, + score DOUBLE PRECISION, + score_display_mode TEXT, + title TEXT, + description TEXT, + display_value TEXT, + numeric_value DOUBLE PRECISION, + help_text TEXT, + details_type TEXT, + details_headings JSONB, + details_meta JSONB + ); + CREATE INDEX idx_lh_audits_run_id ON lh_audits(run_id); + CREATE INDEX idx_lh_audits_run_audit ON lh_audits(run_id, audit_id); + CREATE INDEX idx_lh_audits_audit_id ON lh_audits(audit_id); + + CREATE TABLE lh_audit_items ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + audit_row_id BIGINT NOT NULL REFERENCES lh_audits(id) ON DELETE CASCADE, + item_index INTEGER NOT NULL, + row_data JSONB NOT NULL + ); + CREATE INDEX idx_lh_audit_items_audit_row ON lh_audit_items(audit_row_id); + + CREATE TABLE pipeline_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + is_unknown BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + + CREATE TABLE llm_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + is_secret BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + + CREATE TABLE llm_cache ( + cache_key TEXT PRIMARY KEY, + response_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + """) + + +def downgrade() -> None: + op.execute(""" + DROP TABLE IF EXISTS llm_cache CASCADE; + DROP TABLE IF EXISTS llm_config CASCADE; + DROP TABLE IF EXISTS pipeline_config CASCADE; + DROP TABLE IF EXISTS lh_audit_items CASCADE; + DROP TABLE IF EXISTS lh_audits CASCADE; + DROP TABLE IF EXISTS keyword_suggest_cache CASCADE; + DROP TABLE IF EXISTS keyword_history CASCADE; + DROP TABLE IF EXISTS keyword_data CASCADE; + DROP TABLE IF EXISTS google_data CASCADE; + DROP TABLE IF EXISTS report_payload CASCADE; + DROP TABLE IF EXISTS lighthouse_page_summaries CASCADE; + DROP TABLE IF EXISTS lighthouse_runs CASCADE; + DROP TABLE IF EXISTS lighthouse_summary CASCADE; + DROP TABLE IF EXISTS nodes CASCADE; + DROP TABLE IF EXISTS edges CASCADE; + DROP TABLE IF EXISTS crawl_results CASCADE; + DROP TABLE IF EXISTS crawl_runs CASCADE; + """) diff --git a/alembic/versions/002_perf_columns_and_indexes.py b/alembic/versions/002_perf_columns_and_indexes.py new file mode 100644 index 00000000..41ec2e3b --- /dev/null +++ b/alembic/versions/002_perf_columns_and_indexes.py @@ -0,0 +1,66 @@ +"""Denormalized columns and indexes for read/write performance. + +Revision ID: 002 +Revises: 001 +Create Date: 2026-06-02 +""" +from __future__ import annotations + +from alembic import op + +revision = "002" +down_revision = "001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE report_payload + ADD COLUMN IF NOT EXISTS site_name TEXT, + ADD COLUMN IF NOT EXISTS canonical_domain TEXT; + + UPDATE report_payload + SET site_name = COALESCE(site_name, data->>'site_name'), + canonical_domain = COALESCE(canonical_domain, NULL) + WHERE site_name IS NULL; + + ALTER TABLE crawl_results + ADD COLUMN IF NOT EXISTS status TEXT, + ADD COLUMN IF NOT EXISTS title TEXT; + + UPDATE crawl_results + SET status = COALESCE(status, data->>'status'), + title = COALESCE(title, data->>'title') + WHERE status IS NULL OR title IS NULL; + + CREATE INDEX IF NOT EXISTS idx_report_payload_generated_at + ON report_payload (generated_at DESC); + + CREATE INDEX IF NOT EXISTS idx_report_payload_canonical_domain + ON report_payload (canonical_domain); + + CREATE INDEX IF NOT EXISTS idx_crawl_results_run_status + ON crawl_results (crawl_run_id, status); + + DROP INDEX IF EXISTS idx_kw_history_keyword; + CREATE INDEX IF NOT EXISTS idx_kw_history_keyword_id + ON keyword_history (keyword, id DESC); + """) + + +def downgrade() -> None: + op.execute(""" + DROP INDEX IF EXISTS idx_kw_history_keyword_id; + CREATE INDEX IF NOT EXISTS idx_kw_history_keyword ON keyword_history (keyword); + + DROP INDEX IF EXISTS idx_crawl_results_run_status; + DROP INDEX IF EXISTS idx_report_payload_canonical_domain; + DROP INDEX IF EXISTS idx_report_payload_generated_at; + + ALTER TABLE crawl_results DROP COLUMN IF EXISTS title; + ALTER TABLE crawl_results DROP COLUMN IF EXISTS status; + + ALTER TABLE report_payload DROP COLUMN IF EXISTS canonical_domain; + ALTER TABLE report_payload DROP COLUMN IF EXISTS site_name; + """) diff --git a/data/pipeline-config.txt b/data/pipeline-config.txt new file mode 100644 index 00000000..a1494a71 --- /dev/null +++ b/data/pipeline-config.txt @@ -0,0 +1,78 @@ +# WebsiteProfiling config (shadow of pipeline_config table) +# Regenerated automatically by the web UI on every Save/Run. +# To use for CLI: python -m src --config pipeline-config.txt + +# --- Crawl --- +start_url = https://codefrydev.in +max_pages = 20 +concurrency = 8 +timeout = 12 +max_depth = 6 +polite_delay = 0.2 +ignore_robots = false +allow_external = false +store_outlinks = true +store_content_excerpt = true +content_excerpt_max_chars = 4096 +preserve_crawl_history = true +crawl_stream_to_db = false +crawl_exclude_urls = + +# --- Report --- +outbound_domain_max_rows = 200 +include_keyword_opportunities = true +site_name = +report_title = SEO report +max_fetch_for_edges = 300 +same_domain_only = true +max_nodes_plot = 400 +run_security_scan = true +security_scan_active = false +security_max_urls_probe = 20 + +# --- Lighthouse --- +lighthouse_url = +lighthouse_mode = navigation +lighthouse_strategy = desktop +lighthouse_categories = performance,accessibility,best-practices,seo +lighthouse_iterations = 1 +run_lighthouse = true +run_lighthouse_on_pages = true +lighthouse_max_pages = 2 +lighthouse_concurrency = 2 + +# --- Content analysis --- +enable_duplicate_detection = true +enable_language_detection = true +analysis_fuzzy_threshold = 92 +analysis_simhash_hamming = 0 +analysis_dup_max_pages = 2000 + +# --- Pipeline --- +run_crawl = true +run_report = true +run_plot = true + +# --- Google (GSC & GA4) --- +enable_google_search_console = false +enable_google_analytics = false +google_date_range_days = 28 +google_url_gap_list_limit = 200 + +# --- Basics --- +keyword_max_pages = 200 +keyword_gsc_max_rows = 25000 +brand_name = +keyword_seeds = + +# --- Expansion --- +enable_google_suggest = false +enable_google_trends = false +enable_wikipedia_topic = false +enable_datamuse = false +keyword_suggest_top_n = 20 +keyword_max_suggest_results = 8 + +# --- Advanced --- +warning_mapper_input = +warning_mapper_input_type = lighthouse diff --git a/docker-compose.yml b/docker-compose.yml index e97bc9b6..3d34d7ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,37 @@ services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: website_profiling + POSTGRES_USER: profiling + POSTGRES_PASSWORD: profiling + volumes: + - pg-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U profiling -d website_profiling"] + interval: 5s + timeout: 3s + retries: 5 + web: build: context: . dockerfile: Dockerfile image: website-profiling:latest + depends_on: + postgres: + condition: service_healthy ports: - "3000:3000" environment: WEBSITE_PROFILING_ROOT: /app - REPORT_DB_PATH: /data/report.db + DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling + DATA_DIR: /data PYTHON: /opt/venv/bin/python NODE_ENV: production GOOGLE_SECRETS_PATH: /data/.secrets/google.json - # Lighthouse in Docker: system Chromium + no-sandbox (Chrome requirement in containers) CHROME_PATH: /usr/bin/chromium + LIGHTHOUSE_PATH: /usr/local/bin/lighthouse LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu volumes: - profiling-data:/data @@ -25,4 +43,5 @@ services: start_period: 15s volumes: + pg-data: profiling-data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 00000000..af3152b2 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e +cd /app +/opt/venv/bin/alembic upgrade head +cd /app/web && exec npm run start -- -H 0.0.0.0 -p 3000 diff --git a/input.txt.example b/input.txt.example index 02955e2c..0bd9ef9f 100644 --- a/input.txt.example +++ b/input.txt.example @@ -1,5 +1,6 @@ # Reference only — not loaded automatically. Use the web UI (Pipeline runner) to -# save settings to report.db, or pass --config this-file for a one-off CLI run. +# save settings via the web UI (Pipeline runner), or pass --config this-file for a one-off CLI run. +# Requires DATABASE_URL (PostgreSQL). Shadow file is written to DATA_DIR by the UI. # WebsiteProfiling config (key = value). Comment lines start with #. # --- Crawl --- @@ -16,19 +17,16 @@ store_outlinks = true store_content_excerpt = true content_excerpt_max_chars = 4096 -sqlite_db = report.db # true = append crawl runs; false = replace crawl tables but keep reports, Google, keyword history, and crawl_runs metadata preserve_crawl_history = true +# crawl_stream_to_db = false # batch-write pages during crawl (auto when max_pages > 100) # enrich_keywords_after_report = true # optional; when omitted, follows enable_google_search_console -crawl_output = crawl_results.json +# Crawl, edges, nodes, and report payload are stored in PostgreSQL (no JSON/CSV export). # --- Report --- # Optional: cap outbound-domain table rows in report JSON; keyword opportunities on by default outbound_domain_max_rows = 200 include_keyword_opportunities = true -crawl_csv = crawl_results.json -edges_csv = edges.json -nodes_csv = nodes.json site_name = report_title = SEO report max_fetch_for_edges = 300 @@ -45,15 +43,14 @@ lighthouse_mode = navigation lighthouse_strategy = desktop lighthouse_categories = performance,accessibility,best-practices,seo lighthouse_iterations = 1 -lighthouse_output_dir = . run_lighthouse = true # Run on every 200 OK page; results show in Link Explorer inspector run_lighthouse_on_pages = true lighthouse_max_pages = 2 +lighthouse_concurrency = 2 -# --- Keywords (python -m src keywords) --- -keyword_output_dir = . +# --- Keywords (python -m src keywords; stored in PostgreSQL) --- keyword_max_pages = 200 # --- Content analysis (local; pip install covers rapidfuzz, langdetect) --- diff --git a/pipeline-config.txt b/pipeline-config.txt index a7d33b1f..f2389b01 100644 --- a/pipeline-config.txt +++ b/pipeline-config.txt @@ -1,4 +1,4 @@ -# WebsiteProfiling config (shadow of report.db pipeline_config table) +# WebsiteProfiling config (shadow of pipeline_config table in PostgreSQL) # Regenerated automatically by the web UI on every Save/Run. # To use for CLI: python -m src --config pipeline-config.txt @@ -14,28 +14,21 @@ allow_external = false store_outlinks = true store_content_excerpt = true content_excerpt_max_chars = 4096 -sqlite_db = report.db preserve_crawl_history = true -crawl_output = crawl_results.json crawl_exclude_urls = # --- Report --- +# Crawl, edges, nodes, and report payload live in PostgreSQL (no JSON/CSV files). outbound_domain_max_rows = 200 include_keyword_opportunities = true -crawl_csv = crawl_results.json -edges_csv = edges.json -nodes_csv = nodes.json site_name = report_title = SEO report -report_output = site_report.html max_fetch_for_edges = 300 same_domain_only = true max_nodes_plot = 400 run_security_scan = true security_scan_active = false security_max_urls_probe = 20 -security_findings_output = -lighthouse_summary_json = # --- Lighthouse --- lighthouse_url = @@ -43,13 +36,11 @@ lighthouse_mode = navigation lighthouse_strategy = desktop lighthouse_categories = performance,accessibility,best-practices,seo lighthouse_iterations = 1 -lighthouse_output_dir = . run_lighthouse = true run_lighthouse_on_pages = true lighthouse_max_pages = 2 -# --- Keywords --- -keyword_output_dir = . +# --- Keywords (stored in PostgreSQL keyword_data) --- keyword_max_pages = 200 # --- Content analysis (local; no PyTorch) --- @@ -82,7 +73,7 @@ keyword_gsc_max_rows = 25000 keyword_seeds = brand_name = -# --- Advanced --- -warning_mapper_input = +# --- Advanced (warnings command: PostgreSQL by default) --- +# Leave warning_mapper_input blank to map the latest Lighthouse run from DB. +warning_mapper_input = warning_mapper_input_type = lighthouse -warning_mapper_output = diff --git a/requirements.txt b/requirements.txt index 1470c044..15f9af3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,10 @@ google-analytics-admin>=0.22.0 # pytrends is OPTIONAL and frequently rate-limited — uncomment only if you need trend direction # pytrends>=4.9,<5 +# PostgreSQL +psycopg[binary,pool]>=3.2 +sqlalchemy>=2.0.0 +alembic>=1.13 + # Dev / test pytest>=7.0.0 diff --git a/src/website_profiling/analysis/page.py b/src/website_profiling/analysis/page.py index d2f14d6b..6ae3612e 100644 --- a/src/website_profiling/analysis/page.py +++ b/src/website_profiling/analysis/page.py @@ -12,7 +12,7 @@ from ..common import normalize_link -# Max URLs per resource list to limit SQLite / payload size +# Max URLs per resource list to limit DB / payload size LIST_CAP = 200 INLINE_SCRIPT_WARN_BYTES = 8192 _HEADING_ORDER = {"h1": 1, "h2": 2, "h3": 3, "h4": 4, "h5": 5, "h6": 6} diff --git a/src/website_profiling/cli.py b/src/website_profiling/cli.py index 7cb26ce5..03a5871c 100644 --- a/src/website_profiling/cli.py +++ b/src/website_profiling/cli.py @@ -3,39 +3,53 @@ """ import argparse import os +import shutil import sys +import tempfile import pandas as pd from .config import get_bool, get_float, get_int, get_list, load_config, load_config_from_db -def _default_db_path() -> str: - """report.db path: REPORT_DB_PATH env, else report.db in cwd.""" - env = (os.environ.get("REPORT_DB_PATH") or "").strip() - if env: - return os.path.abspath(env) - return os.path.abspath(os.path.join(os.getcwd(), "report.db")) +def _shadow_config_path() -> str: + from .db.storage import get_data_dir + return os.path.join(get_data_dir(), "pipeline-config.txt") -def _shadow_config_path(db_path: str) -> str: - return os.path.join(os.path.dirname(db_path) or os.getcwd(), "pipeline-config.txt") +def _require_database_url() -> None: + from .db.storage import get_database_url + get_database_url() -def _google_db_has_gsc(db_path: str) -> bool: +def _lighthouse_work_dir() -> str: + """Ephemeral directory for Lighthouse CLI JSON; deleted after PostgreSQL write.""" + return tempfile.mkdtemp(prefix="wp-lighthouse-") + + +def _cleanup_lighthouse_work_dir(work_dir: str) -> None: + if not work_dir: + return + tmp_root = os.path.realpath(tempfile.gettempdir()) + if os.path.realpath(work_dir).startswith(tmp_root): + shutil.rmtree(work_dir, ignore_errors=True) + + +def _google_db_has_gsc() -> bool: """True when the latest google_data row contains usable Search Console query data.""" - import json + from .db.storage import _parse_json_field - from .db import db_session, init_schema + from .db import db_session try: - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: cur = conn.execute("SELECT data FROM google_data ORDER BY id DESC LIMIT 1") row = cur.fetchone() if not row: return False - data = json.loads(row[0]) + data = _parse_json_field(row["data"]) + if not isinstance(data, dict): + return False gsc = data.get("gsc_full") or {} return bool(gsc.get("top_queries") or gsc.get("by_page")) except Exception: @@ -89,7 +103,7 @@ def main() -> None: "--config", "-c", default=None, - help="Optional key=value config file (default: pipeline_config in report.db)", + help="Optional key=value config file (default: pipeline_config in PostgreSQL)", ) parser.add_argument( "command", @@ -124,8 +138,8 @@ def main() -> None: # --- Config resolution order --- # 1. --config path: load that file (CLI override). - # 2. pipeline_config table in report.db (UI-managed; REPORT_DB_PATH or cwd/report.db). - # 3. Shadow pipeline-config.txt next to report.db. + # 2. pipeline_config table in PostgreSQL (UI-managed; DATABASE_URL). + # 3. Shadow pipeline-config.txt in DATA_DIR. # 4. Error with hint to save settings in the web UI. cfg: dict[str, str] = {} @@ -139,16 +153,18 @@ def main() -> None: cfg = load_config(cfg_path) cwd = os.path.dirname(cfg_path) or os.getcwd() else: - db_path = _default_db_path() - cfg = load_config_from_db(db_path) - cwd = os.path.dirname(db_path) or os.getcwd() + try: + _require_database_url() + except RuntimeError as e: + print(str(e), file=sys.stderr) + sys.exit(1) + cfg = load_config_from_db() + from .db.storage import get_data_dir + cwd = get_data_dir() if cfg: - print( - f"[Config] Loaded from report.db pipeline_config table ({db_path})", - flush=True, - ) + print("[Config] Loaded from pipeline_config table (PostgreSQL)", flush=True) else: - shadow = _shadow_config_path(db_path) + shadow = _shadow_config_path() if os.path.isfile(shadow): cfg = load_config(shadow) cwd = os.path.dirname(shadow) or os.getcwd() @@ -167,13 +183,7 @@ def path(key: str, default: str) -> str: p = os.path.join(cwd, p) return p - # When set, crawl/report/plot/lighthouse use SQLite instead of JSON/CSV - sqlite_db_raw = (cfg.get("sqlite_db") or "").strip() - db_path = path("sqlite_db", "report.db") if sqlite_db_raw else None - # Docker / hosting: Next.js uses REPORT_DB_PATH for the same DB; align pipeline writes with the UI reader - _env_db = (os.environ.get("REPORT_DB_PATH") or "").strip() - if _env_db and db_path is not None: - db_path = os.path.abspath(_env_db) + use_database = True # Single-command mode: lighthouse, keywords, warnings if args.command == "lighthouse": @@ -187,10 +197,21 @@ def path(key: str, default: str) -> str: lh_categories = cfg.get("lighthouse_categories", "").strip() lh_categories = get_list(cfg, "lighthouse_categories", sep=",") if lh_categories else None lh_iterations = get_int(cfg, "lighthouse_iterations", 3) or 3 - lh_out = cfg.get("lighthouse_output_dir", "").strip() or cwd - if not os.path.isabs(lh_out): - lh_out = os.path.join(cwd, lh_out) - sys.exit(lighthouse_main(url=lh_url, strategy=lh_strategy, iterations=lh_iterations, output_dir=lh_out, db_path=db_path, mode=lh_mode, categories=lh_categories)) + lh_out = _lighthouse_work_dir() + try: + sys.exit( + lighthouse_main( + url=lh_url, + strategy=lh_strategy, + iterations=lh_iterations, + output_dir=lh_out, + use_database=use_database, + mode=lh_mode, + categories=lh_categories, + ) + ) + finally: + _cleanup_lighthouse_work_dir(lh_out) if args.command == "keywords": # --expand-only: just run Suggest expansion and print JSON to stdout if getattr(args, "expand_only", False): @@ -207,13 +228,10 @@ def path(key: str, default: str) -> str: # --enrich-google: skip crawl-based extraction, go straight to enrichment if getattr(args, "enrich_google", False): - if not db_path: - print("keywords --enrich-google requires sqlite_db in config.", file=sys.stderr) - sys.exit(1) print("WebsiteProfiling: keywords Google enrichment only...", flush=True) from .integrations.google.keyword_enrich import run_enrichment try: - run_enrichment(db_path, cfg) + run_enrichment(cfg) print("Keywords enrichment done.", flush=True) sys.exit(0) except Exception as e: @@ -223,23 +241,15 @@ def path(key: str, default: str) -> str: print("WebsiteProfiling: keywords only", flush=True) from .tools.keywords import main as keyword_main kw_url = _require_start_url(cfg, for_step="keywords") - kw_out = cfg.get("keyword_output_dir", "").strip() or cwd - if not os.path.isabs(kw_out): - kw_out = os.path.join(cwd, kw_out) kw_cfg = dict(cfg) - kw_cfg["_cwd"] = cwd - # Pass db_path so keywords.py can write to keyword_data table - if db_path: - kw_cfg["_db_path"] = db_path - rc = keyword_main(base_url=kw_url, output_dir=kw_out, config=kw_cfg) - # Auto-run Google enrichment if configured - if rc == 0 and db_path and ( - get_bool(cfg, "enable_google_suggest", False) or _google_db_has_gsc(db_path) + rc = keyword_main(base_url=kw_url, config=kw_cfg) + if rc == 0 and ( + get_bool(cfg, "enable_google_suggest", False) or _google_db_has_gsc() ): print(" Running Google keyword enrichment...", flush=True) from .integrations.google.keyword_enrich import run_enrichment try: - run_enrichment(db_path, cfg) + run_enrichment(cfg) except Exception as e: print(f" Warning: Google enrichment error (non-fatal): {e}", file=sys.stderr) sys.exit(rc) @@ -248,25 +258,18 @@ def path(key: str, default: str) -> str: from .tools.warnings import main as warning_mapper_main wm_input = cfg.get("warning_mapper_input", "").strip() wm_type = (cfg.get("warning_mapper_input_type") or "lighthouse").lower() - wm_out = cfg.get("warning_mapper_output", "").strip() - if not wm_out: - wm_out = os.path.join(cwd, "warnings_mapped.json") - elif not os.path.isabs(wm_out): - wm_out = os.path.join(cwd, wm_out) - sys.exit(warning_mapper_main(input_path=wm_input, input_type=wm_type, output_path=wm_out)) + if wm_input and not os.path.isabs(wm_input): + wm_input = os.path.join(cwd, wm_input) + sys.exit(warning_mapper_main(input_path=wm_input or None, input_type=wm_type)) if args.command == "enrich": - if not db_path: - print("enrich requires sqlite_db in config.", file=sys.stderr) - sys.exit(1) print("WebsiteProfiling: enrich only (updates latest report payload)...", flush=True) - from .db import db_session, get_latest_crawl_run_id, init_schema, read_crawl, read_report_payload, write_report_payload + from .db import db_session, get_latest_crawl_run_id, read_crawl, read_report_payload, write_report_payload from .analysis import merge_analysis_into_payload, merge_bundles, run_local_enrichment from .llm.enrich import run_llm_enrichment from .llm_config import load_llm_config_from_db, llm_is_enabled - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: run_id = get_latest_crawl_run_id(conn) df = read_crawl(conn, run_id) payload = read_report_payload(conn) @@ -274,8 +277,8 @@ def path(key: str, default: str) -> str: print("No report_payload in DB. Run report first.", file=sys.stderr) sys.exit(1) local_bundle = run_local_enrichment(df, cfg) - llm_cfg = load_llm_config_from_db(db_path) - llm_bundle = run_llm_enrichment(df, llm_cfg, db_path=db_path) if llm_is_enabled(llm_cfg) else {} + llm_cfg = load_llm_config_from_db() + llm_bundle = run_llm_enrichment(df, llm_cfg) if llm_is_enabled(llm_cfg) else {} bundle = merge_bundles(local_bundle, llm_bundle) merge_analysis_into_payload(payload, bundle) write_report_payload(conn, payload) @@ -401,24 +404,18 @@ def path(key: str, default: str) -> str: print(f"Google test failed: {e}", file=sys.stderr) sys.exit(1) - # Full fetch: requires sqlite_db - if not db_path: - print("google command requires sqlite_db in config.", file=sys.stderr) - sys.exit(1) - + # Full fetch print("WebsiteProfiling: Google fetch...", flush=True) - from .db import db_session, get_latest_crawl_run_id, init_schema, read_crawl + from .db import db_session, get_latest_crawl_run_id, read_crawl from .integrations.google.store import write_google_data date_range_days = get_int(cfg, "google_date_range_days", 28) or 28 - # Read crawl URLs for join stats crawl_urls: list[str] = [] start_url_for_join = cfg.get("start_url", "") try: - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: run_id = get_latest_crawl_run_id(conn) if run_id is not None: df = read_crawl(conn, run_id) @@ -447,8 +444,7 @@ def path(key: str, default: str) -> str: sys.exit(1) # Store in google_data table - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: write_google_data(conn, google_data) if google_data.get("errors"): @@ -496,7 +492,7 @@ def path(key: str, default: str) -> str: preserve_crawl_history = get_bool(cfg, "preserve_crawl_history", True) store_content_excerpt = get_bool(cfg, "store_content_excerpt", False) content_excerpt_max_chars = get_int(cfg, "content_excerpt_max_chars", 4096) or 4096 - crawl_output = path("crawl_output", "crawl_results.csv") + crawl_stream_to_db = get_bool(cfg, "crawl_stream_to_db", False) print("Crawling...") run_crawler( start_url=start_url, @@ -508,30 +504,25 @@ def path(key: str, default: str) -> str: max_depth=max_depth, polite_delay=polite_delay, store_outlinks=store_outlinks, - output_csv=crawl_output if not db_path else None, - output_db=db_path, + output_csv=None, + output_db=use_database, show_progress=True, exclude_urls=exclude_urls if exclude_urls else None, preserve_crawl_history=preserve_crawl_history, store_content_excerpt=store_content_excerpt, content_excerpt_max_chars=content_excerpt_max_chars, + crawl_stream_to_db=crawl_stream_to_db, ) print("[Crawl] Done.", flush=True) - print(f"Crawl results: {db_path or crawl_output}") - crawl_csv = crawl_output - else: - crawl_csv = path("crawl_csv", "crawl_results.csv") - edges_csv = path("edges_csv", "edges.csv") - nodes_csv = path("nodes_csv", "nodes.csv") + print("Crawl results: PostgreSQL") # Run Lighthouse on every 200 OK page (when enabled); requires DB and crawl data lighthouse_summary_path_for_report = None - if run_lighthouse_on_pages and db_path: - from .db import db_session, get_latest_crawl_run_id, init_schema, read_crawl + if run_lighthouse_on_pages and use_database: + from .db import db_session, get_latest_crawl_run_id, read_crawl from .lighthouse.runner import run_lighthouse_on_pages as do_lighthouse_on_pages print("[Lighthouse on pages] Starting...", flush=True) - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: run_id = get_latest_crawl_run_id(conn) df = read_crawl(conn, run_id) success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns and not df.empty else pd.DataFrame() @@ -547,18 +538,19 @@ def path(key: str, default: str) -> str: lh_iterations = get_int(cfg, "lighthouse_iterations", 3) or 3 if run_lighthouse_on_pages: lh_iterations = 1 - lh_out = cfg.get("lighthouse_output_dir", "").strip() or cwd - if not os.path.isabs(lh_out): - lh_out = os.path.join(cwd, lh_out) - do_lighthouse_on_pages( - urls=urls_200, - strategy=lh_strategy, - iterations=lh_iterations, - output_dir=lh_out, - db_path=db_path, - mode=lh_mode, - categories=lh_categories if lh_categories else None, - ) + lh_out = _lighthouse_work_dir() + try: + do_lighthouse_on_pages( + urls=urls_200, + strategy=lh_strategy, + iterations=lh_iterations, + output_dir=lh_out, + mode=lh_mode, + categories=lh_categories if lh_categories else None, + concurrency=get_int(cfg, "lighthouse_concurrency", 2) or 2, + ) + finally: + _cleanup_lighthouse_work_dir(lh_out) print("[Lighthouse on pages] Done.", flush=True) # Run single-URL Lighthouse before report when enabled (and not running on all pages) @@ -572,24 +564,25 @@ def path(key: str, default: str) -> str: lh_mode = (cfg.get("lighthouse_mode") or "navigation").strip().lower() or "navigation" lh_categories = get_list(cfg, "lighthouse_categories", sep=",") lh_iterations = get_int(cfg, "lighthouse_iterations", 3) or 3 - lh_out = cfg.get("lighthouse_output_dir", "").strip() or cwd - if not os.path.isabs(lh_out): - lh_out = os.path.join(cwd, lh_out) - exit_code = lighthouse_main(url=lh_url, strategy=lh_strategy, iterations=lh_iterations, output_dir=lh_out, db_path=db_path, mode=lh_mode, categories=lh_categories if lh_categories else None) + lh_out = _lighthouse_work_dir() + try: + exit_code = lighthouse_main( + url=lh_url, + strategy=lh_strategy, + iterations=lh_iterations, + output_dir=lh_out, + use_database=use_database, + mode=lh_mode, + categories=lh_categories if lh_categories else None, + ) + finally: + _cleanup_lighthouse_work_dir(lh_out) if exit_code != 0: sys.exit(exit_code) print("[Lighthouse] Done.", flush=True) - lighthouse_summary_path_for_report = os.path.join(lh_out, "lighthouse_summary.json") if not db_path else None + lighthouse_summary_path_for_report = None if run_report: - if not db_path: - print( - "Report requires sqlite_db. Set sqlite_db = report.db in pipeline config " - "(web UI → Pipeline runner → Save). The Next.js UI reads report.db via /api/report/*.", - file=sys.stderr, - ) - sys.exit(1) - report_output = path("report_output", "site_report.html") max_fetch = get_int(cfg, "max_fetch_for_edges", 300) same_domain = get_bool(cfg, "same_domain_only", True) max_nodes = get_int(cfg, "max_nodes_plot", 400) @@ -599,22 +592,9 @@ def path(key: str, default: str) -> str: run_security_scan_flag = get_bool(cfg, "run_security_scan", True) security_scan_active = get_bool(cfg, "security_scan_active", False) security_max_urls_probe = get_int(cfg, "security_max_urls_probe", 20) or 20 - security_findings_output = (cfg.get("security_findings_output") or "").strip() - if security_findings_output and not os.path.isabs(security_findings_output): - security_findings_output = os.path.join(cwd, security_findings_output) - elif not security_findings_output: - security_findings_output = None - lighthouse_summary_path = (cfg.get("lighthouse_summary_json") or "").strip() - if lighthouse_summary_path and not os.path.isabs(lighthouse_summary_path): - lighthouse_summary_path = os.path.join(cwd, lighthouse_summary_path) - if not lighthouse_summary_path: - lighthouse_summary_path = lighthouse_summary_path_for_report from .reporting.builder import run_simple_report print("[Report] Starting...", flush=True) out = run_simple_report( - crawl_csv=crawl_csv, - edges_csv=edges_csv, - output_html=report_output, max_fetch_for_edges=max_fetch, concurrency=6, timeout=8, @@ -626,20 +606,19 @@ def path(key: str, default: str) -> str: run_security_scan_flag=run_security_scan_flag, security_scan_active=security_scan_active, security_max_urls_probe=security_max_urls_probe, - security_findings_output=security_findings_output, - lighthouse_summary_path=lighthouse_summary_path, - db_path=db_path, + lighthouse_summary_path=lighthouse_summary_path_for_report, + use_database=use_database, config=cfg, ) print("[Report] Done.", flush=True) print(f"Report written: {out}") - if _should_enrich_keywords_after_report(cfg) and _google_db_has_gsc(db_path): + if _should_enrich_keywords_after_report(cfg) and _google_db_has_gsc(): print("[Keywords] Post-report enrichment (GSC data found)...", flush=True) from .integrations.google.keyword_enrich import run_enrichment try: - run_enrichment(db_path, cfg) + run_enrichment(cfg) print("[Keywords] Post-report enrichment done.", flush=True) except Exception as e: print(f"Warning: post-report keyword enrichment failed: {e}", file=sys.stderr) @@ -647,16 +626,13 @@ def path(key: str, default: str) -> str: if run_plot: print("[Plot] Starting...", flush=True) from .tools.plot import run_plot as do_plot - e, n = do_plot( - crawl_csv=crawl_csv, - edges_csv=edges_csv, - nodes_csv=nodes_csv, + e = do_plot( same_domain_only=get_bool(cfg, "same_domain_only", True), max_fetch_for_edges=get_int(cfg, "max_fetch_for_edges", 500), concurrency=8, timeout=10, polite_delay=0.15, - db_path=db_path, + use_database=use_database, ) print("[Plot] Done.", flush=True) - print(f"Edges: {e}, Nodes: {n}") + print(f"Plot data: {e}") diff --git a/src/website_profiling/config.py b/src/website_profiling/config.py index 61835f46..d22b8261 100644 --- a/src/website_profiling/config.py +++ b/src/website_profiling/config.py @@ -1,6 +1,6 @@ """ Parse key=value config files (key = value or key: value, # comments, blank lines). -Also provides load_config_from_db to read pipeline settings from report.db. +Also provides load_config_from_db to read pipeline settings from PostgreSQL. """ import os @@ -62,20 +62,17 @@ def get_list(cfg: dict, key: str, sep: str = ",", default: list[str] | None = No return [s.strip() for s in str(raw).split(sep) if s.strip()] -def load_config_from_db(db_path: str) -> dict[str, str]: +def load_config_from_db() -> dict[str, str]: """ - Load pipeline config from the pipeline_config table in report.db. - Returns a flat {key: value} dict (known keys only; unknown keys are not returned here - since cli.py consumes the result the same way it consumes load_config()). - Returns an empty dict if db_path does not exist, the table is missing, or the table is empty. + Load pipeline config from the pipeline_config table. + Returns a flat {key: value} dict (known keys only). + Returns an empty dict if DATABASE_URL is unset, the table is missing, or empty. """ - if not os.path.isfile(db_path): - return {} try: - from .db import db_session, init_schema # avoid circular at module level - with db_session(db_path) as conn: - init_schema(conn) - from .db.storage import read_pipeline_config + from .db import db_session # avoid circular at module level + from .db.storage import read_pipeline_config + + with db_session() as conn: known, _unknown = read_pipeline_config(conn) return known except Exception: diff --git a/src/website_profiling/crawl/crawler.py b/src/website_profiling/crawl/crawler.py index 3ae368a0..75890c5b 100644 --- a/src/website_profiling/crawl/crawler.py +++ b/src/website_profiling/crawl/crawler.py @@ -357,9 +357,18 @@ def _queue_contains(self, item): except Exception: return False - def crawl(self, show_progress: bool = True): + def crawl( + self, + show_progress: bool = True, + stream_crawl_run_id: Optional[int] = None, + stream_batch_size: int = 500, + ): start_time = time.time() futures = [] + db_writer: Optional[_CrawlDbWriter] = None + if stream_crawl_run_id is not None: + db_writer = _CrawlDbWriter(stream_crawl_run_id, stream_batch_size) + db_writer.start() pbar = tqdm( total=None if self.max_pages == float("inf") else int(self.max_pages), desc="Pages", @@ -445,6 +454,8 @@ def crawl(self, show_progress: bool = True): if self.store_outlinks: res["outlink_targets"] = "[]" self.results.append(res) + if db_writer is not None and res.get("url"): + db_writer.enqueue(res) pbar.update(1) else: remaining.append(f) @@ -455,6 +466,10 @@ def crawl(self, show_progress: bool = True): break pbar.close() + if db_writer is not None: + db_writer.finish() + db_writer.join() + db_writer.raise_if_failed() elapsed = time.time() - start_time df = pd.DataFrame(self.results) if df.empty: @@ -518,6 +533,52 @@ def crawl(self, show_progress: bool = True): return df +class _CrawlDbWriter(threading.Thread): + """Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" + + def __init__(self, crawl_run_id: int, batch_size: int = 500) -> None: + super().__init__(daemon=True) + self.crawl_run_id = crawl_run_id + self.batch_size = max(50, batch_size) + self._queue: Queue = Queue() + self._error: Optional[BaseException] = None + + def enqueue(self, record: dict) -> None: + self._queue.put(record) + + def finish(self) -> None: + self._queue.put(None) + + def run(self) -> None: + from ..db import db_session + from ..db.storage import _crawl_rows_from_df, write_crawl_batch + + buffer: list[dict] = [] + try: + while True: + item = self._queue.get() + if item is None: + if buffer: + chunk = pd.DataFrame(buffer) + with db_session() as conn: + rows = _crawl_rows_from_df(chunk, self.crawl_run_id) + write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + break + buffer.append(item) + if len(buffer) >= self.batch_size: + chunk = pd.DataFrame(buffer) + buffer = [] + with db_session() as conn: + rows = _crawl_rows_from_df(chunk, self.crawl_run_id) + write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + except BaseException as e: + self._error = e + + def raise_if_failed(self) -> None: + if self._error is not None: + raise self._error + + def run_crawler( start_url: str, max_pages: Optional[int] = None, @@ -529,14 +590,15 @@ def run_crawler( polite_delay: float = 0.2, store_outlinks: bool = True, output_csv: Optional[str] = "crawl_results.csv", - output_db: Optional[str] = None, + output_db: bool = False, show_progress: bool = True, exclude_urls: Optional[list[str]] = None, preserve_crawl_history: bool = True, store_content_excerpt: bool = False, content_excerpt_max_chars: int = 4096, + crawl_stream_to_db: bool = False, ) -> pd.DataFrame: - """Run crawler and optionally save to CSV/JSON or SQLite. Returns DataFrame.""" + """Run crawler and optionally save to CSV/JSON or PostgreSQL. Returns DataFrame.""" import sys max_p = max_pages if max_pages is not None else 0 print(f" Crawling {start_url} (max_pages={max_p or 'unlimited'}, concurrency={concurrency})...", flush=True) @@ -554,38 +616,56 @@ def run_crawler( store_content_excerpt=store_content_excerpt, content_excerpt_max_chars=content_excerpt_max_chars, ) - df = crawler.crawl(show_progress=show_progress) - if output_db and not df.empty: + stream_run_id: Optional[int] = None + if output_db: + use_stream = crawl_stream_to_db or (max_pages is not None and max_pages > 100) + if use_stream: + from ..db import backup_db_if_exists, create_crawl_run, db_session, read_historical_data, restore_historical_data + from ..db.storage import ensure_crawl_tables_cleared + + historical = {} + if not preserve_crawl_history: + historical = read_historical_data() + backup_path = backup_db_if_exists() + if backup_path: + print(f" Backed up existing DB to {backup_path}", flush=True) + with db_session() as conn: + if not preserve_crawl_history: + ensure_crawl_tables_cleared(conn) + if historical: + restore_historical_data(conn, historical) + stream_run_id = create_crawl_run(conn, start_url) + print(f" Streaming crawl results to DB (run_id={stream_run_id})...", flush=True) + + df = crawler.crawl( + show_progress=show_progress, + stream_crawl_run_id=stream_run_id, + ) + if output_db and not df.empty and stream_run_id is None: import sys print(" Writing crawl results to DB...", flush=True) - from ..db import backup_db_if_exists, create_crawl_run, db_session, ensure_db_recreated, init_schema, read_historical_data, restore_historical_data, write_crawl + from ..db import backup_db_if_exists, create_crawl_run, db_session, read_historical_data, restore_historical_data, write_crawl + from ..db.storage import ensure_crawl_tables_cleared historical = {} backup_path = None if not preserve_crawl_history: - historical = read_historical_data(output_db) + historical = read_historical_data() n_reports = len(historical.get("report_payload", [])) if n_reports: print(f" Preserving {n_reports} historical report(s) from existing DB...", flush=True) - backup_path = backup_db_if_exists(output_db) + backup_path = backup_db_if_exists() if backup_path: print(f" Backed up existing DB to {backup_path}", flush=True) - ensure_db_recreated(output_db) - with db_session(output_db) as conn: - init_schema(conn) + with db_session() as conn: + if not preserve_crawl_history: + ensure_crawl_tables_cleared(conn) if historical: restore_historical_data(conn, historical) - if backup_path: - from pathlib import Path as _Path - for p in (backup_path, backup_path + "-journal"): - try: - _Path(p).unlink(missing_ok=True) - except OSError: - pass - print(f" Removed temporary backup {backup_path}", flush=True) - # Always record a crawl_run so edges/nodes can use crawl_run_id (see write_edges / read_edges). run_id = create_crawl_run(conn, start_url) write_crawl(conn, df, crawl_run_id=run_id) print(" Crawl DB write complete.", flush=True) + elif output_db and stream_run_id is not None: + print(" Crawl streamed to DB during fetch.", flush=True) elif output_csv and not df.empty: if output_csv.lower().endswith(".json"): df.to_json(output_csv, orient="records", indent=2, date_format="iso", default_handler=str) diff --git a/src/website_profiling/db/__init__.py b/src/website_profiling/db/__init__.py index c9dad3d4..c8f60a86 100644 --- a/src/website_profiling/db/__init__.py +++ b/src/website_profiling/db/__init__.py @@ -1,2 +1,5 @@ -"""SQLite storage layer (re-export from storage).""" +"""PostgreSQL storage layer (re-export from storage). + +Public helpers include :func:`db_session` and :func:`close_db_pool`. +""" from .storage import * # noqa: F403 diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py index 29c1779e..1f171118 100644 --- a/src/website_profiling/db/storage.py +++ b/src/website_profiling/db/storage.py @@ -1,65 +1,124 @@ """ -SQLite data layer for WebsiteProfiling: single DB for crawl, edges, nodes, lighthouse, report payload. +PostgreSQL data layer for WebsiteProfiling: crawl, edges, nodes, lighthouse, report payload. -All DB access should go through :func:`db_session` so one connection at a time per database path -(process-wide lock). That serializes writers like a single-slot queue and avoids lock/readonly issues -on slow or synced volumes. +All DB access should go through :func:`db_session`. Schema is managed by Alembic (``alembic upgrade head``). +Requires ``DATABASE_URL`` in the environment. """ +from __future__ import annotations + +import atexit import json import math import os -import shutil -import sqlite3 -import threading +import subprocess import time from contextlib import contextmanager +from datetime import datetime, timezone from pathlib import Path from typing import Any, Iterator, Optional import pandas as pd +import psycopg +from psycopg import Connection +from psycopg.rows import dict_row +from psycopg.types.json import Json +from psycopg_pool import ConnectionPool + +from urllib.parse import urlparse +_pool: ConnectionPool | None = None +_shutdown_registered = False -_db_path_locks: dict[str, threading.Lock] = {} -_db_path_locks_guard = threading.Lock() +_BOOL_COLS = ("viewport_present", "noindex", "has_schema") +_CRAWL_BATCH_SIZE = 1000 -def _normalize_db_path(db_path: str) -> str: - return os.path.normcase(os.path.abspath(db_path)) +def _env_int(name: str, default: int) -> int: + raw = (os.environ.get(name) or "").strip() + if not raw: + return default + try: + return max(1, int(raw)) + except ValueError: + return default + + +def get_database_url() -> str: + url = (os.environ.get("DATABASE_URL") or "").strip() + if not url: + raise RuntimeError( + "DATABASE_URL is required. Example: postgres://user:pass@localhost:5432/website_profiling" + ) + return url + +def get_data_dir() -> str: + return (os.environ.get("DATA_DIR") or os.getcwd()).strip() or os.getcwd() -def _lock_for_db_path(db_path: str) -> threading.Lock: - key = _normalize_db_path(db_path) - with _db_path_locks_guard: - if key not in _db_path_locks: - _db_path_locks[key] = threading.Lock() - return _db_path_locks[key] +def close_db_pool() -> None: + """Close the process-wide connection pool (idempotent, safe to call multiple times).""" + global _pool + if _pool is not None: + _pool.close() + _pool = None -def _open_sqlite(db_path: str) -> sqlite3.Connection: - """Open SQLite without taking the process-wide DB lock (internal; use :func:`db_session`).""" - Path(db_path).parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(db_path, timeout=30.0) - conn.row_factory = sqlite3.Row - return conn + +def _register_pool_shutdown() -> None: + global _shutdown_registered + if not _shutdown_registered: + atexit.register(close_db_pool) + _shutdown_registered = True + + +def _get_pool() -> ConnectionPool: + global _pool + if _pool is None: + _pool = ConnectionPool( + conninfo=get_database_url(), + min_size=_env_int("DB_POOL_MIN", 2), + max_size=_env_int("DB_POOL_MAX", 20), + open=True, + kwargs={"row_factory": dict_row}, + ) + _register_pool_shutdown() + return _pool @contextmanager -def db_session(db_path: str) -> Iterator[sqlite3.Connection]: - """Serialize access to ``db_path``: one connection at a time, then close (mutex per absolute path).""" - lock = _lock_for_db_path(db_path) - lock.acquire() - try: - conn = _open_sqlite(db_path) +def db_session() -> Iterator[Connection]: + """Yield a PostgreSQL connection from the process pool.""" + with _get_pool().connection() as conn: + yield conn + + +def init_schema(conn: Connection | None = None) -> None: + """No-op at runtime; schema is applied via Alembic migrations.""" + + +def _now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + +def _json_val(obj: Any) -> Json: + return Json(_sanitize_for_json(obj)) + + +def _parse_json_field(val: Any) -> Any: + if val is None: + return None + if isinstance(val, (dict, list)): + return val + if isinstance(val, str): try: - yield conn - finally: - conn.close() - finally: - lock.release() + return json.loads(val) + except json.JSONDecodeError: + return val + return val def _sanitize_for_json(obj: Any) -> Any: - """Recursively replace NaN/Inf and numpy types so JSON is valid (no literal NaN).""" + """Recursively replace NaN/Inf and numpy types so JSON is valid.""" if obj is None: return None if isinstance(obj, (bool, str)): @@ -74,51 +133,55 @@ def _sanitize_for_json(obj: Any) -> Any: return {k: _sanitize_for_json(v) for k, v in obj.items()} if isinstance(obj, list): return [_sanitize_for_json(v) for v in obj] - if hasattr(obj, "item"): # numpy scalar + if hasattr(obj, "item"): try: return _sanitize_for_json(obj.item()) except (ValueError, AttributeError): return None - if hasattr(obj, "isoformat"): # datetime + if hasattr(obj, "isoformat"): return obj.isoformat() return obj -def backup_db_if_exists(db_path: str, skip_in_ci: bool = True) -> Optional[str]: - """Copy db_path to a timestamped backup file and return the backup path, or None. +def _executemany(conn: Connection, sql: str, params: list, *, page_size: int = 500) -> None: + if not params: + return + with conn.cursor() as cur: + for i in range(0, len(params), page_size): + cur.executemany(sql, params[i : i + page_size]) + - Returns None without creating a backup when: - - skip_in_ci is True and the process is running in GitHub Actions or a generic CI environment. - - The db_path file does not exist. - """ +def backup_db_if_exists(skip_in_ci: bool = True) -> Optional[str]: + """Run pg_dump to DATA_DIR/backups/ and return the dump path, or None.""" if skip_in_ci and ( os.environ.get("GITHUB_ACTIONS") == "true" or os.environ.get("CI") == "true" ): return None - p = Path(db_path) - if not p.exists() or not p.is_file(): - return None + data_dir = Path(get_data_dir()) + backup_dir = data_dir / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) suffix = time.strftime("%Y%m%d-%H%M%S") - backup = p.parent / f"{p.name}.backup-{suffix}" - shutil.copy2(str(p), str(backup)) - journal = Path(str(p) + "-journal") - if journal.exists(): - try: - shutil.copy2(str(journal), str(backup) + "-journal") - except OSError: - pass - return str(backup) - + out_path = backup_dir / f"website_profiling-{suffix}.dump" + try: + subprocess.run( + [ + "pg_dump", + "-Fc", + "-f", + str(out_path), + get_database_url(), + ], + check=True, + capture_output=True, + timeout=300, + ) + return str(out_path) + except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return None -def read_historical_data(db_path: str) -> dict[str, list]: - """Read rows from historical tables in an existing DB before it is overwritten. - Returns a dict mapping table name -> list of row dicts. - Tables captured: report_payload, lighthouse_*, google_data, keyword_data, - keyword_history, keyword_suggest_cache, crawl_runs. - crawl_results / edges / nodes are intentionally excluded (they belong to the new crawl). - Returns empty lists for all tables when the DB file does not exist. - """ +def read_historical_data() -> dict[str, list]: + """Read historical tables before a crawl overwrite (excludes crawl_results/edges/nodes).""" tables = [ "report_payload", "lighthouse_summary", @@ -133,15 +196,13 @@ def read_historical_data(db_path: str) -> dict[str, list]: "crawl_runs", ] result: dict[str, list] = {t: [] for t in tables} - p = Path(db_path) - if not p.exists() or not p.is_file(): - return result try: - with db_session(db_path) as conn: + with db_session() as conn: for table in tables: try: - cur = conn.execute(f"SELECT * FROM {table}") - result[table] = [dict(row) for row in cur.fetchall()] + with conn.cursor() as cur: + cur.execute(f"SELECT * FROM {table}") + result[table] = [dict(row) for row in cur.fetchall()] except Exception: pass except Exception: @@ -149,304 +210,138 @@ def read_historical_data(db_path: str) -> dict[str, list]: return result -def restore_historical_data(conn: sqlite3.Connection, data: dict[str, list]) -> None: - """Insert previously-read historical rows back into a freshly-created DB. - - Uses INSERT OR IGNORE with explicit ids so rows are idempotent and the - original row ordering (and thus UI report list ordering) is preserved. - Silently skips any row that fails to insert. - """ - for row in data.get("report_payload", []): - try: - conn.execute( - "INSERT OR IGNORE INTO report_payload (id, generated_at, data) VALUES (?, ?, ?)", - (row.get("id"), row.get("generated_at"), row.get("data")), - ) - except Exception: - pass - - for row in data.get("lighthouse_summary", []): - try: - conn.execute( - "INSERT OR IGNORE INTO lighthouse_summary (id, created_at, data) VALUES (?, ?, ?)", - (row.get("id"), row.get("created_at"), row.get("data")), - ) - except Exception: - pass +def restore_historical_data(conn: Connection, data: dict[str, list]) -> None: + """Insert previously-read historical rows (preserves explicit ids where provided).""" - for row in data.get("lighthouse_runs", []): - try: - conn.execute( - "INSERT OR IGNORE INTO lighthouse_runs (id, created_at, url, strategy, run_index, data) VALUES (?, ?, ?, ?, ?, ?)", - (row.get("id"), row.get("created_at"), row.get("url"), row.get("strategy"), row.get("run_index"), row.get("data")), - ) - except Exception: - pass - - for row in data.get("lighthouse_page_summaries", []): - try: - conn.execute( - "INSERT OR IGNORE INTO lighthouse_page_summaries (url, created_at, data) VALUES (?, ?, ?)", - (row.get("url"), row.get("created_at"), row.get("data")), - ) - except Exception: - pass - - for row in data.get("lh_audits", []): - try: - conn.execute( - """INSERT OR IGNORE INTO lh_audits (id, run_id, audit_id, category_id, score, score_display_mode, - title, description, display_value, numeric_value, help_text, details_type, details_headings, details_meta) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - row.get("id"), - row.get("run_id"), - row.get("audit_id"), - row.get("category_id"), - row.get("score"), - row.get("score_display_mode"), - row.get("title"), - row.get("description"), - row.get("display_value"), - row.get("numeric_value"), - row.get("help_text"), - row.get("details_type"), - row.get("details_headings"), - row.get("details_meta"), - ), - ) - except Exception: - pass - - for row in data.get("lh_audit_items", []): - try: - conn.execute( - "INSERT OR IGNORE INTO lh_audit_items (id, audit_row_id, item_index, row_data) VALUES (?, ?, ?, ?)", - (row.get("id"), row.get("audit_row_id"), row.get("item_index"), row.get("row_data")), - ) - except Exception: - pass - - for row in data.get("google_data", []): - try: - conn.execute( - "INSERT OR IGNORE INTO google_data (id, fetched_at, data) VALUES (?, ?, ?)", - (row.get("id"), row.get("fetched_at"), row.get("data")), - ) - except Exception: - pass - - for row in data.get("keyword_data", []): - try: - conn.execute( - "INSERT OR IGNORE INTO keyword_data (id, fetched_at, data) VALUES (?, ?, ?)", - (row.get("id"), row.get("fetched_at"), row.get("data")), - ) - except Exception: - pass - - for row in data.get("keyword_history", []): + def _bulk( + sql: str, + rows: list[dict], + keys: list[str], + transform: Any | None = None, + ) -> None: + if not rows: + return + params: list[tuple] = [] + for row in rows: + vals = [] + for k in keys: + v = row.get(k) + if transform and k in transform: + v = transform[k](v) + vals.append(v) + params.append(tuple(vals)) try: - conn.execute( - "INSERT OR IGNORE INTO keyword_history " - "(id, keyword, fetched_at, position, clicks, impressions, ctr) " - "VALUES (?, ?, ?, ?, ?, ?, ?)", - ( - row.get("id"), - row.get("keyword"), - row.get("fetched_at"), - row.get("position"), - row.get("clicks"), - row.get("impressions"), - row.get("ctr"), - ), - ) + _executemany(conn, sql, params, page_size=500) except Exception: - pass + for p in params: + try: + conn.execute(sql, p) + except Exception: + pass - for row in data.get("keyword_suggest_cache", []): - try: - conn.execute( - "INSERT OR IGNORE INTO keyword_suggest_cache (cache_key, fetched_at, data) VALUES (?, ?, ?)", - (row.get("cache_key"), row.get("fetched_at"), row.get("data")), - ) - except Exception: - pass + json_t = lambda v: _json_val(_parse_json_field(v)) - for row in data.get("crawl_runs", []): - try: - conn.execute( - "INSERT OR IGNORE INTO crawl_runs (id, created_at, start_url) VALUES (?, ?, ?)", - (row.get("id"), row.get("created_at"), row.get("start_url")), - ) - except Exception: - pass + _bulk( + """INSERT INTO report_payload (id, generated_at, site_name, canonical_domain, data) + VALUES (%s, %s, %s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("report_payload", []), + ["id", "generated_at", "site_name", "canonical_domain", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO lighthouse_summary (id, created_at, data) + VALUES (%s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("lighthouse_summary", []), + ["id", "created_at", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO lighthouse_runs (id, created_at, url, strategy, run_index, data) + VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("lighthouse_runs", []), + ["id", "created_at", "url", "strategy", "run_index", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO lighthouse_page_summaries (url, created_at, data) + VALUES (%s, %s, %s) ON CONFLICT (url) DO NOTHING""", + data.get("lighthouse_page_summaries", []), + ["url", "created_at", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO lh_audits (id, run_id, audit_id, category_id, score, score_display_mode, + title, description, display_value, numeric_value, help_text, details_type, + details_headings, details_meta) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (id) DO NOTHING""", + data.get("lh_audits", []), + [ + "id", "run_id", "audit_id", "category_id", "score", "score_display_mode", + "title", "description", "display_value", "numeric_value", "help_text", + "details_type", "details_headings", "details_meta", + ], + {"details_headings": json_t, "details_meta": json_t}, + ) + _bulk( + """INSERT INTO lh_audit_items (id, audit_row_id, item_index, row_data) + VALUES (%s, %s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("lh_audit_items", []), + ["id", "audit_row_id", "item_index", "row_data"], + {"row_data": json_t}, + ) + _bulk( + """INSERT INTO google_data (id, fetched_at, data) + VALUES (%s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("google_data", []), + ["id", "fetched_at", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO keyword_data (id, fetched_at, data) + VALUES (%s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("keyword_data", []), + ["id", "fetched_at", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO keyword_history + (id, keyword, fetched_at, position, clicks, impressions, ctr) + VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("keyword_history", []), + ["id", "keyword", "fetched_at", "position", "clicks", "impressions", "ctr"], + ) + _bulk( + """INSERT INTO keyword_suggest_cache (cache_key, fetched_at, data) + VALUES (%s, %s, %s) ON CONFLICT (cache_key) DO NOTHING""", + data.get("keyword_suggest_cache", []), + ["cache_key", "fetched_at", "data"], + {"data": json_t}, + ) + _bulk( + """INSERT INTO crawl_runs (id, created_at, start_url) + VALUES (%s, %s, %s) ON CONFLICT (id) DO NOTHING""", + data.get("crawl_runs", []), + ["id", "created_at", "start_url"], + ) conn.commit() -def ensure_db_recreated(db_path: str) -> None: - """Delete existing DB file (and journal) so the next :func:`db_session` creates a fresh DB.""" - for p in (db_path, db_path + "-journal"): - if Path(p).exists(): - try: - Path(p).unlink() - except OSError: - pass - - -def get_connection(db_path: str) -> sqlite3.Connection: - """Open SQLite (no lock). Prefer :func:`db_session` so access is serialized per DB path.""" - return _open_sqlite(db_path) - - -def init_schema(conn: sqlite3.Connection) -> None: - """Create tables if they do not exist. crawl_results is created by write_crawl from DataFrame.""" - conn.executescript(""" - CREATE TABLE IF NOT EXISTS crawl_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL, - start_url TEXT - ); - - CREATE TABLE IF NOT EXISTS edges ( - crawl_run_id INTEGER NOT NULL, - from_url TEXT NOT NULL, - to_url TEXT NOT NULL, - PRIMARY KEY (crawl_run_id, from_url, to_url) - ); - - CREATE TABLE IF NOT EXISTS nodes ( - crawl_run_id INTEGER NOT NULL, - url TEXT NOT NULL, - count INTEGER NOT NULL, - PRIMARY KEY (crawl_run_id, url) - ); - - CREATE TABLE IF NOT EXISTS lighthouse_summary ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS lighthouse_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - created_at TEXT NOT NULL, - url TEXT NOT NULL, - strategy TEXT NOT NULL, - run_index INTEGER NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS lighthouse_page_summaries ( - url TEXT PRIMARY KEY, - created_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS report_payload ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - generated_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS google_data ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS keyword_data ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS keyword_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - keyword TEXT NOT NULL, - fetched_at TEXT NOT NULL, - position REAL, - clicks INTEGER, - impressions INTEGER, - ctr REAL - ); - CREATE INDEX IF NOT EXISTS idx_kw_history_keyword ON keyword_history(keyword); - - CREATE TABLE IF NOT EXISTS keyword_suggest_cache ( - cache_key TEXT PRIMARY KEY, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS lh_audits ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - audit_id TEXT NOT NULL, - category_id TEXT, - score REAL, - score_display_mode TEXT, - title TEXT, - description TEXT, - display_value TEXT, - numeric_value REAL, - help_text TEXT, - details_type TEXT, - details_headings TEXT, - details_meta TEXT, - FOREIGN KEY (run_id) REFERENCES lighthouse_runs(id) - ); - CREATE INDEX IF NOT EXISTS idx_lh_audits_run_id ON lh_audits(run_id); - CREATE INDEX IF NOT EXISTS idx_lh_audits_run_audit ON lh_audits(run_id, audit_id); - CREATE INDEX IF NOT EXISTS idx_lh_audits_audit_id ON lh_audits(audit_id); - - CREATE TABLE IF NOT EXISTS lh_audit_items ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - audit_row_id INTEGER NOT NULL, - item_index INTEGER NOT NULL, - row_data TEXT NOT NULL, - FOREIGN KEY (audit_row_id) REFERENCES lh_audits(id) - ); - CREATE INDEX IF NOT EXISTS idx_lh_audit_items_audit_row ON lh_audit_items(audit_row_id); - - CREATE TABLE IF NOT EXISTS pipeline_config ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - is_unknown INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS llm_config ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - is_secret INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS llm_cache ( - cache_key TEXT PRIMARY KEY, - response_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - """) +def ensure_crawl_tables_cleared(conn: Connection) -> None: + """Clear crawl-scoped tables before a non-append crawl (preserves reports, Google, etc.).""" + conn.execute("TRUNCATE crawl_results, edges, nodes RESTART IDENTITY CASCADE") conn.commit() -def read_pipeline_config(conn: sqlite3.Connection) -> tuple[dict[str, str], list[dict[str, str]]]: - """ - Return (known_entries, unknown_entries) from the pipeline_config table. - known_entries: {key: value} for is_unknown=0 rows. - unknown_entries: [{key, value}] for is_unknown=1 rows. - Returns ({}, []) if the table is empty or an error occurs. - """ +def read_pipeline_config(conn: Connection) -> tuple[dict[str, str], list[dict[str, str]]]: try: cur = conn.execute("SELECT key, value, is_unknown FROM pipeline_config ORDER BY key") rows = cur.fetchall() known: dict[str, str] = {} unknown: list[dict[str, str]] = [] for row in rows: - k, v, is_unk = str(row["key"]), str(row["value"]), int(row["is_unknown"] or 0) - if is_unk: + k, v = str(row["key"]), str(row["value"]) + if row["is_unknown"]: unknown.append({"key": k, "value": v}) else: known[k] = v @@ -456,39 +351,30 @@ def read_pipeline_config(conn: sqlite3.Connection) -> tuple[dict[str, str], list def write_pipeline_config( - conn: sqlite3.Connection, + conn: Connection, entries: dict[str, str], unknown_keys: list[dict[str, str]] | None = None, ) -> None: - """ - Atomically replace all pipeline_config rows with the provided entries. - entries: {key: value} — known schema keys (is_unknown=0). - unknown_keys: [{key, value}] — preserved verbatim (is_unknown=1). - """ - now = time.strftime("%Y-%m-%d %H:%M:%S") + now = _now_iso() if unknown_keys is None: unknown_keys = [] - conn.execute("BEGIN") - try: + with conn.transaction(): conn.execute("DELETE FROM pipeline_config") for k, v in entries.items(): conn.execute( - "INSERT INTO pipeline_config (key, value, is_unknown, updated_at) VALUES (?, ?, 0, ?)", + "INSERT INTO pipeline_config (key, value, is_unknown, updated_at) VALUES (%s, %s, false, %s)", (str(k), str(v), now), ) for item in unknown_keys: conn.execute( - "INSERT OR REPLACE INTO pipeline_config (key, value, is_unknown, updated_at) VALUES (?, ?, 1, ?)", + """INSERT INTO pipeline_config (key, value, is_unknown, updated_at) + VALUES (%s, %s, true, %s) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, is_unknown = true, updated_at = EXCLUDED.updated_at""", (str(item["key"]), str(item.get("value", "")), now), ) - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise -def read_llm_config(conn: sqlite3.Connection) -> dict[str, str]: - """Return {key: value} from llm_config table (includes secrets).""" +def read_llm_config(conn: Connection) -> dict[str, str]: try: cur = conn.execute("SELECT key, value FROM llm_config ORDER BY key") return {str(row["key"]): str(row["value"]) for row in cur.fetchall()} @@ -496,208 +382,263 @@ def read_llm_config(conn: sqlite3.Connection) -> dict[str, str]: return {} -def write_llm_config(conn: sqlite3.Connection, entries: dict[str, str], secret_keys: set[str] | None = None) -> None: - """Replace all llm_config rows. secret_keys marks keys stored with is_secret=1.""" - now = time.strftime("%Y-%m-%d %H:%M:%S") +def write_llm_config(conn: Connection, entries: dict[str, str], secret_keys: set[str] | None = None) -> None: + now = _now_iso() secret_keys = secret_keys or set() - conn.execute("BEGIN") - try: + with conn.transaction(): conn.execute("DELETE FROM llm_config") for k, v in entries.items(): - is_secret = 1 if k in secret_keys else 0 conn.execute( - "INSERT INTO llm_config (key, value, is_secret, updated_at) VALUES (?, ?, ?, ?)", - (str(k), str(v), is_secret, now), + "INSERT INTO llm_config (key, value, is_secret, updated_at) VALUES (%s, %s, %s, %s)", + (str(k), str(v), k in secret_keys, now), ) - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise -def read_llm_cache(conn: sqlite3.Connection, cache_key: str) -> Optional[str]: +def read_llm_cache(conn: Connection, cache_key: str) -> Optional[str]: try: - cur = conn.execute("SELECT response_json FROM llm_cache WHERE cache_key = ?", (cache_key,)) + cur = conn.execute("SELECT response_json FROM llm_cache WHERE cache_key = %s", (cache_key,)) row = cur.fetchone() - return str(row["response_json"]) if row else None + if not row: + return None + val = row["response_json"] + return json.dumps(val) if isinstance(val, (dict, list)) else str(val) except Exception: return None -def write_llm_cache(conn: sqlite3.Connection, cache_key: str, response_json: str) -> None: - now = time.strftime("%Y-%m-%d %H:%M:%S") +def write_llm_cache(conn: Connection, cache_key: str, response_json: str) -> None: + now = _now_iso() + try: + payload = json.loads(response_json) + except json.JSONDecodeError: + payload = response_json conn.execute( - "INSERT OR REPLACE INTO llm_cache (cache_key, response_json, created_at) VALUES (?, ?, ?)", - (cache_key, response_json, now), + """INSERT INTO llm_cache (cache_key, response_json, created_at) + VALUES (%s, %s, %s) + ON CONFLICT (cache_key) DO UPDATE SET response_json = EXCLUDED.response_json, created_at = EXCLUDED.created_at""", + (cache_key, _json_val(payload), now), ) conn.commit() -def _crawl_results_has_run_id(conn: sqlite3.Connection) -> bool: - """True if crawl_results exists and includes crawl_run_id (append-by-run crawls).""" - try: - cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crawl_results'") - if cur.fetchone() is None: - return False - cur = conn.execute("PRAGMA table_info(crawl_results)") - return any(row[1] == "crawl_run_id" for row in cur.fetchall()) - except Exception: - return False - - -def create_crawl_run(conn: sqlite3.Connection, start_url: Optional[str] = None) -> int: - """Insert a new crawl run and return its id.""" - conn.execute( - "INSERT INTO crawl_runs (created_at, start_url) VALUES (?, ?)", - (time.strftime("%Y-%m-%d %H:%M:%S"), start_url), +def create_crawl_run(conn: Connection, start_url: Optional[str] = None) -> int: + cur = conn.execute( + "INSERT INTO crawl_runs (created_at, start_url) VALUES (%s, %s) RETURNING id", + (_now_iso(), start_url), ) + row = cur.fetchone() conn.commit() - cur = conn.execute("SELECT last_insert_rowid()") - return int(cur.fetchone()[0]) + return int(row["id"]) -def get_latest_crawl_run_id(conn: sqlite3.Connection) -> Optional[int]: - """Return the latest crawl run id, or None if no runs.""" +def get_latest_crawl_run_id(conn: Connection) -> Optional[int]: try: cur = conn.execute("SELECT id FROM crawl_runs ORDER BY id DESC LIMIT 1") row = cur.fetchone() - return int(row[0]) if row else None + return int(row["id"]) if row else None except Exception: return None -def get_crawl_run_info(conn: sqlite3.Connection, run_id: int) -> Optional[dict[str, Any]]: - """Return dict with created_at, start_url for the given run_id, or None.""" +def get_crawl_run_info(conn: Connection, run_id: int) -> Optional[dict[str, Any]]: try: - cur = conn.execute("SELECT created_at, start_url FROM crawl_runs WHERE id = ?", (run_id,)) + cur = conn.execute("SELECT created_at, start_url FROM crawl_runs WHERE id = %s", (run_id,)) row = cur.fetchone() if row is None: return None - return {"created_at": row[0], "start_url": row[1]} + return {"created_at": row["created_at"], "start_url": row["start_url"]} except Exception: return None -def _ensure_crawl_table_from_df(conn: sqlite3.Connection, df: pd.DataFrame) -> None: - """Recreate crawl_results table to match DataFrame columns (for varying crawler output).""" - conn.execute("DROP TABLE IF EXISTS crawl_results") - conn.commit() - df.to_sql("crawl_results", conn, index=False, if_exists="replace") - conn.commit() +def _df_row_to_crawl_json(row: pd.Series) -> dict[str, Any]: + out: dict[str, Any] = {} + for col in row.index: + if col in ("url", "crawl_run_id"): + continue + val = row[col] + if pd.isna(val): + out[col] = None + elif hasattr(val, "item"): + out[col] = _sanitize_for_json(val.item()) + else: + out[col] = _sanitize_for_json(val) + return out + + +def _extract_hostname(url: str) -> str: + try: + host = urlparse(str(url or "")).hostname + return host.lower() if host else "" + except Exception: + return "" + + +def _canonical_domain_from_report(conn: Connection, report_data: dict[str, Any]) -> str: + run_id = report_data.get("crawl_run_id") + start_url = "" + if run_id is not None: + info = get_crawl_run_info(conn, int(run_id)) + if info: + start_url = str(info.get("start_url") or "") + top_pages = report_data.get("top_pages") or [] + fallback_url = "" + if top_pages and isinstance(top_pages[0], dict): + fallback_url = str(top_pages[0].get("url") or "") + if not fallback_url: + links = report_data.get("links") or [] + if links and isinstance(links[0], dict): + fallback_url = str(links[0].get("url") or "") + return _extract_hostname(start_url) or _extract_hostname(fallback_url) + + +_CRAWL_INSERT_SQL = """INSERT INTO crawl_results (crawl_run_id, url, status, title, data) +VALUES (%s, %s, %s, %s, %s) +ON CONFLICT (crawl_run_id, url) DO UPDATE SET + status = EXCLUDED.status, + title = EXCLUDED.title, + data = EXCLUDED.data""" + + +def _crawl_rows_from_df(df: pd.DataFrame, crawl_run_id: int) -> list[tuple]: + rows: list[tuple] = [] + if df.empty or "url" not in df.columns: + return rows + data_cols = [c for c in df.columns if c not in ("url", "crawl_run_id")] + for rec in df.to_dict(orient="records"): + url = str(rec.get("url", "")).rstrip("/") + if not url: + continue + payload = {c: _sanitize_for_json(rec[c]) if not pd.isna(rec.get(c)) else None for c in data_cols} + status = str(rec.get("status") or "") if "status" in rec else None + title = str(rec.get("title") or "") if "title" in rec else None + rows.append((crawl_run_id, url, status, title, _json_val(payload))) + return rows + + +def write_crawl_batch( + conn: Connection, + rows: list[tuple], + crawl_run_id: int, + *, + commit: bool = True, +) -> None: + """Insert a batch of crawl rows (each tuple: run_id, url, status, title, data Json).""" + if not rows: + return + _executemany(conn, _CRAWL_INSERT_SQL, rows, page_size=_CRAWL_BATCH_SIZE) + if commit: + conn.commit() -def write_crawl(conn: sqlite3.Connection, df: pd.DataFrame, crawl_run_id: Optional[int] = None) -> None: - """Write crawl results. If crawl_run_id is set, append rows for that run; else replace table (legacy).""" +def write_crawl(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int] = None) -> None: if df.empty: if crawl_run_id is None: - init_schema(conn) try: conn.execute("DELETE FROM crawl_results") + conn.commit() except Exception: pass - conn.commit() return + df = df.copy() if "url" in df.columns: df["url"] = df["url"].astype(str).str.rstrip("/") - for col in df.columns: - if df[col].dtype == bool: - df[col] = df[col].astype(int) - if crawl_run_id is not None: - df["crawl_run_id"] = crawl_run_id - try: - cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crawl_results'") - table_exists = cur.fetchone() is not None - if not table_exists or not _crawl_results_has_run_id(conn): - if table_exists: - conn.execute("DROP TABLE crawl_results") - conn.commit() - cols = ["crawl_run_id"] + [c for c in df.columns if c != "crawl_run_id"] - df[cols].to_sql("crawl_results", conn, index=False, if_exists="replace") - else: - df.to_sql("crawl_results", conn, index=False, if_exists="append", method="multi") - finally: - df.drop(columns=["crawl_run_id"], inplace=True, errors="ignore") - conn.commit() - return - df.to_sql("crawl_results", conn, index=False, if_exists="replace") - conn.commit() + with conn.transaction(): + if crawl_run_id is not None: + conn.execute("DELETE FROM crawl_results WHERE crawl_run_id = %s", (crawl_run_id,)) + target_run_id = crawl_run_id + else: + conn.execute("DELETE FROM crawl_results") + rid = get_latest_crawl_run_id(conn) + if rid is None: + cur = conn.execute( + "INSERT INTO crawl_runs (created_at, start_url) VALUES (%s, %s) RETURNING id", + (_now_iso(), None), + ) + rid = int(cur.fetchone()["id"]) + target_run_id = rid + + rows = _crawl_rows_from_df(df, target_run_id) + if rows: + _executemany(conn, _CRAWL_INSERT_SQL, rows, page_size=_CRAWL_BATCH_SIZE) -def read_crawl(conn: sqlite3.Connection, run_id: Optional[int] = None) -> pd.DataFrame: - """Read crawl_results into a DataFrame. If run_id is None, use latest crawl run.""" +def read_crawl(conn: Connection, run_id: Optional[int] = None) -> pd.DataFrame: try: if run_id is None: run_id = get_latest_crawl_run_id(conn) - if run_id is None: - df = pd.read_sql("SELECT * FROM crawl_results", conn) - elif _crawl_results_has_run_id(conn): - df = pd.read_sql("SELECT * FROM crawl_results WHERE crawl_run_id = ?", conn, params=(run_id,)) - else: - df = pd.read_sql("SELECT * FROM crawl_results", conn) + if run_id is None: + cur = conn.execute("SELECT url, data FROM crawl_results") else: - if _crawl_results_has_run_id(conn): - df = pd.read_sql("SELECT * FROM crawl_results WHERE crawl_run_id = ?", conn, params=(run_id,)) - else: - df = pd.read_sql("SELECT * FROM crawl_results", conn) + cur = conn.execute( + "SELECT url, data FROM crawl_results WHERE crawl_run_id = %s", + (run_id,), + ) + rows = cur.fetchall() + if not rows: + return pd.DataFrame() + records = [] + for row in rows: + rec = {"url": row["url"]} + data = _parse_json_field(row["data"]) or {} + if isinstance(data, dict): + rec.update(data) + records.append(rec) + df = pd.DataFrame(records) + for c in _BOOL_COLS: + if c in df.columns: + df[c] = df[c].astype(bool) + return df except Exception: return pd.DataFrame() - if df.empty: - return df - if "crawl_run_id" in df.columns: - df = df.drop(columns=["crawl_run_id"], errors="ignore") - bool_cols = [ - "viewport_present", "noindex", "has_schema", - ] - for c in bool_cols: - if c in df.columns: - df[c] = df[c].astype(bool) - return df -def write_edges(conn: sqlite3.Connection, edges: list[tuple[str, str]], crawl_run_id: Optional[int] = None) -> None: - """Write edges. If crawl_run_id is set, insert for that run; else replace edges for the latest crawl run.""" +def write_edges(conn: Connection, edges: list[tuple[str, str]], crawl_run_id: Optional[int] = None) -> None: if crawl_run_id is None: conn.execute("DELETE FROM edges") if edges: rid = get_latest_crawl_run_id(conn) if rid is not None: - conn.executemany( - "INSERT INTO edges (crawl_run_id, from_url, to_url) VALUES (?, ?, ?)", + _executemany( + conn, + "INSERT INTO edges (crawl_run_id, from_url, to_url) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING", [(rid, a.rstrip("/"), b.rstrip("/")) for a, b in edges], ) conn.commit() return - conn.execute("DELETE FROM edges WHERE crawl_run_id = ?", (crawl_run_id,)) + conn.execute("DELETE FROM edges WHERE crawl_run_id = %s", (crawl_run_id,)) if edges: - conn.executemany( - "INSERT INTO edges (crawl_run_id, from_url, to_url) VALUES (?, ?, ?)", + _executemany( + conn, + "INSERT INTO edges (crawl_run_id, from_url, to_url) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING", [(crawl_run_id, a.rstrip("/"), b.rstrip("/")) for a, b in edges], ) conn.commit() -def read_edges(conn: sqlite3.Connection, run_id: Optional[int] = None) -> list[tuple[str, str]]: - """Read edges. If run_id is None, use latest crawl run.""" +def read_edges(conn: Connection, run_id: Optional[int] = None) -> list[tuple[str, str]]: try: if run_id is None: run_id = get_latest_crawl_run_id(conn) if run_id is None: return [] - cur = conn.execute("SELECT from_url, to_url FROM edges WHERE crawl_run_id = ?", (run_id,)) - return [tuple(row) for row in cur.fetchall()] + cur = conn.execute( + "SELECT from_url, to_url FROM edges WHERE crawl_run_id = %s", + (run_id,), + ) + return [(row["from_url"], row["to_url"]) for row in cur.fetchall()] except Exception: return [] -def write_nodes(conn: sqlite3.Connection, df: pd.DataFrame, crawl_run_id: Optional[int] = None) -> None: - """Write nodes. If crawl_run_id is set, insert for that run; else replace (legacy).""" +def write_nodes(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int] = None) -> None: if df.empty: if crawl_run_id is None: conn.execute("DELETE FROM nodes") else: - conn.execute("DELETE FROM nodes WHERE crawl_run_id = ?", (crawl_run_id,)) + conn.execute("DELETE FROM nodes WHERE crawl_run_id = %s", (crawl_run_id,)) conn.commit() return ndf = df.copy() @@ -711,167 +652,198 @@ def write_nodes(conn: sqlite3.Connection, df: pd.DataFrame, crawl_run_id: Option conn.execute("DELETE FROM nodes") conn.commit() return - conn.execute("DELETE FROM nodes WHERE crawl_run_id = ?", (rid,)) - ndf["crawl_run_id"] = rid - ndf[["crawl_run_id", "url", "count"]].to_sql("nodes", conn, index=False, if_exists="append", method="multi") - conn.commit() - return - conn.execute("DELETE FROM nodes WHERE crawl_run_id = ?", (crawl_run_id,)) - ndf["crawl_run_id"] = crawl_run_id - ndf[["crawl_run_id", "url", "count"]].to_sql("nodes", conn, index=False, if_exists="append", method="multi") + crawl_run_id = rid + conn.execute("DELETE FROM nodes WHERE crawl_run_id = %s", (crawl_run_id,)) + _executemany( + conn, + "INSERT INTO nodes (crawl_run_id, url, count) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING", + [ + (crawl_run_id, str(r["url"]), int(r["count"])) + for _, r in ndf.iterrows() + ], + ) conn.commit() -def read_nodes(conn: sqlite3.Connection, run_id: Optional[int] = None) -> pd.DataFrame: - """Read nodes. If run_id is None, use latest crawl run.""" +def read_nodes(conn: Connection, run_id: Optional[int] = None) -> pd.DataFrame: try: if run_id is None: run_id = get_latest_crawl_run_id(conn) if run_id is None: return pd.DataFrame(columns=["url", "count"]) - return pd.read_sql("SELECT url, count FROM nodes WHERE crawl_run_id = ?", conn, params=(run_id,)) + cur = conn.execute( + "SELECT url, count FROM nodes WHERE crawl_run_id = %s", + (run_id,), + ) + rows = cur.fetchall() + if not rows: + return pd.DataFrame(columns=["url", "count"]) + return pd.DataFrame(rows) except Exception: return pd.DataFrame(columns=["url", "count"]) -def write_lighthouse_summary(conn: sqlite3.Connection, summary: dict[str, Any]) -> None: - """Append a lighthouse summary row (JSON in data column).""" +def write_lighthouse_summary(conn: Connection, summary: dict[str, Any]) -> None: conn.execute( - "INSERT INTO lighthouse_summary (created_at, data) VALUES (?, ?)", - (time.strftime("%Y-%m-%d %H:%M:%S"), json.dumps(_sanitize_for_json(summary), default=str)), + "INSERT INTO lighthouse_summary (created_at, data) VALUES (%s, %s)", + (_now_iso(), _json_val(summary)), ) conn.commit() -def read_lighthouse_summary(conn: sqlite3.Connection) -> Optional[dict[str, Any]]: - """Return the latest lighthouse summary dict, or None.""" +def read_lighthouse_summary(conn: Connection) -> Optional[dict[str, Any]]: try: - cur = conn.execute( - "SELECT data FROM lighthouse_summary ORDER BY id DESC LIMIT 1" - ) + cur = conn.execute("SELECT data FROM lighthouse_summary ORDER BY id DESC LIMIT 1") row = cur.fetchone() if row is None: return None - return json.loads(row[0]) + data = _parse_json_field(row["data"]) + return data if isinstance(data, dict) else None except Exception: return None def write_lighthouse_run( - conn: sqlite3.Connection, + conn: Connection, url: str, strategy: str, run_index: int, data: dict[str, Any], ) -> int: - """Append one raw Lighthouse run report (full JSON) to lighthouse_runs. Returns new row id.""" - conn.execute( - "INSERT INTO lighthouse_runs (created_at, url, strategy, run_index, data) VALUES (?, ?, ?, ?, ?)", - (time.strftime("%Y-%m-%d %H:%M:%S"), url, strategy, run_index, json.dumps(_sanitize_for_json(data), default=str)), + cur = conn.execute( + """INSERT INTO lighthouse_runs (created_at, url, strategy, run_index, data) + VALUES (%s, %s, %s, %s, %s) RETURNING id""", + (_now_iso(), url, strategy, run_index, _json_val(data)), ) + row = cur.fetchone() conn.commit() - cur = conn.execute("SELECT last_insert_rowid()") - return int(cur.fetchone()[0]) + return int(row["id"]) -def write_lh_audits_from_run(conn: sqlite3.Connection, run_id: int, lhr_data: dict[str, Any]) -> None: - """Parse LHR and insert lh_audits + lh_audit_items for the given lighthouse_runs.id.""" +def write_lh_audits_from_run(conn: Connection, run_id: int, lhr_data: dict[str, Any]) -> None: from ..lighthouse.schema import lhr_to_audit_rows audit_rows, item_refs = lhr_to_audit_rows(lhr_data) - id_map: list[int] = [] - for row in audit_rows: - conn.execute( + if not audit_rows: + return + + def _headings_val(row: dict) -> Any: + h = row.get("details_headings") + if isinstance(h, str) and h: + return _json_val(json.loads(h)) + return _json_val(h) + + def _meta_val(row: dict) -> Any: + m = row.get("details_meta") + if isinstance(m, str) and m: + return _json_val(json.loads(m)) + return _json_val(m) + + audit_params = [ + ( + run_id, + row["audit_id"], + row["category_id"], + row["score"], + row["score_display_mode"], + row["title"], + row["description"], + row["display_value"], + row["numeric_value"], + row["help_text"], + row["details_type"], + _headings_val(row), + _meta_val(row), + ) + for row in audit_rows + ] + + with conn.transaction(): + _executemany( + conn, """INSERT INTO lh_audits (run_id, audit_id, category_id, score, score_display_mode, - title, description, display_value, numeric_value, help_text, details_type, details_headings, details_meta) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""", - ( - run_id, - row["audit_id"], - row["category_id"], - row["score"], - row["score_display_mode"], - row["title"], - row["description"], - row["display_value"], - row["numeric_value"], - row["help_text"], - row["details_type"], - row["details_headings"], - row["details_meta"], - ), + title, description, display_value, numeric_value, help_text, details_type, + details_headings, details_meta) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""", + audit_params, + page_size=200, ) - id_map.append(int(conn.execute("SELECT last_insert_rowid()").fetchone()[0])) - for audit_idx, item_index, rd in item_refs: - audit_row_id = id_map[audit_idx] - conn.execute( - "INSERT INTO lh_audit_items (audit_row_id, item_index, row_data) VALUES (?,?,?)", - (audit_row_id, item_index, json.dumps(_sanitize_for_json(rd), default=str)), + cur = conn.execute( + "SELECT id FROM lh_audits WHERE run_id = %s ORDER BY id", + (run_id,), ) - conn.commit() + id_map = [int(r["id"]) for r in cur.fetchall()] + if len(id_map) != len(audit_rows): + return + item_params = [ + (id_map[audit_idx], item_index, _json_val(rd)) + for audit_idx, item_index, rd in item_refs + ] + if item_params: + _executemany( + conn, + "INSERT INTO lh_audit_items (audit_row_id, item_index, row_data) VALUES (%s, %s, %s)", + item_params, + page_size=500, + ) -def read_lh_runs_by_url(conn: sqlite3.Connection) -> dict[str, list[int]]: - """Map url -> ordered list of lighthouse_runs.id (ascending by id).""" +def read_lh_runs_by_url(conn: Connection) -> dict[str, list[int]]: out: dict[str, list[int]] = {} try: cur = conn.execute("SELECT id, url FROM lighthouse_runs ORDER BY id") for row in cur.fetchall(): - u = str(row[1]).strip().rstrip("/") - out.setdefault(u, []).append(int(row[0])) + u = str(row["url"]).strip().rstrip("/") + out.setdefault(u, []).append(int(row["id"])) except Exception: pass return out -def read_lighthouse_run_json(conn: sqlite3.Connection, run_id: int) -> Optional[dict[str, Any]]: - """Return parsed LHR JSON for a lighthouse_runs row, or None.""" +def read_lighthouse_run_json(conn: Connection, run_id: int) -> Optional[dict[str, Any]]: + try: + cur = conn.execute("SELECT data FROM lighthouse_runs WHERE id = %s", (run_id,)) + row = cur.fetchone() + if row is None: + return None + data = _parse_json_field(row["data"]) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def read_latest_lighthouse_run_json(conn: Connection) -> Optional[dict[str, Any]]: + """Return full Lighthouse JSON for the most recent lighthouse_runs row.""" try: - cur = conn.execute("SELECT data FROM lighthouse_runs WHERE id = ?", (run_id,)) + cur = conn.execute("SELECT data FROM lighthouse_runs ORDER BY id DESC LIMIT 1") row = cur.fetchone() if row is None: return None - return json.loads(row[0]) + data = _parse_json_field(row["data"]) + return data if isinstance(data, dict) else None except Exception: return None -def read_lh_audits_with_items(conn: sqlite3.Connection, run_id: int) -> list[dict[str, Any]]: - """Return audits in Lighthouse-like shape: id, title, score, details.items, etc.""" +def read_lh_audits_with_items(conn: Connection, run_id: int) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] try: - cur = conn.execute( - "SELECT * FROM lh_audits WHERE run_id = ? ORDER BY id", - (run_id,), - ) - for row in cur.fetchall(): - d = dict(row) + cur = conn.execute("SELECT * FROM lh_audits WHERE run_id = %s ORDER BY id", (run_id,)) + for d in cur.fetchall(): aid = d.get("audit_id") or "" - headings = None - if d.get("details_headings"): - try: - headings = json.loads(d["details_headings"]) - except (TypeError, json.JSONDecodeError): - headings = None - meta: dict[str, Any] = {} - if d.get("details_meta"): - try: - raw_meta = json.loads(d["details_meta"]) - if isinstance(raw_meta, dict): - meta = raw_meta - except (TypeError, json.JSONDecodeError): - meta = {} + headings = _parse_json_field(d.get("details_headings")) + meta = _parse_json_field(d.get("details_meta")) or {} + if not isinstance(meta, dict): + meta = {} cur_items = conn.execute( - "SELECT row_data FROM lh_audit_items WHERE audit_row_id = ? ORDER BY item_index", + "SELECT row_data FROM lh_audit_items WHERE audit_row_id = %s ORDER BY item_index", (d["id"],), ) items: list[Any] = [] - for (rd,) in cur_items.fetchall(): - try: - items.append(json.loads(rd)) - except (TypeError, json.JSONDecodeError): - items.append({}) + for item_row in cur_items.fetchall(): + rd = _parse_json_field(item_row["row_data"]) + items.append(rd if isinstance(rd, dict) else {}) details: dict[str, Any] = dict(meta) if d.get("details_type"): @@ -900,59 +872,71 @@ def read_lh_audits_with_items(conn: sqlite3.Connection, run_id: int) -> list[dic return out -def write_lighthouse_page_summary( - conn: sqlite3.Connection, - url: str, - summary: dict[str, Any], -) -> None: - """Write or replace Lighthouse summary for a single URL (latest run wins).""" +def write_lighthouse_page_summary(conn: Connection, url: str, summary: dict[str, Any]) -> None: conn.execute( - """INSERT OR REPLACE INTO lighthouse_page_summaries (url, created_at, data) - VALUES (?, ?, ?)""", - ( - url, - time.strftime("%Y-%m-%d %H:%M:%S"), - json.dumps(_sanitize_for_json(summary), default=str), - ), + """INSERT INTO lighthouse_page_summaries (url, created_at, data) + VALUES (%s, %s, %s) + ON CONFLICT (url) DO UPDATE SET created_at = EXCLUDED.created_at, data = EXCLUDED.data""", + (url, _now_iso(), _json_val(summary)), ) conn.commit() -def read_lighthouse_page_summaries(conn: sqlite3.Connection) -> dict[str, Any]: - """Return dict mapping url -> summary dict for all per-URL Lighthouse summaries.""" +def read_lighthouse_page_summaries(conn: Connection) -> dict[str, Any]: out: dict[str, Any] = {} + try: + cur = conn.execute("SELECT url, data FROM lighthouse_page_summaries") + for row in cur.fetchall(): + data = _parse_json_field(row["data"]) + if isinstance(data, dict): + out[str(row["url"])] = data + except Exception: + pass + return out + + +def read_llm_cache_batch(conn: Connection, cache_keys: list[str]) -> dict[str, dict[str, Any]]: + if not cache_keys: + return {} + out: dict[str, dict[str, Any]] = {} try: cur = conn.execute( - "SELECT url, data FROM lighthouse_page_summaries" + "SELECT cache_key, response_json FROM llm_cache WHERE cache_key = ANY(%s)", + (cache_keys,), ) for row in cur.fetchall(): - try: - out[str(row[0])] = json.loads(row[1]) - except (TypeError, json.JSONDecodeError): - continue + key = str(row["cache_key"]) + val = row["response_json"] + if isinstance(val, dict): + out[key] = val + elif isinstance(val, str): + try: + out[key] = json.loads(val) + except json.JSONDecodeError: + pass except Exception: pass return out -def write_report_payload(conn: sqlite3.Connection, report_data: dict[str, Any]) -> None: - """Insert the report payload JSON (used by frontend). NaN/Inf sanitized so JSON is valid.""" +def write_report_payload(conn: Connection, report_data: dict[str, Any]) -> None: + site_name = str(report_data.get("site_name") or "") + canonical_domain = _canonical_domain_from_report(conn, report_data) conn.execute( - "INSERT INTO report_payload (generated_at, data) VALUES (?, ?)", - (time.strftime("%Y-%m-%d %H:%M:%S"), json.dumps(_sanitize_for_json(report_data), default=str)), + """INSERT INTO report_payload (generated_at, site_name, canonical_domain, data) + VALUES (%s, %s, %s, %s)""", + (_now_iso(), site_name, canonical_domain, _json_val(report_data)), ) conn.commit() -def read_report_payload(conn: sqlite3.Connection) -> Optional[dict[str, Any]]: - """Return the latest report payload dict, or None.""" +def read_report_payload(conn: Connection) -> Optional[dict[str, Any]]: try: - cur = conn.execute( - "SELECT data FROM report_payload ORDER BY id DESC LIMIT 1" - ) + cur = conn.execute("SELECT data FROM report_payload ORDER BY id DESC LIMIT 1") row = cur.fetchone() if row is None: return None - return json.loads(row[0]) + data = _parse_json_field(row["data"]) + return data if isinstance(data, dict) else None except Exception: return None diff --git a/src/website_profiling/integrations/google/auth.py b/src/website_profiling/integrations/google/auth.py index 396c7f0c..da6ae20e 100644 --- a/src/website_profiling/integrations/google/auth.py +++ b/src/website_profiling/integrations/google/auth.py @@ -5,7 +5,7 @@ Path resolution order: 1. $GOOGLE_SECRETS_PATH env var - 2. dirname($REPORT_DB_PATH)/.secrets/google.json + 2. $DATA_DIR/.secrets/google.json 3. dirname(credentials_path from config) -- falls back to repo root """ from __future__ import annotations @@ -29,10 +29,10 @@ def _resolve_secrets_path(credentials_path: str | None = None) -> str: if explicit: return os.path.abspath(explicit) - # 2. Sibling to REPORT_DB_PATH (Docker volume) - db_env = os.environ.get("REPORT_DB_PATH", "").strip() - if db_env: - return os.path.join(os.path.dirname(os.path.abspath(db_env)), ".secrets", "google.json") + # 2. DATA_DIR (Docker volume /data) + data_dir = os.environ.get("DATA_DIR", "").strip() + if data_dir: + return os.path.join(os.path.abspath(data_dir), ".secrets", "google.json") # 3. Relative to credentials_path config key, or repo root if credentials_path and credentials_path.strip(): diff --git a/src/website_profiling/integrations/google/fetch.py b/src/website_profiling/integrations/google/fetch.py index d824a9f1..bb075647 100644 --- a/src/website_profiling/integrations/google/fetch.py +++ b/src/website_profiling/integrations/google/fetch.py @@ -1,6 +1,6 @@ """ Orchestrate GSC + GA4 fetching. Returns a structured google_data dict -suitable for storage in the google_data SQLite table and merging into report_payload. +suitable for storage in the google_data table and merging into report_payload. """ from __future__ import annotations @@ -161,9 +161,9 @@ def fetch_google_data( "fetched_at": datetime.now(timezone.utc).isoformat(), "date_range": {"start": date_start, "end": date_end}, "gsc": gsc_payload, - "gsc_full": gsc_data, # full data incl. by_page -- stored in SQLite only + "gsc_full": gsc_data, # full data incl. by_page -- stored in google_data only "ga4": ga4_payload, - "ga4_full": ga4_data, # full data incl. by_path -- stored in SQLite only + "ga4_full": ga4_data, # full data incl. by_path -- stored in google_data only "url_join": url_join, "errors": errors, } diff --git a/src/website_profiling/integrations/google/keyword_enrich.py b/src/website_profiling/integrations/google/keyword_enrich.py index 9ffa6115..58728f26 100644 --- a/src/website_profiling/integrations/google/keyword_enrich.py +++ b/src/website_profiling/integrations/google/keyword_enrich.py @@ -3,7 +3,7 @@ Merges data from: - Site crawl keywords (from tools/keywords.py) - - GSC queries (from google_data SQLite table -- no extra API calls) + - GSC queries (from google_data table -- no extra API calls) - Google Suggest: web + YouTube + question-prefixed - Datamuse (optional): semantic expansion - Wikipedia (optional): parent topic @@ -22,7 +22,6 @@ import json import re -import sqlite3 from datetime import datetime, timezone from typing import Any @@ -210,7 +209,6 @@ def compute_traffic_potential( # ── Main enrichment ─────────────────────────────────────────────────────────── def run_enrichment( - db_path: str, cfg: dict[str, Any], ) -> dict[str, Any]: """ @@ -218,12 +216,10 @@ def run_enrichment( computes all derived metrics, writes to keyword_data + keyword_history. Returns the enriched data dict. """ - import sqlite3 as _sqlite3 - from ...db.storage import db_session, init_schema + from ...db.storage import db_session from .keyword_store import ( write_keyword_data, append_keyword_history, - ensure_tables, ) from .store import read_latest_google_data from ..google.suggest import batch_expand @@ -242,10 +238,7 @@ def run_enrichment( print(" [Keywords] Running enrichment pipeline...", flush=True) - with db_session(db_path) as conn: - init_schema(conn) - ensure_tables(conn) - + with db_session() as conn: # 1. Load existing GSC data from google_data table gsc_queries: dict[str, dict] = {} # normalized_kw -> {position, impressions, clicks, ctr, url} gsc_by_page: dict[str, dict] = {} # url -> page data diff --git a/src/website_profiling/integrations/google/keyword_store.py b/src/website_profiling/integrations/google/keyword_store.py index 1df46785..0451a829 100644 --- a/src/website_profiling/integrations/google/keyword_store.py +++ b/src/website_profiling/integrations/google/keyword_store.py @@ -1,77 +1,37 @@ """ -Read/write keyword_data, keyword_history, and keyword_suggest_cache SQLite tables. - -keyword_data: latest enriched keyword snapshot (one JSON blob per run) -keyword_history: per-keyword time-series rows for position sparklines -keyword_suggest_cache: cache for Google Suggest responses (TTL-based) +Read/write keyword_data, keyword_history, and keyword_suggest_cache tables. """ from __future__ import annotations -import json -import sqlite3 -import time from datetime import datetime, timezone from typing import Any -TABLE_DDL = """ -CREATE TABLE IF NOT EXISTS keyword_data ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS keyword_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - keyword TEXT NOT NULL, - fetched_at TEXT NOT NULL, - position REAL, - clicks INTEGER, - impressions INTEGER, - ctr REAL -); -CREATE INDEX IF NOT EXISTS idx_kw_history_keyword ON keyword_history(keyword); - -CREATE TABLE IF NOT EXISTS keyword_suggest_cache ( - cache_key TEXT PRIMARY KEY, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL -); -""" - +from psycopg import Connection +from psycopg.types.json import Json -def ensure_tables(conn: sqlite3.Connection) -> None: - conn.executescript(TABLE_DDL) - conn.commit() +from ...db.storage import _parse_json_field, _sanitize_for_json -# ── keyword_data ────────────────────────────────────────────────────────────── - -def write_keyword_data(conn: sqlite3.Connection, data: dict[str, Any]) -> None: +def write_keyword_data(conn: Connection, data: dict[str, Any]) -> None: """Insert a new keyword_data snapshot.""" - ensure_tables(conn) fetched_at = data.get("fetched_at") or datetime.now(timezone.utc).isoformat() conn.execute( - "INSERT INTO keyword_data (fetched_at, data) VALUES (?, ?)", - (fetched_at, json.dumps(data, default=str)), + "INSERT INTO keyword_data (fetched_at, data) VALUES (%s, %s)", + (fetched_at, Json(_sanitize_for_json(data))), ) conn.commit() -def read_latest_keyword_data(conn: sqlite3.Connection) -> dict[str, Any] | None: - """ - Return the latest keyword_data row stripped of full history blobs. - Rows are capped at 500 for the payload. - """ - ensure_tables(conn) +def read_latest_keyword_data(conn: Connection) -> dict[str, Any] | None: + """Return the latest keyword_data row stripped of full history blobs.""" try: - cur = conn.execute( - "SELECT data FROM keyword_data ORDER BY id DESC LIMIT 1" - ) + cur = conn.execute("SELECT data FROM keyword_data ORDER BY id DESC LIMIT 1") row = cur.fetchone() if row is None: return None - data = json.loads(row[0]) - # Cap rows for payload to avoid bloat + data = _parse_json_field(row["data"]) + if not isinstance(data, dict): + return None if isinstance(data.get("rows"), list) and len(data["rows"]) > 1000: data["rows"] = data["rows"][:1000] return data @@ -79,15 +39,12 @@ def read_latest_keyword_data(conn: sqlite3.Connection) -> dict[str, Any] | None: return None -# ── keyword_history ─────────────────────────────────────────────────────────── - -def append_keyword_history(conn: sqlite3.Connection, rows: list[dict[str, Any]]) -> None: +def append_keyword_history(conn: Connection, rows: list[dict[str, Any]]) -> None: """Append per-keyword time-series rows for position tracking.""" - ensure_tables(conn) fetched_at = datetime.now(timezone.utc).isoformat() conn.executemany( - "INSERT INTO keyword_history (keyword, fetched_at, position, clicks, impressions, ctr) " - "VALUES (?, ?, ?, ?, ?, ?)", + """INSERT INTO keyword_history (keyword, fetched_at, position, clicks, impressions, ctr) + VALUES (%s, %s, %s, %s, %s, %s)""", [ ( r.get("keyword", ""), @@ -105,25 +62,24 @@ def append_keyword_history(conn: sqlite3.Connection, rows: list[dict[str, Any]]) def read_keyword_history( - conn: sqlite3.Connection, + conn: Connection, keyword: str, limit: int = 30, ) -> list[dict[str, Any]]: """Return time-series rows for a single keyword (for sparklines).""" - ensure_tables(conn) try: cur = conn.execute( - "SELECT fetched_at, position, clicks, impressions, ctr " - "FROM keyword_history WHERE keyword = ? ORDER BY id DESC LIMIT ?", + """SELECT fetched_at, position, clicks, impressions, ctr + FROM keyword_history WHERE keyword = %s ORDER BY id DESC LIMIT %s""", (keyword, limit), ) return [ { - "fetched_at": row[0], - "position": row[1], - "clicks": row[2], - "impressions": row[3], - "ctr": row[4], + "fetched_at": row["fetched_at"], + "position": row["position"], + "clicks": row["clicks"], + "impressions": row["impressions"], + "ctr": row["ctr"], } for row in cur.fetchall() ] diff --git a/src/website_profiling/integrations/google/store.py b/src/website_profiling/integrations/google/store.py index d2c1c740..b4d29122 100644 --- a/src/website_profiling/integrations/google/store.py +++ b/src/website_profiling/integrations/google/store.py @@ -1,63 +1,47 @@ """ -Read/write the google_data SQLite table. +Read/write the google_data table. The table stores the latest Google data snapshot (GSC + GA4). -Data survives report rebuilds because it is in a separate table from report_payload. """ from __future__ import annotations import json -import sqlite3 import time from typing import Any, Optional +from psycopg import Connection +from psycopg.types.json import Json -TABLE_DDL = """ -CREATE TABLE IF NOT EXISTS google_data ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL -); -""" - +from ...db.storage import _parse_json_field, _sanitize_for_json -def ensure_table(conn: sqlite3.Connection) -> None: - conn.execute(TABLE_DDL) - conn.commit() - -def write_google_data(conn: sqlite3.Connection, data: dict[str, Any]) -> None: +def write_google_data(conn: Connection, data: dict[str, Any]) -> None: """Insert a new google_data row. Older rows are kept (historical).""" - ensure_table(conn) fetched_at = data.get("fetched_at") or time.strftime("%Y-%m-%d %H:%M:%S") conn.execute( - "INSERT INTO google_data (fetched_at, data) VALUES (?, ?)", - (fetched_at, json.dumps(data, default=str)), + "INSERT INTO google_data (fetched_at, data) VALUES (%s, %s)", + (fetched_at, Json(_sanitize_for_json(data))), ) conn.commit() -def read_latest_google_data(conn: sqlite3.Connection) -> Optional[dict[str, Any]]: +def read_latest_google_data(conn: Connection) -> Optional[dict[str, Any]]: """ Return the latest google_data row as a dict suitable for report_payload["google"]. - Strips full by_page/by_path from the returned dict (those are only for SQLite lookups). - Returns None if no data exists. + Strips full by_page/by_path from the returned dict (those stay in DB for lookups). """ - ensure_table(conn) try: - cur = conn.execute( - "SELECT data FROM google_data ORDER BY id DESC LIMIT 1" - ) + cur = conn.execute("SELECT data FROM google_data ORDER BY id DESC LIMIT 1") row = cur.fetchone() if row is None: return None - data = json.loads(row[0]) - # Return payload-safe subset (no full by_page / by_path blobs) + data = _parse_json_field(row["data"]) + if not isinstance(data, dict): + return None return _to_payload_shape(data) except Exception: return None def _to_payload_shape(data: dict[str, Any]) -> dict[str, Any]: - """Strip gsc_full/ga4_full keys from the payload -- those stay in SQLite.""" - result = {k: v for k, v in data.items() if k not in ("gsc_full", "ga4_full")} - return result + """Strip gsc_full/ga4_full keys from the payload.""" + return {k: v for k, v in data.items() if k not in ("gsc_full", "ga4_full")} diff --git a/src/website_profiling/integrations/google/suggest.py b/src/website_profiling/integrations/google/suggest.py index f69820b0..5f93c766 100644 --- a/src/website_profiling/integrations/google/suggest.py +++ b/src/website_profiling/integrations/google/suggest.py @@ -8,14 +8,13 @@ Also performs question-prefixed expansion (who/what/why/when/where/how/can/should/vs) to surface People-Also-Ask-style queries without SERP scraping. -Caches results in keyword_suggest_cache SQLite table (TTL-based). +Caches results in keyword_suggest_cache table (TTL-based). Uses ThreadPoolExecutor for concurrency (default 4 workers). """ from __future__ import annotations import json import random -import sqlite3 import time import urllib.parse from concurrent.futures import ThreadPoolExecutor, as_completed @@ -23,6 +22,10 @@ from typing import Any import requests +from psycopg import Connection +from psycopg.types.json import Json + +from ...db.storage import _parse_json_field SUGGEST_URL = "https://suggestqueries.google.com/complete/search" USER_AGENT = ( @@ -120,13 +123,13 @@ def batch_expand( country: str = "us", sources: tuple[str, ...] = ("web", "youtube", "questions"), max_workers: int = 4, - cache_conn: sqlite3.Connection | None = None, + cache_conn: Connection | None = None, cache_ttl_days: int = 7, ) -> dict[str, dict[str, list[str]]]: """ Expand a list of seed keywords using Google Suggest. Returns { seed: { "web": [...], "youtube": [...], "questions": [...] } } - Uses concurrent requests and SQLite cache. + Uses concurrent requests and PostgreSQL cache (keyword_suggest_cache). """ result: dict[str, dict[str, list[str]]] = { seed: {s: [] for s in sources} for seed in seeds @@ -149,6 +152,7 @@ def batch_expand( if not tasks_to_fetch: return result + pending_cache: list[tuple[str, str, list[str]]] = [] with ThreadPoolExecutor(max_workers=max_workers) as pool: futures = {pool.submit(_fetch_one, task): task for task in tasks_to_fetch} for future in as_completed(futures): @@ -157,73 +161,72 @@ def batch_expand( if seed in result: result[seed][source] = suggestions if cache_conn is not None: - _write_cache(cache_conn, seed, source, suggestions) + pending_cache.append((seed, source, suggestions)) except Exception: pass + if cache_conn is not None and pending_cache: + flush_suggest_cache(cache_conn, pending_cache) + return result # ─── Cache helpers ──────────────────────────────────────────────────────────── -_CACHE_DDL = """ -CREATE TABLE IF NOT EXISTS keyword_suggest_cache ( - cache_key TEXT PRIMARY KEY, - fetched_at TEXT NOT NULL, - data TEXT NOT NULL -); -""" - - -def ensure_cache_table(conn: sqlite3.Connection) -> None: - conn.execute(_CACHE_DDL) - conn.commit() - def _cache_key(seed: str, source: str) -> str: return f"{source}:{seed}" def _read_cache( - conn: sqlite3.Connection, + conn: Connection, seed: str, source: str, ttl_days: int = 7, ) -> list[str] | None: try: - ensure_cache_table(conn) cur = conn.execute( - "SELECT fetched_at, data FROM keyword_suggest_cache WHERE cache_key = ?", + "SELECT fetched_at, data FROM keyword_suggest_cache WHERE cache_key = %s", (_cache_key(seed, source),), ) row = cur.fetchone() if row is None: return None - fetched_at = datetime.fromisoformat(row[0].replace("Z", "+00:00")) + fetched_raw = row["fetched_at"] + if hasattr(fetched_raw, "isoformat"): + fetched_at = fetched_raw if fetched_raw.tzinfo else fetched_raw.replace(tzinfo=timezone.utc) + else: + fetched_at = datetime.fromisoformat(str(fetched_raw).replace("Z", "+00:00")) age_days = (datetime.now(timezone.utc) - fetched_at).total_seconds() / 86400 if age_days > ttl_days: return None - return json.loads(row[1]) + data = _parse_json_field(row["data"]) + return data if isinstance(data, list) else json.loads(data) if isinstance(data, str) else None except Exception: return None -def _write_cache( - conn: sqlite3.Connection, - seed: str, - source: str, - data: list[str], +def flush_suggest_cache( + conn: Connection, + entries: list[tuple[str, str, list[str]]], ) -> None: + """Bulk-write suggest cache rows (main thread only — safe with one connection).""" + if not entries: + return + now = datetime.now(timezone.utc).isoformat() + rows = [ + (_cache_key(seed, source), now, Json(data)) + for seed, source, data in entries + ] try: - ensure_cache_table(conn) - conn.execute( - "INSERT OR REPLACE INTO keyword_suggest_cache (cache_key, fetched_at, data) VALUES (?, ?, ?)", - ( - _cache_key(seed, source), - datetime.now(timezone.utc).isoformat(), - json.dumps(data), - ), - ) + with conn.cursor() as cur: + for i in range(0, len(rows), 500): + cur.executemany( + """INSERT INTO keyword_suggest_cache (cache_key, fetched_at, data) + VALUES (%s, %s, %s) + ON CONFLICT (cache_key) DO UPDATE SET fetched_at = EXCLUDED.fetched_at, data = EXCLUDED.data""", + rows[i : i + 500], + ) conn.commit() except Exception: pass diff --git a/src/website_profiling/lighthouse/runner.py b/src/website_profiling/lighthouse/runner.py index 23230b73..defa67e8 100644 --- a/src/website_profiling/lighthouse/runner.py +++ b/src/website_profiling/lighthouse/runner.py @@ -1,7 +1,7 @@ """ Run Lighthouse locally via CLI for a given URL; return machine-readable summary with median metrics. Writes raw_runs/, summary.json, diagnostics.json, human_summary.txt, and optionally report.html. -Uses global lighthouse if on PATH, otherwise runs via npx (which will install it automatically). +Uses global lighthouse if on PATH (or LIGHTHOUSE_PATH), otherwise runs via npx (serialized to avoid cache races). Requires: Node + npm, Chrome/Chromium. """ import json @@ -11,6 +11,7 @@ import statistics import subprocess import sys +import threading from datetime import datetime, timezone from typing import Any @@ -25,6 +26,9 @@ "Chrome or Chromium is also required for headless mode." ) +# Serialise npx-on-demand installs — parallel npx runs corrupt /root/.npm/_npx cache in Docker. +_NPX_LIGHTHOUSE_LOCK = threading.Lock() + def _build_report_html_content(summary: dict[str, Any]) -> str: """Build report.html content (for DB or file). Returns HTML string.""" @@ -91,6 +95,9 @@ def _url_safe(s: str) -> str: def _lighthouse_cmd() -> list[str]: """Return argv prefix: [resolved lighthouse] or [resolved npx, -y, lighthouse]. Paths from shutil.which (portable).""" + explicit = (os.environ.get("LIGHTHOUSE_PATH") or os.environ.get("LIGHTHOUSE_BIN") or "").strip() + if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK): + return [explicit] lh = shutil.which("lighthouse") if lh is not None: return [lh] @@ -100,9 +107,18 @@ def _lighthouse_cmd() -> list[str]: raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG) +def _uses_npx(cmd: list[str]) -> bool: + base = os.path.basename(cmd[0]).lower() + return base in ("npx", "npx.cmd") + + def is_lighthouse_available() -> bool: """Return True if lighthouse or npx is on PATH (so we can run Lighthouse).""" - return shutil.which("lighthouse") is not None or shutil.which("npx") is not None + try: + _lighthouse_cmd() + return True + except RuntimeError: + return False def _preset_for_strategy(strategy: str) -> str: @@ -150,6 +166,14 @@ def run_lighthouse_once( if categories: cmd.append("--only-categories=" + ",".join(categories)) try: + if _uses_npx(base): + with _NPX_LIGHTHOUSE_LOCK: + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=300, + ) return subprocess.run( cmd, capture_output=True, @@ -416,72 +440,86 @@ def run_lighthouse_on_pages( strategy: str = "mobile", iterations: int = 1, output_dir: str = ".", - db_path: str | None = None, mode: str = "navigation", categories: str | list[str] | None = None, + concurrency: int = 2, ) -> None: """ - Run Lighthouse audit on each URL and store per-URL summary in DB. + Run Lighthouse audit on each URL and store per-URL summary in PostgreSQL. Failures for a single URL are logged and do not stop the rest. """ if not urls: return - if not db_path: - raise ValueError("run_lighthouse_on_pages requires db_path") if not is_lighthouse_available(): raise RuntimeError( "Node/npm not found. Install Node.js (https://nodejs.org); then run: npm install -g lighthouse. " "Chrome or Chromium is also required for headless mode." ) + from concurrent.futures import ThreadPoolExecutor, as_completed + from ..db import ( db_session, - init_schema, write_lh_audits_from_run, write_lighthouse_page_summary, write_lighthouse_run, ) - with db_session(db_path) as conn: - init_schema(conn) - strategy = strategy.lower() if strategy else "mobile" - if strategy not in ("mobile", "desktop"): - strategy = "mobile" - iterations = max(1, int(iterations)) - categories = _parse_categories(categories) if categories else None - total = len(urls) + strategy = strategy.lower() if strategy else "mobile" + if strategy not in ("mobile", "desktop"): + strategy = "mobile" + iterations = max(1, int(iterations)) + categories = _parse_categories(categories) if categories else None + total = len(urls) + workers = max(1, min(int(concurrency or 2), 8)) + + def _audit_one(url: str) -> None: + print(f"[Lighthouse on pages] {url}", flush=True) + summary = run_lighthouse_audit( + url=url, + strategy=strategy, + iterations=iterations, + output_dir=output_dir, + mode=mode, + categories=categories, + ) + lhr: dict[str, Any] | None = None + for raw_path in reversed(summary.get("raw_reports") or []): + if os.path.isfile(raw_path): + try: + with open(raw_path, "r", encoding="utf-8") as f: + lhr = json.load(f) + break + except (OSError, json.JSONDecodeError): + continue + with db_session() as conn: + write_lighthouse_page_summary(conn, url, summary) + if lhr is not None: + run_id = write_lighthouse_run(conn, url, strategy, 1, lhr) + write_lh_audits_from_run(conn, run_id, lhr) + for raw_path in summary.get("raw_reports") or []: + if os.path.isfile(raw_path): + try: + os.remove(raw_path) + except OSError: + pass + + if workers == 1: for idx, url in enumerate(urls): try: print(f"[Lighthouse on pages] {idx + 1}/{total}: {url}", flush=True) - summary = run_lighthouse_audit( - url=url, - strategy=strategy, - iterations=iterations, - output_dir=output_dir, - mode=mode, - categories=categories, - ) - write_lighthouse_page_summary(conn, url, summary) - lhr: dict[str, Any] | None = None - for raw_path in reversed(summary.get("raw_reports") or []): - if os.path.isfile(raw_path): - try: - with open(raw_path, "r", encoding="utf-8") as f: - lhr = json.load(f) - break - except (OSError, json.JSONDecodeError): - continue - if lhr is not None: - run_id = write_lighthouse_run(conn, url, strategy, 1, lhr) - write_lh_audits_from_run(conn, run_id, lhr) - # Delete raw run files after storing summary in DB (same as single-URL path) - for raw_path in summary.get("raw_reports") or []: - if os.path.isfile(raw_path): - try: - os.remove(raw_path) - except OSError: - pass + _audit_one(url) except Exception as e: print(f" Skipped (error): {e}", file=sys.stderr, flush=True) + else: + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(_audit_one, url): url for url in urls} + for future in as_completed(futures): + url = futures[future] + try: + future.result() + except Exception as e: + print(f" Skipped {url} (error): {e}", file=sys.stderr, flush=True) + print(f"[Lighthouse on pages] Done. Wrote {total} URL(s) to DB.", flush=True) @@ -491,13 +529,12 @@ def main( iterations: int = 3, output_dir: str = ".", summary_path: str | None = None, - db_path: str | None = None, + use_database: bool = True, mode: str = "navigation", categories: str | list[str] | None = None, ) -> int: """ - Run Lighthouse audit and write summary to JSON file and/or SQLite. Returns 0 on success, non-zero on error. - mode: 'navigation' (default), 'timespan', or 'snapshot'. categories: optional for --only-categories. + Run Lighthouse audit and write summary to JSON file and/or PostgreSQL. Returns 0 on success, non-zero on error. """ try: summary = run_lighthouse_audit( @@ -512,22 +549,19 @@ def main( print(str(e), file=sys.stderr) return 1 - # Store report HTML in summary so it is saved to DB when db_path is set + # Store report HTML in summary so it is saved to PostgreSQL when DATABASE_URL is set print(" Building report HTML...", flush=True) summary["report_html"] = _build_report_html_content(summary) - if db_path: - # Persist everything to DB only (no artifact files on disk) + if use_database: from ..db import ( db_session, - init_schema, write_lh_audits_from_run, write_lighthouse_summary, write_lighthouse_run, ) print(" Saving summary to DB...", flush=True) - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: write_lighthouse_summary(conn, summary) raw_reports = summary.get("raw_reports") or [] for i, raw_path in enumerate(raw_reports): @@ -546,7 +580,7 @@ def main( pass print(" Lighthouse DB write complete.", flush=True) print(summary.get("human_summary", "")) - print(f"All Lighthouse data saved to SQLite: {db_path} (summary, diagnostics, human summary, report HTML, raw runs)") + print(f"All Lighthouse data saved to PostgreSQL (summary, diagnostics, human summary, report HTML, raw runs)") else: # No DB: write all artifacts to output_dir print(" Writing summary.json...", flush=True) diff --git a/src/website_profiling/llm/enrich.py b/src/website_profiling/llm/enrich.py index f5d8efea..733ec196 100644 --- a/src/website_profiling/llm/enrich.py +++ b/src/website_profiling/llm/enrich.py @@ -6,7 +6,8 @@ import os import re from collections import Counter -from typing import Any, Optional +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Callable, Optional import pandas as pd @@ -65,15 +66,12 @@ def _cache_key(task: str, model: str, payload: str) -> str: return h -def _read_cache(db_path: Optional[str], key: str) -> Optional[dict[str, Any]]: - if not db_path or not os.path.isfile(db_path): - return None +def _read_cache(key: str) -> Optional[dict[str, Any]]: try: - from ..db import db_session, init_schema + from ..db import db_session from ..db.storage import read_llm_cache - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: raw = read_llm_cache(conn, key) if raw: return json.loads(raw) @@ -82,36 +80,99 @@ def _read_cache(db_path: Optional[str], key: str) -> Optional[dict[str, Any]]: return None -def _write_cache(db_path: Optional[str], key: str, data: dict[str, Any]) -> None: - if not db_path: - return +def _write_cache(key: str, data: dict[str, Any]) -> None: try: - from ..db import db_session, init_schema + from ..db import db_session from ..db.storage import write_llm_cache - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: write_llm_cache(conn, key, json.dumps(data)) except Exception: pass +def _llm_concurrency(cfg: dict[str, str]) -> int: + return max(1, min(_cfg_int(cfg, "llm_concurrency", 2) or 2, 8)) + + +def _run_llm_batches( + client: Any, + task: str, + system: str, + batches: list[dict[str, Any]], + cfg: dict[str, str], + apply_batch: Callable[[dict[str, Any], dict[str, Any]], None], +) -> None: + """Run LLM batches with batched cache lookup and optional parallel API calls.""" + if not batches: + return + model = (cfg.get("llm_model") or cfg.get("llm_provider") or "").strip() + keyed: list[tuple[str, dict[str, Any], str]] = [] + for payload in batches: + payload_str = json.dumps(payload, sort_keys=True) + ck = _cache_key(task, model, payload_str) + keyed.append((ck, payload, payload_str)) + + cached_map: dict[str, dict[str, Any]] = {} + try: + from ..db import db_session + from ..db.storage import read_llm_cache_batch + + with db_session() as conn: + cached_map = read_llm_cache_batch(conn, [k for k, _, _ in keyed]) + except Exception: + pass + + pending: list[tuple[str, dict[str, Any]]] = [] + for ck, payload, _ in keyed: + hit = cached_map.get(ck) + if hit is not None: + apply_batch(payload, hit) + else: + pending.append((ck, payload)) + + if not pending: + return + + workers = _llm_concurrency(cfg) + + def _one(item: tuple[str, dict[str, Any]]) -> tuple[str, dict[str, Any], dict[str, Any]]: + ck, payload = item + result = client.complete_json(system, json.dumps(payload)) + _write_cache(ck, result) + return ck, payload, result + + if workers <= 1 or len(pending) <= 1: + for item in pending: + _, payload, result = _one(item) + apply_batch(payload, result) + return + + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(_one, item) for item in pending] + for future in as_completed(futures): + try: + _, payload, result = future.result() + apply_batch(payload, result) + except Exception: + pass + + def _call_cached( client: Any, task: str, system: str, user_payload: dict[str, Any], cfg: dict[str, str], - db_path: Optional[str], ) -> dict[str, Any]: model = (cfg.get("llm_model") or cfg.get("llm_provider") or "").strip() payload_str = json.dumps(user_payload, sort_keys=True) ck = _cache_key(task, model, payload_str) - cached = _read_cache(db_path, ck) + cached = _read_cache(ck) if cached is not None: return cached result = client.complete_json(system, json.dumps(user_payload)) - _write_cache(db_path, ck, result) + _write_cache(ck, result) return result @@ -136,13 +197,12 @@ def _run_ner( client: Any, items: list[dict[str, str]], cfg: dict[str, str], - db_path: Optional[str], ) -> dict[str, dict[str, Any]]: batch_size = max(1, _cfg_int(cfg, "llm_batch_size", 5)) out: dict[str, dict[str, Any]] = {} - for i in range(0, len(items), batch_size): - batch = items[i : i + batch_size] - data = _call_cached(client, "ner", NER_SYSTEM, {"pages": batch}, cfg, db_path) + batches = [{"pages": items[i : i + batch_size]} for i in range(0, len(items), batch_size)] + + def apply_batch(_payload: dict[str, Any], data: dict[str, Any]) -> None: for p in data.get("pages") or []: u = str(p.get("url") or "").strip().rstrip("/") if not u: @@ -152,6 +212,8 @@ def _run_ner( "entity_count": int(p.get("entity_count") or 0), "top_entity_labels": labels, } + + _run_llm_batches(client, "ner", NER_SYSTEM, batches, cfg, apply_batch) return out @@ -159,13 +221,12 @@ def _run_keyphrases( client: Any, items: list[dict[str, str]], cfg: dict[str, str], - db_path: Optional[str], ) -> dict[str, dict[str, Any]]: batch_size = max(1, _cfg_int(cfg, "llm_batch_size", 5)) out: dict[str, dict[str, Any]] = {} - for i in range(0, len(items), batch_size): - batch = items[i : i + batch_size] - data = _call_cached(client, "keyphrases", KEYPHRASES_SYSTEM, {"pages": batch}, cfg, db_path) + batches = [{"pages": items[i : i + batch_size]} for i in range(0, len(items), batch_size)] + + def apply_batch(_payload: dict[str, Any], data: dict[str, Any]) -> None: for p in data.get("pages") or []: u = str(p.get("url") or "").strip().rstrip("/") if not u: @@ -173,6 +234,8 @@ def _run_keyphrases( phrases = p.get("phrases") or [] pairs = [[str(x[0]), float(x[1])] for x in phrases if isinstance(x, (list, tuple)) and len(x) >= 2] out[u] = {"phrases": pairs} + + _run_llm_batches(client, "keyphrases", KEYPHRASES_SYSTEM, batches, cfg, apply_batch) return out @@ -180,20 +243,17 @@ def _run_similar_internal( client: Any, items: list[dict[str, str]], cfg: dict[str, str], - db_path: Optional[str], ) -> dict[str, list[dict[str, Any]]]: top_k = min(_cfg_int(cfg, "llm_similar_top_k", 5) or 5, 15) all_urls = [x["url"] for x in items] out: dict[str, list[dict[str, Any]]] = {} batch_size = max(1, min(_cfg_int(cfg, "llm_batch_size", 5), 3)) - for i in range(0, len(items), batch_size): - batch = items[i : i + batch_size] - payload = { - "pages": batch, - "candidate_urls": all_urls[:80], - "top_k": top_k, - } - data = _call_cached(client, "similar", SIMILAR_SYSTEM, payload, cfg, db_path) + batches = [ + {"pages": items[i : i + batch_size], "candidate_urls": all_urls[:80], "top_k": top_k} + for i in range(0, len(items), batch_size) + ] + + def apply_batch(_payload: dict[str, Any], data: dict[str, Any]) -> None: for p in data.get("pages") or []: u = str(p.get("url") or "").strip().rstrip("/") if not u: @@ -204,13 +264,14 @@ def _run_similar_internal( sim.append({"url": str(s["url"]), "score": round(float(s.get("score") or 0), 4)}) if sim: out[u] = sim + + _run_llm_batches(client, "similar", SIMILAR_SYSTEM, batches, cfg, apply_batch) return out def cluster_keywords_llm( keywords: list[str], cfg: dict[str, str] | None, - db_path: Optional[str] = None, ) -> list[dict[str, Any]]: if not keywords or not cfg or not llm_is_enabled(cfg): return [] @@ -227,7 +288,6 @@ def cluster_keywords_llm( KEYWORD_CLUSTER_SYSTEM, {"keywords": kws}, cfg, - db_path, ) clusters = data.get("clusters") or [] out: list[dict[str, Any]] = [] @@ -253,7 +313,6 @@ def cluster_keywords_llm( def run_llm_enrichment( df: pd.DataFrame, cfg: dict[str, str] | None, - db_path: Optional[str] = None, ) -> dict[str, Any]: bundle: dict[str, Any] = { "spacy_by_url": {}, @@ -278,19 +337,19 @@ def run_llm_enrichment( if _cfg_bool(cfg, "llm_enable_ner", True): try: - bundle["spacy_by_url"] = _run_ner(client, items, cfg, db_path) + bundle["spacy_by_url"] = _run_ner(client, items, cfg) except Exception as e: bundle["ml_errors"].append(f"LLM NER: {e}") if _cfg_bool(cfg, "llm_enable_keyphrases", True): try: - bundle["keyphrases_by_url"] = _run_keyphrases(client, items, cfg, db_path) + bundle["keyphrases_by_url"] = _run_keyphrases(client, items, cfg) except Exception as e: bundle["ml_errors"].append(f"LLM keyphrases: {e}") if _cfg_bool(cfg, "llm_enable_similar_internal", True): try: - bundle["similar_internal_by_url"] = _run_similar_internal(client, items, cfg, db_path) + bundle["similar_internal_by_url"] = _run_similar_internal(client, items, cfg) except Exception as e: bundle["ml_errors"].append(f"LLM similar pages: {e}") diff --git a/src/website_profiling/llm_config.py b/src/website_profiling/llm_config.py index ed82b34f..4ca55da0 100644 --- a/src/website_profiling/llm_config.py +++ b/src/website_profiling/llm_config.py @@ -1,5 +1,5 @@ """ -Load LLM settings from report.db llm_config table only (UI-managed). +Load LLM settings from llm_config table only (UI-managed). Not read from pipeline-config.txt or --config files. """ from __future__ import annotations @@ -14,15 +14,12 @@ } -def load_llm_config_from_db(db_path: str) -> dict[str, str]: - if not db_path or not os.path.isfile(db_path): - return {} +def load_llm_config_from_db() -> dict[str, str]: try: - from .db import db_session, init_schema + from .db import db_session from .db.storage import read_llm_config - with db_session(db_path) as conn: - init_schema(conn) + with db_session() as conn: cfg = read_llm_config(conn) except Exception: return {} diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index 0408a239..03bc8850 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -1,5 +1,5 @@ """ -Generate report data from crawl and write to SQLite. The Next.js UI in web/ reads report.db via /api/report/*. +Generate report data from crawl and write to PostgreSQL. The Next.js UI in web/ reads via /api/report/*. """ import hashlib import json @@ -19,11 +19,9 @@ from ..common import ( LINK_COLUMN_NAMES, - load_dataframe, load_edges, normalize_link, parse_links_serialized, - save_edges, ) from ..tools.keywords import cluster_keywords, extract_candidates_from_df, score_keywords from ..config import get_bool, get_int @@ -236,7 +234,7 @@ def build_edges_from_df( polite_delay: float, ) -> list[tuple[str, str]]: """Build or load edges; return list of (from, to) tuples.""" - edges = load_edges(edges_csv) + edges = load_edges(edges_csv) if (edges_csv or "").strip() else [] if edges: return edges @@ -849,9 +847,6 @@ def _build_keyword_opportunities(df: pd.DataFrame, config: dict[str, str] | None def run_simple_report( - crawl_csv: str, - edges_csv: str = "edges.csv", - output_html: str = "site_report.html", max_fetch_for_edges: int = 300, concurrency: int = 6, timeout: int = 8, @@ -863,50 +858,39 @@ def run_simple_report( run_security_scan_flag: bool = True, security_scan_active: bool = False, security_max_urls_probe: int = 20, - security_findings_output: Optional[str] = None, lighthouse_summary_path: Optional[str] = None, - db_path: Optional[str] = None, config: Optional[dict[str, str]] = None, + use_database: bool = True, ) -> str: - """Load crawl data, build edges if needed, write report payload to SQLite. Returns db_path. Requires db_path (Next.js UI in web/ reads via /api/report/*).""" + """Load crawl data from PostgreSQL, build report payload, write to report_payload.""" + if not use_database: + raise ValueError("Report requires DATABASE_URL (PostgreSQL). Configure via Docker or local Postgres.") + + from ..db import ( + db_session, + get_crawl_run_info, + get_latest_crawl_run_id, + read_crawl, + read_edges, + read_lighthouse_summary, + write_edges, + ) run_id = None crawl_run_created_at: Optional[str] = None - if db_path: - from ..db import ( - db_session, - get_crawl_run_info, - get_latest_crawl_run_id, - init_schema, - read_crawl, - read_edges, - read_lighthouse_summary, - write_edges, - ) - print(" Loading crawl data from DB...", flush=True) - with db_session(db_path) as conn: - init_schema(conn) - run_id = get_latest_crawl_run_id(conn) - if run_id is not None: - info = get_crawl_run_info(conn, run_id) - crawl_run_created_at = info["created_at"] if info else None - df = read_crawl(conn, run_id) - edges = read_edges(conn, run_id) - global_lighthouse_summary = read_lighthouse_summary(conn) - lighthouse_by_url = build_lighthouse_by_url_for_report(conn) - lighthouse_summary = global_lighthouse_summary - print(f" Loaded {len(df)} URLs, {len(edges)} edges.", flush=True) - if df.empty and not edges: - raise FileNotFoundError(f"No crawl or edges data in DB: {db_path}") - else: - if not os.path.exists(crawl_csv): - raise FileNotFoundError(f"Crawl data not found: {crawl_csv}") - print(" Loading crawl data from file...", flush=True) - df = load_dataframe(crawl_csv) - edges = [] - lighthouse_summary = None - lighthouse_by_url = {} - global_lighthouse_summary = None - print(f" Loaded {len(df)} URLs.", flush=True) + print(" Loading crawl data from DB...", flush=True) + with db_session() as conn: + run_id = get_latest_crawl_run_id(conn) + if run_id is not None: + info = get_crawl_run_info(conn, run_id) + crawl_run_created_at = info["created_at"] if info else None + df = read_crawl(conn, run_id) + edges = read_edges(conn, run_id) + global_lighthouse_summary = read_lighthouse_summary(conn) + lighthouse_by_url = build_lighthouse_by_url_for_report(conn) + lighthouse_summary = global_lighthouse_summary + print(f" Loaded {len(df)} URLs, {len(edges)} edges.", flush=True) + if df.empty and not edges: + raise FileNotFoundError("No crawl or edges data in database. Run crawl first.") if "url" not in df.columns and not df.empty: raise ValueError("Crawl DataFrame missing required column 'url'") @@ -918,13 +902,12 @@ def run_simple_report( expected_host = _derive_expected_host(start_url or "", df) if lighthouse_by_url and expected_host: lighthouse_by_url = filter_lighthouse_by_host(lighthouse_by_url, expected_host) - if db_path: - lighthouse_summary = _pick_lighthouse_summary( - lighthouse_by_url, - start_url or "", - global_lighthouse_summary, - expected_host, - ) + lighthouse_summary = _pick_lighthouse_summary( + lighthouse_by_url, + start_url or "", + global_lighthouse_summary, + expected_host, + ) site_display = (site_name or "").strip() or (urlparse(start_url or "").netloc if start_url else "") or "Site" report_display_title = (report_title or "").strip() or f"{site_display} — Crawl Report" @@ -932,14 +915,12 @@ def run_simple_report( if not edges and not df.empty: print(" Building edges from crawl data...", flush=True) edges = build_edges_from_df( - df, edges_csv, same_domain_only, max_fetch_for_edges, concurrency, timeout, 0.12 + df, "", same_domain_only, max_fetch_for_edges, concurrency, timeout, 0.12 ) print(f" Edges: {len(edges)}.", flush=True) - if edges and db_path: - with db_session(db_path) as conn: + if edges: + with db_session() as conn: write_edges(conn, edges, run_id) - elif edges and not db_path: - save_edges(edges, edges_csv) # Long report work (ML, graph, network) runs without a DB handle; payload write uses db_session again. @@ -969,31 +950,14 @@ def run_simple_report( polite_delay=0.2, ) print(f" Security scan: {len(security_findings)} findings.", flush=True) - if security_findings_output: - with open(security_findings_output, "w", encoding="utf-8") as fh: - json.dump(security_findings, fh, indent=2, default=str) print(" Content analysis (local + optional LLM)...", flush=True) local_bundle = run_local_enrichment(df, config) - llm_cfg = load_llm_config_from_db(db_path) if db_path else {} - llm_bundle = run_llm_enrichment(df, llm_cfg, db_path=db_path) if llm_is_enabled(llm_cfg) else {} + llm_cfg = load_llm_config_from_db() + llm_bundle = run_llm_enrichment(df, llm_cfg) if llm_is_enabled(llm_cfg) else {} ml_bundle = merge_bundles(local_bundle, llm_bundle) print(" Building report categories...", flush=True) - if not db_path: - lighthouse_summary = None - if lighthouse_summary_path and os.path.isfile(lighthouse_summary_path): - try: - with open(lighthouse_summary_path, "r", encoding="utf-8") as fh: - lighthouse_summary = json.load(fh) - if lighthouse_summary and expected_host: - if not _hosts_match( - _url_hostname(str(lighthouse_summary.get("url") or "")), - expected_host, - ): - lighthouse_summary = None - except (OSError, json.JSONDecodeError): - pass categories = build_categories( df, edges, summary_seo, site_level, start_url or "", @@ -1327,13 +1291,13 @@ def _bool_col(col): print(" Building content analytics...", flush=True) content_analytics = _build_content_analytics(df) semantic_keyword_clusters: list[dict[str, Any]] = [] - llm_cfg_for_clusters = load_llm_config_from_db(db_path) if db_path else {} - if db_path and llm_is_enabled(llm_cfg_for_clusters): + llm_cfg_for_clusters = load_llm_config_from_db() + if llm_is_enabled(llm_cfg_for_clusters): try: llm_cfg = llm_cfg_for_clusters if str(llm_cfg.get("llm_enable_keyword_clusters", "")).lower() in ("true", "1", "yes"): words = [x["word"] for x in (content_analytics.get("top_keywords_site") or []) if x.get("word")] - semantic_keyword_clusters = cluster_keywords_llm(words, llm_cfg, db_path=db_path) + semantic_keyword_clusters = cluster_keywords_llm(words, llm_cfg) except Exception as e: ml_bundle.setdefault("ml_errors", []).append(str(e)) outbound_max = get_int(config or {}, "outbound_domain_max_rows", 200) or 200 @@ -1389,7 +1353,7 @@ def _bool_col(col): "keyword_opportunities": keyword_opportunities, "ml_errors": ml_bundle.get("ml_errors") or [], } - if db_path and run_id is not None: + if run_id is not None: report_data["crawl_run_id"] = run_id report_data["crawl_run_created_at"] = crawl_run_created_at if lighthouse_summary: @@ -1397,36 +1361,27 @@ def _bool_col(col): report_data["lighthouse_diagnostics"] = lighthouse_summary.get("diagnostics") or [] report_data["lighthouse_human_summary"] = lighthouse_summary.get("human_summary_full") or lighthouse_summary.get("human_summary") or "" report_data["lighthouse_by_url"] = lighthouse_by_url - if db_path: - print(" Writing report payload to DB...", flush=True) - from ..db import db_session as _db, init_schema as _init, write_report_payload as db_write_report_payload - with _db(db_path) as conn: - _init(conn) - # Carry forward Google data from dedicated table so report rebuilds preserve it - try: - from ..integrations.google.store import read_latest_google_data - google_data = read_latest_google_data(conn) - if google_data: - report_data["google"] = google_data - except Exception: - pass - # Carry forward enriched keyword data (cap at top 500 by traffic potential) - try: - from ..integrations.google.keyword_store import read_latest_keyword_data - kw_data = read_latest_keyword_data(conn) - if kw_data: - # Cap rows to keep payload lean - rows = kw_data.get("rows") or [] - if len(rows) > 500: - rows = rows[:500] - kw_data = {**kw_data, "rows": rows} - report_data["keywords"] = kw_data - except Exception: - pass - db_write_report_payload(conn, report_data) - return db_path - raise ValueError( - "Report requires sqlite_db. Set sqlite_db = report.db in your config; " - "the Next.js UI in web/ reads report.db via /api/report/*." - ) + print(" Writing report payload to DB...", flush=True) + from ..db import db_session as _db, write_report_payload as db_write_report_payload + with _db() as conn: + try: + from ..integrations.google.store import read_latest_google_data + google_data = read_latest_google_data(conn) + if google_data: + report_data["google"] = google_data + except Exception: + pass + try: + from ..integrations.google.keyword_store import read_latest_keyword_data + kw_data = read_latest_keyword_data(conn) + if kw_data: + rows = kw_data.get("rows") or [] + if len(rows) > 500: + rows = rows[:500] + kw_data = {**kw_data, "rows": rows} + report_data["keywords"] = kw_data + except Exception: + pass + db_write_report_payload(conn, report_data) + return "postgresql" diff --git a/src/website_profiling/reporting/categories.py b/src/website_profiling/reporting/categories.py index 56d9880b..3bcfa1b3 100644 --- a/src/website_profiling/reporting/categories.py +++ b/src/website_profiling/reporting/categories.py @@ -228,7 +228,7 @@ def category_core_web_vitals_from_lighthouse(lighthouse_summary: dict) -> dict: issues.append(_issue( msg, priority="High" if (f.get("score") or 0) < 0.5 else "Medium", - recommendation="See Lighthouse report for fix; run 'python -m src warnings' with the Lighthouse JSON for one-line fixes.", + recommendation="See Lighthouse diagnostics in the report, or run 'python -m src warnings' to refresh mapped fixes in PostgreSQL.", )) if not issues and perf_score is not None and perf_score < 80: recommendations.append("Improve Core Web Vitals (LCP, CLS, TBT) per Lighthouse recommendations.") diff --git a/src/website_profiling/tools/keywords.py b/src/website_profiling/tools/keywords.py index f53402f6..68dd7d2f 100644 --- a/src/website_profiling/tools/keywords.py +++ b/src/website_profiling/tools/keywords.py @@ -1,11 +1,8 @@ """ -SEO keyword discovery and scoring from on-site content. Crawls site (or uses existing crawl), -extracts candidate keywords from titles, headings, meta, URL slugs; scores and clusters them; -outputs ranked CSV, clusters JSON, and human summary. +SEO keyword discovery and scoring from on-site content. Uses PostgreSQL crawl data, +extracts candidate keywords, scores and clusters them, and writes to keyword_data. """ -import csv import json -import os import re import sys from datetime import datetime, timezone @@ -48,7 +45,6 @@ def _slug_tokens(url: str) -> list[str]: segments = [s for s in path.split("/") if s and s not in ("html", "php", "asp", "aspx", "jsp")] out = [] for seg in segments: - # Split on hyphen/underscore and clean words = re.findall(r"\b[\w']+\b", seg.replace("-", " ").replace("_", " ").lower()) out.extend(words) return out @@ -92,11 +88,9 @@ def _relevance_tfidf(candidates: dict[str, dict], corpus_size: int) -> dict[str, """Simple TF-IDF style: relevance = (count / total_docs) * log(corpus_size / doc_freq).""" total_docs = corpus_size or 1 doc_freq = {k: len(v["sources"]) for k, v in candidates.items()} - max_df = max(doc_freq.values()) or 1 scores: dict[str, float] = {} for kw, data in candidates.items(): df = doc_freq.get(kw, 1) - # Higher when term is repeated but not everywhere (idf) idf = 1.0 + (total_docs / max(df, 1)) ** 0.5 tf = min(1.0, (data["count"] or 0) / max(total_docs, 1)) scores[kw] = min(1.0, (tf * idf) / 10.0) @@ -110,16 +104,14 @@ def score_keywords( ) -> list[dict[str, Any]]: """ Score each candidate. Without external data: search_volume and difficulty are estimated; - relevance from TF-IDF; ctr_est placeholder. Composite = volume*w_v + relevance*w_r + ctr_est*w_c + (1-difficulty)*w_e. + relevance from TF-IDF; ctr_est placeholder. """ weights = weights or DEFAULT_WEIGHTS relevance_scores = _relevance_tfidf(candidates, corpus_size or len(candidates)) results: list[dict[str, Any]] = [] for kw, data in candidates.items(): - # Estimate volume: no API -> use frequency on site as proxy (normalized 0..1) raw_vol = (data.get("count") or 0) / max(corpus_size or 1, 1) * 100 volume = min(1.0, raw_vol) - # Difficulty: no API -> middle default so ease = 0.5 difficulty = 50.0 ease = 1.0 - (difficulty / 100.0) relevance = relevance_scores.get(kw, 0.5) @@ -131,7 +123,6 @@ def score_keywords( + weights.get("ctr_est", 0.15) * ctr_est + weights.get("ease", 0.15) * ease ) - # recommended_action: heuristic if len(data.get("sources") or []) > 1: action = "internal link" elif relevance > 0.7: @@ -155,13 +146,9 @@ def score_keywords( def cluster_keywords(scored: list[dict[str, Any]]) -> list[dict[str, Any]]: - """ - Group similar keywords by shared tokens (simple overlap). Each cluster has - top keyword (by score), cluster score (average of keyword scores), and list of keywords. - """ + """Group similar keywords by shared tokens (simple overlap).""" if not scored: return [] - # Build clusters: keywords that share at least one token go together (greedy). clusters: list[set[str]] = [] kw_to_tokens: dict[str, set[str]] = {} for s in scored: @@ -201,29 +188,32 @@ def cluster_keywords(scored: list[dict[str, Any]]) -> list[dict[str, Any]]: return out +def _load_crawl_from_db() -> pd.DataFrame: + from ..db import db_session, get_latest_crawl_run_id, read_crawl + + with db_session() as conn: + run_id = get_latest_crawl_run_id(conn) + return read_crawl(conn, run_id) + + def run_keyword_pipeline( base_url: str, - output_dir: str, config: dict[str, str] | None = None, - crawl_csv_path: str | None = None, max_pages: int = 200, ) -> dict[str, Any]: """ - Run crawl (or load existing crawl), extract and score keywords, cluster, write CSV and JSON. - Returns summary dict with paths and human summary. + Run crawl (or load latest from PostgreSQL), extract and score keywords, cluster. + Writes keyword_data to PostgreSQL. Returns summary dict. """ config = config or {} from ..config import get_list + exclude_urls = get_list(config, "crawl_exclude_urls", sep=",") - os.makedirs(output_dir, exist_ok=True) - cwd = os.getcwd() + df = _load_crawl_from_db() - if crawl_csv_path and os.path.isfile(crawl_csv_path): - from ..common import load_dataframe - df = load_dataframe(crawl_csv_path) - else: + if df.empty: from ..crawl.crawler import run_crawler - crawl_out = os.path.join(output_dir, "keyword_crawl.json") + run_crawler( start_url=base_url, max_pages=max_pages, @@ -234,17 +224,15 @@ def run_keyword_pipeline( max_depth=6, polite_delay=0.2, store_outlinks=False, - output_csv=crawl_out, + output_csv=None, + output_db=True, show_progress=True, exclude_urls=exclude_urls if exclude_urls else None, ) - from ..common import load_dataframe - df = load_dataframe(crawl_out) + df = _load_crawl_from_db() if df.empty: return { - "top_keywords_path": None, - "clusters_path": None, "human_summary": "No crawl data; no keywords extracted.", "quick_wins": [], "high_value": [], @@ -257,16 +245,10 @@ def run_keyword_pipeline( clusters = cluster_keywords(scored) semantic_clusters: list[dict[str, Any]] = [] - llm_cfg: dict[str, str] = {} - db_raw = (config.get("sqlite_db") or "").strip() - if db_raw: - db_path = db_raw if os.path.isabs(db_raw) else os.path.join(output_dir, db_raw) - env_db = (os.environ.get("REPORT_DB_PATH") or "").strip() - if env_db: - db_path = os.path.abspath(env_db) + try: from ..llm_config import load_llm_config_from_db, llm_is_enabled - llm_cfg = load_llm_config_from_db(db_path) + llm_cfg = load_llm_config_from_db() if llm_is_enabled(llm_cfg) and str(llm_cfg.get("llm_enable_keyword_clusters", "")).lower() in ( "true", "1", @@ -276,29 +258,13 @@ def run_keyword_pipeline( from ..llm.enrich import cluster_keywords_llm top_kw = [s["keyword"] for s in scored[:200] if s.get("keyword")] - semantic_clusters = cluster_keywords_llm(top_kw, llm_cfg, db_path=db_path) + semantic_clusters = cluster_keywords_llm(top_kw, llm_cfg) except Exception as e: print(f"Semantic keywords skipped: {e}", file=sys.stderr) + except Exception: + pass ts = datetime.now(timezone.utc).isoformat() - top_path = os.path.join(output_dir, "top_keywords.csv") - clusters_path = os.path.join(output_dir, "clusters.json") - - with open(top_path, "w", newline="", encoding="utf-8") as f: - cols = ["keyword", "score", "volume", "difficulty", "relevance", "ctr_est", "current_rank", "recommended_action", "source"] - w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") - w.writeheader() - for row in scored: - w.writerow({k: row.get(k) for k in cols}) - - out_meta = { - "timestamp": ts, - "config": {"url": base_url, "weights": weights, "data_sources": ["site"]}, - "clusters": clusters, - "clusters_semantic": semantic_clusters, - } - with open(clusters_path, "w", encoding="utf-8") as f: - json.dump(out_meta, f, indent=2, default=str) quick_wins = [s for s in scored if s.get("difficulty", 100) < 60][:10] high_value = [s for s in scored if (s.get("volume") or 0) >= 0.5][:10] @@ -310,33 +276,27 @@ def run_keyword_pipeline( ] human_summary = " ".join(summary_lines) - # Write scored keywords to keyword_data SQLite table (for Google enrichment merge) - if db_path := (config or {}).get("_db_path"): - try: - import sqlite3 as _sqlite - from ..db.storage import db_session as _db, init_schema as _init - from ..integrations.google.keyword_store import write_keyword_data, ensure_tables - - rows_for_db = [ - {**r, "sources": ["site"]} - for r in scored - ] - blob = { - "fetched_at": ts, - "total_keywords": len(rows_for_db), - "rows": rows_for_db, - "source": "site", - } - with _db(db_path) as conn: - _init(conn) - ensure_tables(conn) - write_keyword_data(conn, blob) - except Exception as e: - print(f" Warning: could not write keyword_data to SQLite: {e}", file=sys.stderr) + try: + from ..db.storage import db_session as _db + from ..integrations.google.keyword_store import write_keyword_data + + rows_for_db = [{**r, "sources": ["site"]} for r in scored] + blob = { + "fetched_at": ts, + "total_keywords": len(rows_for_db), + "rows": rows_for_db, + "source": "site", + "clusters": clusters, + "clusters_semantic": semantic_clusters, + "config": {"url": base_url, "weights": weights, "data_sources": ["site"]}, + } + with _db() as conn: + write_keyword_data(conn, blob) + print(" Keywords stored in PostgreSQL (keyword_data).", flush=True) + except Exception as e: + print(f" Warning: could not write keyword_data to database: {e}", file=sys.stderr) return { - "top_keywords_path": top_path, - "clusters_path": clusters_path, "human_summary": human_summary, "quick_wins": quick_wins[:10], "high_value": high_value[:10], @@ -346,31 +306,19 @@ def run_keyword_pipeline( def main( base_url: str, - output_dir: str, config: dict[str, str] | None = None, ) -> int: - """ - Run keyword pipeline and print summary. Returns 0 on success. - """ + """Run keyword pipeline and print summary. Returns 0 on success.""" + config = config or {} try: - crawl_csv = (config or {}).get("crawl_csv", "").strip() - cwd = (config or {}).get("_cwd") or os.getcwd() - if crawl_csv and not os.path.isabs(crawl_csv): - crawl_csv = os.path.join(cwd, crawl_csv) max_pages = int((config or {}).get("keyword_max_pages") or 0) or 200 summary = run_keyword_pipeline( base_url=base_url, - output_dir=output_dir, config=config, - crawl_csv_path=crawl_csv if os.path.isfile(crawl_csv) else None, max_pages=max_pages, ) except Exception as e: print(str(e), file=sys.stderr) return 1 print(summary.get("human_summary", "")) - if summary.get("top_keywords_path"): - print(f"top_keywords.csv: {summary['top_keywords_path']}") - if summary.get("clusters_path"): - print(f"clusters.json: {summary['clusters_path']}") return 0 diff --git a/src/website_profiling/tools/plot.py b/src/website_profiling/tools/plot.py index 87425463..1f397e41 100644 --- a/src/website_profiling/tools/plot.py +++ b/src/website_profiling/tools/plot.py @@ -1,49 +1,38 @@ """ -Build edges from crawl data and persist nodes/edges (DB or files). +Build edges from crawl data and persist nodes/edges to PostgreSQL. """ -import os from typing import Optional import pandas as pd -from ..common import load_dataframe, load_edges, save_dataframe, save_edges from ..reporting.builder import build_edges_from_df def run_plot( - crawl_csv: str, - edges_csv: str = "edges.csv", - nodes_csv: str = "nodes.csv", same_domain_only: bool = True, max_fetch_for_edges: int = 500, concurrency: int = 8, timeout: int = 10, polite_delay: float = 0.15, - db_path: Optional[str] = None, -) -> tuple[str, str]: + use_database: bool = True, +) -> str: """ - Load crawl data, build edges (and nodes), write to DB or CSV/JSON. - Returns (edges_csv path, nodes_csv path). + Load crawl data, build edges (and nodes), write to PostgreSQL. + Returns a storage label (``postgresql``). """ + if not use_database: + raise ValueError("Plot requires DATABASE_URL (PostgreSQL).") + run_id = None - if db_path: - print(" Loading crawl and edges from DB...", flush=True) - from ..db import db_session, get_latest_crawl_run_id, init_schema, read_crawl, read_edges - with db_session(db_path) as conn: - init_schema(conn) - run_id = get_latest_crawl_run_id(conn) - df = read_crawl(conn, run_id) - edges = read_edges(conn, run_id) - print(f" Loaded {len(df)} URLs, {len(edges)} edges.", flush=True) - if df.empty and not edges: - raise FileNotFoundError(f"No crawl or edges data in DB: {db_path}") - else: - if not os.path.exists(crawl_csv): - raise FileNotFoundError(f"Crawl data not found: {crawl_csv}") - print(" Loading crawl data from file...", flush=True) - df = load_dataframe(crawl_csv) - edges = [] - print(f" Loaded {len(df)} URLs.", flush=True) + print(" Loading crawl and edges from DB...", flush=True) + from ..db import db_session, get_latest_crawl_run_id, read_crawl, read_edges + with db_session() as conn: + run_id = get_latest_crawl_run_id(conn) + df = read_crawl(conn, run_id) + edges = read_edges(conn, run_id) + print(f" Loaded {len(df)} URLs, {len(edges)} edges.", flush=True) + if df.empty and not edges: + raise FileNotFoundError("No crawl or edges data in database.") if not df.empty and "url" not in df.columns: raise ValueError("Crawl DataFrame missing 'url' column") @@ -55,31 +44,20 @@ def run_plot( if not edges and not df.empty: print(" Building edges from crawl data...", flush=True) edges = build_edges_from_df( - df, edges_csv, same_domain_only, max_fetch_for_edges, concurrency, timeout, polite_delay + df, "", same_domain_only, max_fetch_for_edges, concurrency, timeout, polite_delay ) print(f" Edges: {len(edges)}.", flush=True) - if not edges and not db_path: - edges = load_edges(edges_csv) - if edges: edges_df = pd.DataFrame(edges, columns=["from", "to"]) - if db_path: - print(" Writing edges and nodes to DB...", flush=True) - from ..db import db_session, get_latest_crawl_run_id, init_schema, write_edges as db_write_edges, write_nodes as db_write_nodes - with db_session(db_path) as conn: - init_schema(conn) - rid = run_id if run_id is not None else get_latest_crawl_run_id(conn) - db_write_edges(conn, edges, rid) - nodes = pd.Series(list(edges_df["from"]) + list(edges_df["to"])) - nodes = nodes.value_counts().reset_index() - nodes.columns = ["url", "count"] - db_write_nodes(conn, nodes, rid) - else: - save_edges(edges, edges_csv) + print(" Writing edges and nodes to DB...", flush=True) + from ..db import db_session, get_latest_crawl_run_id, write_edges as db_write_edges, write_nodes as db_write_nodes + with db_session() as conn: + rid = run_id if run_id is not None else get_latest_crawl_run_id(conn) + db_write_edges(conn, edges, rid) nodes = pd.Series(list(edges_df["from"]) + list(edges_df["to"])) nodes = nodes.value_counts().reset_index() nodes.columns = ["url", "count"] - save_dataframe(nodes, nodes_csv) + db_write_nodes(conn, nodes, rid) - return edges_csv, nodes_csv + return "postgresql" diff --git a/src/website_profiling/tools/warnings.py b/src/website_profiling/tools/warnings.py index 11f9d7db..71b1b752 100644 --- a/src/website_profiling/tools/warnings.py +++ b/src/website_profiling/tools/warnings.py @@ -1,6 +1,7 @@ """ Map site warnings (Lighthouse, axe, or plain list) to detection method, affected metrics, -severity, and one-line actionable fix. Outputs JSON mapping and human summary. +severity, and one-line actionable fix. Default path: read Lighthouse from PostgreSQL, +write mapped warnings to report_payload.warnings_mapped. """ import json import os @@ -425,6 +426,42 @@ def parse_plain_list(path: str) -> list[dict[str, Any]]: return results +def map_warnings_from_data( + data: dict[str, Any], + input_type: str, +) -> list[dict[str, Any]]: + """Map warnings from in-memory Lighthouse, axe, or plain-line list data.""" + input_type = (input_type or "lighthouse").lower() + if input_type == "lighthouse": + return _parse_lighthouse_data(data) + if input_type == "axe": + violations = data.get("violations") or [] + results: list[dict[str, Any]] = [] + for v in violations: + rule_id = v.get("id") or "" + help_text = v.get("help") or "" + desc = v.get("description") or "" + warning = f"{help_text}: {desc}"[:200] + entry = _resolve_entry(rule_id, help_text, desc) + refs: dict[str, Any] = {"lighthouse_audit_id": rule_id} + nodes = v.get("nodes") or [] + if nodes: + refs["nodes"] = [n.get("target") or n.get("html") for n in nodes[:10]] + results.append(_build_output_item(warning, entry, refs)) + return results + if input_type in ("list", "plain", "text"): + lines = data.get("lines") or [] + results = [] + for line in lines: + line = str(line).strip() + if not line: + continue + entry = _resolve_entry("", line, line) + results.append(_build_output_item(line, entry, None)) + return results + raise ValueError(f"Unknown input_type: {input_type}. Use lighthouse, axe, or list.") + + def map_warnings( input_path: str, input_type: str, @@ -460,23 +497,43 @@ def human_summary_paragraph(items: list[dict[str, Any]], top_n: int = 5) -> str: def main( - input_path: str, + input_path: str | None = None, input_type: str = "lighthouse", - output_path: str = "warnings_mapped.json", ) -> int: """ - Run warning mapper and write JSON + print human summary. - Returns 0 on success, non-zero on error. + Map Lighthouse/axe/list warnings to actionable fixes. + Default: read latest Lighthouse run from PostgreSQL, write to report_payload. + Optional input_path overrides with a local file (axe/list or external Lighthouse JSON). """ - if not input_path or not input_path.strip(): - print("warning_mapper_input is required. Set it in config or pass the Lighthouse/axe/list file path.", file=sys.stderr) - return 1 - input_path = input_path.strip() - if not os.path.isabs(input_path): - input_path = os.path.abspath(input_path) + input_type = (input_type or "lighthouse").lower() + input_path = (input_path or "").strip() + try: - print(f" Reading input: {input_path} (type={input_type})...", flush=True) - items = map_warnings(input_path, input_type) + if input_path: + print(f" Reading input file: {input_path} (type={input_type})...", flush=True) + items = map_warnings(input_path, input_type) + else: + from ..db import db_session, read_latest_lighthouse_run_json + + print(" Loading latest Lighthouse run from PostgreSQL...", flush=True) + with db_session() as conn: + if input_type == "lighthouse": + data = read_latest_lighthouse_run_json(conn) + if not data: + print( + "No Lighthouse runs in PostgreSQL. Run lighthouse first, " + "or set warning_mapper_input to a JSON file path.", + file=sys.stderr, + ) + return 1 + items = map_warnings_from_data(data, input_type) + else: + print( + "warning_mapper_input is required for axe/list types. " + "Set a file path in config or use input_type=lighthouse for DB mode.", + file=sys.stderr, + ) + return 1 print(f" Mapped {len(items)} warnings.", flush=True) except FileNotFoundError as e: print(str(e), file=sys.stderr) @@ -490,9 +547,13 @@ def main( "warnings": items, "human_summary": human_summary_paragraph(items, 5), } - print(f" Writing output: {output_path}...", flush=True) - with open(output_path, "w", encoding="utf-8") as f: - json.dump(output, f, indent=2, default=str) + + from ..db import db_session, read_report_payload, write_report_payload + + with db_session() as conn: + payload = read_report_payload(conn) or {} + payload["warnings_mapped"] = output + write_report_payload(conn, payload) + print(" Warnings stored in PostgreSQL (report_payload.warnings_mapped).", flush=True) print(output["human_summary"]) - print(f"Written to {output_path}") return 0 diff --git a/tests/test_llm_config.py b/tests/test_llm_config.py index cf34a8bc..78ec8338 100644 --- a/tests/test_llm_config.py +++ b/tests/test_llm_config.py @@ -1,31 +1,35 @@ -"""Tests for LLM config loading.""" +"""Tests for LLM config loading (PostgreSQL).""" from __future__ import annotations import os -import tempfile -from website_profiling.db import db_session, init_schema +import pytest + +from website_profiling.db import db_session from website_profiling.db.storage import write_llm_config from website_profiling.llm_config import llm_is_enabled, load_llm_config_from_db -def test_load_llm_config_from_db(): - with tempfile.TemporaryDirectory() as tmp: - db_path = os.path.join(tmp, "report.db") - with db_session(db_path) as conn: - init_schema(conn) - write_llm_config( - conn, - { - "llm_enabled": "true", - "llm_provider": "ollama", - "llm_model": "llama3.2", - }, - secret_keys=set(), - ) - cfg = load_llm_config_from_db(db_path) - assert cfg.get("llm_provider") == "ollama" - assert llm_is_enabled(cfg) +@pytest.fixture(scope="module") +def require_database_url(): + if not (os.environ.get("DATABASE_URL") or "").strip(): + pytest.skip("DATABASE_URL not set — start Postgres and run alembic upgrade head") + + +def test_load_llm_config_from_db(require_database_url): + with db_session() as conn: + write_llm_config( + conn, + { + "llm_enabled": "true", + "llm_provider": "ollama", + "llm_model": "llama3.2", + }, + secret_keys=set(), + ) + cfg = load_llm_config_from_db() + assert cfg.get("llm_provider") == "ollama" + assert llm_is_enabled(cfg) def test_llm_disabled_by_default(): diff --git a/tests/test_storage_bulk.py b/tests/test_storage_bulk.py new file mode 100644 index 00000000..f4c928cb --- /dev/null +++ b/tests/test_storage_bulk.py @@ -0,0 +1,53 @@ +"""Bulk PostgreSQL storage tests.""" +from __future__ import annotations + +import os + +import pandas as pd +import pytest +from psycopg.types.json import Json + +from website_profiling.db import db_session, read_crawl, write_crawl +from website_profiling.db.storage import create_crawl_run, write_crawl_batch + + +@pytest.fixture(scope="module") +def pg_conn(): + if not (os.environ.get("DATABASE_URL") or "").strip(): + pytest.skip("DATABASE_URL not set — start Postgres and run alembic upgrade head") + with db_session() as conn: + yield conn + + +def test_write_crawl_bulk_round_trip(pg_conn): + run_id = create_crawl_run(pg_conn, "https://example.com") + rows = [] + for i in range(100): + rows.append( + ( + run_id, + f"https://example.com/page-{i}", + "200", + f"Page {i}", + Json({"status": "200", "title": f"Page {i}"}), + ) + ) + write_crawl_batch(pg_conn, rows, run_id, commit=True) + + df = read_crawl(pg_conn, run_id) + assert len(df) == 100 + assert "url" in df.columns + assert df["status"].astype(str).str.startswith("200").all() + + +def test_write_crawl_dataframe_executemany(pg_conn): + run_id = create_crawl_run(pg_conn, "https://bulk.example.com") + data = { + "url": [f"https://bulk.example.com/{i}" for i in range(50)], + "status": ["200"] * 50, + "title": [f"T{i}" for i in range(50)], + } + df = pd.DataFrame(data) + write_crawl(pg_conn, df, crawl_run_id=run_id) + out = read_crawl(pg_conn, run_id) + assert len(out) == 50 diff --git a/tests/test_suggest_cache_concurrent.py b/tests/test_suggest_cache_concurrent.py new file mode 100644 index 00000000..5b654c12 --- /dev/null +++ b/tests/test_suggest_cache_concurrent.py @@ -0,0 +1,55 @@ +"""Thread-safe suggest cache tests.""" +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest + +from website_profiling.db import db_session +from website_profiling.integrations.google.suggest import batch_expand, flush_suggest_cache + + +@pytest.fixture(scope="module") +def pg_conn(): + if not (os.environ.get("DATABASE_URL") or "").strip(): + pytest.skip("DATABASE_URL not set — start Postgres and run alembic upgrade head") + with db_session() as conn: + yield conn + + +def _fake_fetch(task, timeout=8.0): + seed, source, lang, country = task + return seed, source, [f"{seed}-{source}-a", f"{seed}-{source}-b"] + + +def test_batch_expand_flushes_cache_on_main_thread(pg_conn): + seeds = [f"seed{i}" for i in range(6)] + with patch( + "website_profiling.integrations.google.suggest._fetch_one", + side_effect=_fake_fetch, + ): + result = batch_expand( + seeds, + sources=("web",), + max_workers=4, + cache_conn=pg_conn, + ) + assert len(result) == 6 + for seed in seeds: + assert result[seed]["web"] + + cur = pg_conn.execute( + "SELECT COUNT(*) AS n FROM keyword_suggest_cache WHERE cache_key LIKE 'web:%'" + ) + row = cur.fetchone() + assert int(row["n"]) >= 6 + + +def test_flush_suggest_cache_executemany(pg_conn): + entries = [(f"bulk{i}", "web", [f"kw{i}"]) for i in range(10)] + flush_suggest_cache(pg_conn, entries) + cur = pg_conn.execute( + "SELECT COUNT(*) AS n FROM keyword_suggest_cache WHERE cache_key LIKE 'web:bulk%'" + ) + assert int(cur.fetchone()["n"]) >= 10 diff --git a/web/app/(reports)/[slug]/page.tsx b/web/app/(reports)/[slug]/page.tsx new file mode 100644 index 00000000..62bc1493 --- /dev/null +++ b/web/app/(reports)/[slug]/page.tsx @@ -0,0 +1,18 @@ +import ReportShell from '@/ReportShell'; +import { pathSlugToViewId } from '@/routes'; +import { notFound } from 'next/navigation'; +import type { ReactElement } from 'react'; + +export const dynamic = 'force-dynamic'; + +export default async function SlugPage({ + params, +}: { + params: Promise<{ slug: string }>; +}): Promise { + const { slug } = await params; + if (!pathSlugToViewId(slug)) { + notFound(); + } + return ; +} diff --git a/web/app/(reports)/layout.tsx b/web/app/(reports)/layout.tsx new file mode 100644 index 00000000..fe212c17 --- /dev/null +++ b/web/app/(reports)/layout.tsx @@ -0,0 +1,6 @@ +import { ReportAppClient } from '@/ReportShell'; +import type { ReactNode } from 'react'; + +export default function ReportsLayout({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/web/app/(reports)/not-found.tsx b/web/app/(reports)/not-found.tsx new file mode 100644 index 00000000..e0c7169d --- /dev/null +++ b/web/app/(reports)/not-found.tsx @@ -0,0 +1,28 @@ +import Link from 'next/link'; + +export default function ReportsNotFound() { + return ( +
+
+

Page not found

+

+ That report view does not exist. Choose a valid section from the app. +

+
+ + Go to Home + + + Open Pipeline + +
+
+
+ ); +} diff --git a/web/app/[slug]/page.jsx b/web/app/[slug]/page.jsx deleted file mode 100644 index 0518bffe..00000000 --- a/web/app/[slug]/page.jsx +++ /dev/null @@ -1,8 +0,0 @@ -import ReportShell from '@/ReportShell'; - -export const dynamic = 'force-dynamic'; - -export default async function SlugPage({ params }) { - const { slug } = await params; - return ; -} diff --git a/web/app/api/integrations/google/auth/route.js b/web/app/api/integrations/google/auth/route.ts similarity index 62% rename from web/app/api/integrations/google/auth/route.js rename to web/app/api/integrations/google/auth/route.ts index 1ee4ae94..f26c15d4 100644 --- a/web/app/api/integrations/google/auth/route.js +++ b/web/app/api/integrations/google/auth/route.ts @@ -1,7 +1,12 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { randomUUID } from 'crypto'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { readSecrets } from '@/server/googleSecrets'; +import { + GOOGLE_OAUTH_RETURN_COOKIE, + validateOAuthReturnPath, +} from '@/server/oauthReturn'; +import type { ApiRouteHandler } from '@/types/api'; export const runtime = 'nodejs'; @@ -10,11 +15,18 @@ const SCOPES = [ 'https://www.googleapis.com/auth/analytics.readonly', ].join(' '); +const OAUTH_COOKIE_OPTS = { + httpOnly: true, + maxAge: 300, + path: '/', + sameSite: 'lax' as const, +}; + /** * GET /api/integrations/google/auth - * Generates a CSRF state token, sets an httpOnly cookie, redirects to Google OAuth. + * Generates a CSRF state token, stores return path, redirects to Google OAuth. */ -export async function GET(request) { +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; @@ -33,11 +45,12 @@ export async function GET(request) { error: 'No Google Client ID configured. Go to Integrations and complete Step 1 first.', }, - { status: 400 } + { status: 400 }, ); } const state = randomUUID(); + const returnTo = validateOAuthReturnPath(request.nextUrl.searchParams.get('returnTo')); const params = new URLSearchParams({ client_id: clientId, @@ -52,12 +65,7 @@ export async function GET(request) { const authUrl = `https://accounts.google.com/o/oauth2/v2/auth?${params.toString()}`; const response = NextResponse.redirect(authUrl); - // CSRF state cookie: httpOnly, 5 minute TTL, SameSite=Lax - response.cookies.set('google_oauth_state', state, { - httpOnly: true, - maxAge: 300, - path: '/', - sameSite: 'lax', - }); + response.cookies.set('google_oauth_state', state, OAUTH_COOKIE_OPTS); + response.cookies.set(GOOGLE_OAUTH_RETURN_COOKIE, returnTo, OAUTH_COOKIE_OPTS); return response; -} +}; diff --git a/web/app/api/integrations/google/callback/route.js b/web/app/api/integrations/google/callback/route.js deleted file mode 100644 index 7e9d2143..00000000 --- a/web/app/api/integrations/google/callback/route.js +++ /dev/null @@ -1,92 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { readSecrets, writeSecrets } from '@/server/googleSecrets'; - -export const runtime = 'nodejs'; - -/** - * GET /api/integrations/google/callback - * Validates CSRF state, exchanges code for tokens, stores refresh token. - * Redirects to /?integrations=open&auth=success|error - */ -export async function GET(request) { - const denied = forbiddenIfNotLocal(request); - if (denied) return denied; - - const { searchParams } = request.nextUrl; - const code = searchParams.get('code'); - const state = searchParams.get('state'); - const errorParam = searchParams.get('error'); - - const appBase = process.env.GOOGLE_REDIRECT_URI - ? process.env.GOOGLE_REDIRECT_URI.replace('/api/integrations/google/callback', '') - : 'http://localhost:3000'; - - if (errorParam) { - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent(errorParam)}` - ); - } - - // CSRF validation - const cookieState = request.cookies.get('google_oauth_state')?.value; - if (!state || !cookieState || state !== cookieState) { - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent('Invalid state parameter. Please try connecting again.')}` - ); - } - - if (!code) { - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent('No authorization code received.')}` - ); - } - - const secrets = readSecrets() || {}; - const clientId = secrets.clientId || process.env.GOOGLE_CLIENT_ID || ''; - const clientSecret = secrets.clientSecret || process.env.GOOGLE_CLIENT_SECRET || ''; - const redirectUri = - process.env.GOOGLE_REDIRECT_URI || - `http://localhost:3000/api/integrations/google/callback`; - - if (!clientId || !clientSecret) { - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent('Client credentials missing. Complete Step 1 in Integrations.')}` - ); - } - - try { - const tokenRes = await fetch('https://oauth2.googleapis.com/token', { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams({ - code, - client_id: clientId, - client_secret: clientSecret, - redirect_uri: redirectUri, - grant_type: 'authorization_code', - }).toString(), - }); - - const tokenData = await tokenRes.json(); - if (!tokenRes.ok || !tokenData.refresh_token) { - const reason = tokenData.error_description || tokenData.error || 'Token exchange failed'; - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent(reason)}` - ); - } - - writeSecrets({ authMode: 'oauth', refreshToken: tokenData.refresh_token }); - - const response = NextResponse.redirect( - `${appBase}/?integrations=open&auth=success` - ); - // Clear CSRF cookie - response.cookies.set('google_oauth_state', '', { maxAge: 0, path: '/' }); - return response; - } catch (e) { - return NextResponse.redirect( - `${appBase}/?integrations=open&auth=error&reason=${encodeURIComponent(e.message)}` - ); - } -} diff --git a/web/app/api/integrations/google/callback/route.ts b/web/app/api/integrations/google/callback/route.ts new file mode 100644 index 00000000..923ff8ef --- /dev/null +++ b/web/app/api/integrations/google/callback/route.ts @@ -0,0 +1,144 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { readSecrets, writeSecrets } from '@/server/googleSecrets'; +import { + GOOGLE_OAUTH_RETURN_COOKIE, + oauthRedirectUrl, + validateOAuthReturnPath, +} from '@/server/oauthReturn'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; + +interface TokenExchangeResponse { + refresh_token?: string; + error?: string; + error_description?: string; +} + +function appBaseFromEnv(): string { + return process.env.GOOGLE_REDIRECT_URI + ? process.env.GOOGLE_REDIRECT_URI.replace('/api/integrations/google/callback', '') + : 'http://localhost:3000'; +} + +function oauthErrorRedirect( + appBase: string, + returnPath: string, + reason: string, +): NextResponse { + return NextResponse.redirect( + oauthRedirectUrl(appBase, returnPath, { + integrations: 'open', + auth: 'error', + reason, + }), + ); +} + +function clearOAuthCookies(response: NextResponse): void { + response.cookies.set('google_oauth_state', '', { maxAge: 0, path: '/' }); + response.cookies.set(GOOGLE_OAUTH_RETURN_COOKIE, '', { maxAge: 0, path: '/' }); +} + +/** + * GET /api/integrations/google/callback + * Validates CSRF state, exchanges code for tokens, stores refresh token. + * Redirects to stored return path with integrations=open&auth=success|error. + */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + const { searchParams } = request.nextUrl; + const code = searchParams.get('code'); + const state = searchParams.get('state'); + const errorParam = searchParams.get('error'); + + const appBase = appBaseFromEnv(); + const returnPath = validateOAuthReturnPath( + request.cookies.get(GOOGLE_OAUTH_RETURN_COOKIE)?.value, + ); + + if (errorParam) { + const response = oauthErrorRedirect(appBase, returnPath, errorParam); + clearOAuthCookies(response); + return response; + } + + const cookieState = request.cookies.get('google_oauth_state')?.value; + if (!state || !cookieState || state !== cookieState) { + const response = oauthErrorRedirect( + appBase, + returnPath, + 'Invalid state parameter. Please try connecting again.', + ); + clearOAuthCookies(response); + return response; + } + + if (!code) { + const response = oauthErrorRedirect( + appBase, + returnPath, + 'No authorization code received.', + ); + clearOAuthCookies(response); + return response; + } + + const secrets = readSecrets() || {}; + const clientId = secrets.clientId || process.env.GOOGLE_CLIENT_ID || ''; + const clientSecret = secrets.clientSecret || process.env.GOOGLE_CLIENT_SECRET || ''; + const redirectUri = + process.env.GOOGLE_REDIRECT_URI || + `http://localhost:3000/api/integrations/google/callback`; + + if (!clientId || !clientSecret) { + const response = oauthErrorRedirect( + appBase, + returnPath, + 'Client credentials missing. Complete Step 1 in Integrations.', + ); + clearOAuthCookies(response); + return response; + } + + try { + const tokenRes = await fetch('https://oauth2.googleapis.com/token', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + code, + client_id: clientId, + client_secret: clientSecret, + redirect_uri: redirectUri, + grant_type: 'authorization_code', + }).toString(), + }); + + const tokenData = (await tokenRes.json()) as TokenExchangeResponse; + if (!tokenRes.ok || !tokenData.refresh_token) { + const reason = tokenData.error_description || tokenData.error || 'Token exchange failed'; + const response = oauthErrorRedirect(appBase, returnPath, reason); + clearOAuthCookies(response); + return response; + } + + writeSecrets({ authMode: 'oauth', refreshToken: tokenData.refresh_token }); + + const response = NextResponse.redirect( + oauthRedirectUrl(appBase, returnPath, { + integrations: 'open', + auth: 'success', + }), + ); + clearOAuthCookies(response); + return response; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + const response = oauthErrorRedirect(appBase, returnPath, msg); + clearOAuthCookies(response); + return response; + } +}; diff --git a/web/app/api/integrations/google/credentials/route.js b/web/app/api/integrations/google/credentials/route.ts similarity index 77% rename from web/app/api/integrations/google/credentials/route.js rename to web/app/api/integrations/google/credentials/route.ts index 95df18f9..50c05a15 100644 --- a/web/app/api/integrations/google/credentials/route.js +++ b/web/app/api/integrations/google/credentials/route.ts @@ -1,6 +1,7 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { writeSecrets, getPublicStatus } from '@/server/googleSecrets'; +import type { ApiRouteHandler, GoogleCredentialsPostBody, GoogleSecrets } from '@/types/api'; export const runtime = 'nodejs'; @@ -8,12 +9,12 @@ export const runtime = 'nodejs'; * Body: { clientId?, clientSecret?, refreshToken?, gscSiteUrl?, ga4PropertyId?, dateRangeDays? } * Merges into existing .secrets/google.json (atomic write). */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; try { - const body = await request.json().catch(() => ({})); - const patch = {}; + const body = (await request.json().catch(() => ({}))) as GoogleCredentialsPostBody; + const patch: Partial = {}; if (typeof body.clientId === 'string' && body.clientId.trim()) { patch.clientId = body.clientId.trim(); @@ -36,7 +37,7 @@ export async function POST(request) { error: 'Analytics property ID must be a numeric ID (e.g. 123456789). The G-XXXXXXX code is a Measurement ID — find the numeric ID in GA4 Admin > Property Settings.', }, - { status: 400 } + { status: 400 }, ); } patch.ga4PropertyId = v || null; @@ -52,6 +53,7 @@ export async function POST(request) { writeSecrets(patch); return NextResponse.json({ ok: true, status: getPublicStatus() }); } catch (e) { - return NextResponse.json({ error: e.message }, { status: 500 }); + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/integrations/google/credentials/upload/route.js b/web/app/api/integrations/google/credentials/upload/route.ts similarity index 55% rename from web/app/api/integrations/google/credentials/upload/route.js rename to web/app/api/integrations/google/credentials/upload/route.ts index 3d2d97a9..06fc8800 100644 --- a/web/app/api/integrations/google/credentials/upload/route.js +++ b/web/app/api/integrations/google/credentials/upload/route.ts @@ -1,47 +1,58 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { writeSecrets, getPublicStatus } from '@/server/googleSecrets'; +import type { ApiRouteHandler, GoogleCredentialsUploadBody, GoogleServiceAccount } from '@/types/api'; export const runtime = 'nodejs'; +function isServiceAccount(value: unknown): value is GoogleServiceAccount { + return ( + value != null && + typeof value === 'object' && + (value as GoogleServiceAccount).type === 'service_account' && + typeof (value as GoogleServiceAccount).client_email === 'string' && + typeof (value as GoogleServiceAccount).private_key === 'string' + ); +} + /** * POST /api/integrations/google/credentials/upload * Accepts a JSON body with { fileContent: "" } - * Validates the service account key structure and saves it. */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; try { - const body = await request.json().catch(() => ({})); + const body = (await request.json().catch(() => ({}))) as GoogleCredentialsUploadBody; const raw = body.fileContent; if (!raw || typeof raw !== 'string') { return NextResponse.json({ error: 'fileContent is required' }, { status: 400 }); } - let parsed; + let parsed: unknown; try { parsed = JSON.parse(raw); } catch { return NextResponse.json( { error: "This doesn't look like a valid JSON file." }, - { status: 400 } + { status: 400 }, ); } - if (parsed.type !== 'service_account' || !parsed.client_email || !parsed.private_key) { + if (!isServiceAccount(parsed)) { return NextResponse.json( { error: "This doesn't look like a Google service account key file. Make sure you downloaded the JSON key from Google Cloud Console > IAM & Admin > Service Accounts.", }, - { status: 400 } + { status: 400 }, ); } writeSecrets({ authMode: 'service_account', serviceAccount: parsed }); return NextResponse.json({ ok: true, status: getPublicStatus() }); } catch (e) { - return NextResponse.json({ error: e.message }, { status: 500 }); + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/integrations/google/disconnect/route.js b/web/app/api/integrations/google/disconnect/route.ts similarity index 62% rename from web/app/api/integrations/google/disconnect/route.js rename to web/app/api/integrations/google/disconnect/route.ts index 2162ce8d..83f15db6 100644 --- a/web/app/api/integrations/google/disconnect/route.js +++ b/web/app/api/integrations/google/disconnect/route.ts @@ -1,6 +1,7 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { writeSecrets, getPublicStatus } from '@/server/googleSecrets'; +import type { ApiRouteHandler } from '@/types/api'; export const runtime = 'nodejs'; @@ -8,7 +9,7 @@ export const runtime = 'nodejs'; * POST /api/integrations/google/disconnect * Clears tokens (refreshToken, serviceAccount) but keeps property IDs. */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; @@ -16,6 +17,7 @@ export async function POST(request) { writeSecrets({ refreshToken: null, serviceAccount: null, authMode: null }); return NextResponse.json({ ok: true, status: getPublicStatus() }); } catch (e) { - return NextResponse.json({ error: e.message }, { status: 500 }); + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/integrations/google/keywords/by-page/route.js b/web/app/api/integrations/google/keywords/by-page/route.js deleted file mode 100644 index c5f48d95..00000000 --- a/web/app/api/integrations/google/keywords/by-page/route.js +++ /dev/null @@ -1,71 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { getReportDbPath } from '@/server/reportSqlite'; -import initSqlJs from 'sql.js'; -import fs from 'fs'; - -export const runtime = 'nodejs'; - -/** - * GET /api/integrations/google/keywords/by-page?url=https://example.com/page - * - * Returns all keyword_data rows for a given page URL, - * plus cannibalisation entries involving that URL. - */ -export async function GET(request) { - const guard = forbiddenIfNotLocal(request); - if (guard) return guard; - - const { searchParams } = new URL(request.url); - const pageUrl = (searchParams.get('url') || '').trim(); - - if (!pageUrl) { - return NextResponse.json({ error: 'url parameter is required' }, { status: 400 }); - } - - const dbPath = getReportDbPath(); - if (!dbPath || !fs.existsSync(dbPath)) { - return NextResponse.json({ keywords: [], cannibalisation: [] }); - } - - try { - const SQL = await initSqlJs(); - const buf = fs.readFileSync(dbPath); - const db = new SQL.Database(new Uint8Array(buf)); - - try { - const res = db.exec('SELECT data FROM keyword_data ORDER BY id DESC LIMIT 1'); - if (!res.length || !res[0].values.length) { - return NextResponse.json({ keywords: [], cannibalisation: [] }); - } - - const data = JSON.parse(res[0].values[0][0]); - const allRows = data.rows || []; - - const normalizedTarget = pageUrl.toLowerCase().replace(/\/$/, ''); - const pageKeywords = allRows.filter((r) => { - const u = (r.gsc_url || '').toLowerCase().replace(/\/$/, ''); - return u === normalizedTarget || u.includes(normalizedTarget) || normalizedTarget.includes(u); - }); - - const cannib = (data.cannibalisation || []).filter((c) => - (c.pages || []).some((p) => { - const u = (p.url || '').toLowerCase().replace(/\/$/, ''); - return u === normalizedTarget; - }), - ); - - return NextResponse.json({ - url: pageUrl, - keyword_count: pageKeywords.length, - keywords: pageKeywords, - cannibalisation: cannib, - fetched_at: data.fetched_at, - }); - } finally { - db.close(); - } - } catch (err) { - return NextResponse.json({ error: String(err) }, { status: 500 }); - } -} diff --git a/web/app/api/integrations/google/keywords/by-page/route.ts b/web/app/api/integrations/google/keywords/by-page/route.ts new file mode 100644 index 00000000..373dff8f --- /dev/null +++ b/web/app/api/integrations/google/keywords/by-page/route.ts @@ -0,0 +1,85 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { withDb } from '@/server/db'; +import type { ApiRouteHandler } from '@/types/api'; +import type { PoolClient } from 'pg'; + +export const runtime = 'nodejs'; + +function parseJsonField(val: unknown): Record | null { + if (val == null) return null; + if (typeof val === 'object' && !Array.isArray(val)) return val as Record; + try { + const parsed: unknown = JSON.parse(String(val)); + if (parsed != null && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } + return null; + } catch { + return null; + } +} + +interface KeywordRow { + gsc_url?: string; + [key: string]: unknown; +} + +interface CannibalisationEntry { + pages?: Array<{ url?: string }>; + [key: string]: unknown; +} + +/** + * GET /api/integrations/google/keywords/by-page?url=https://example.com/page + */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const guard = forbiddenIfNotLocal(request); + if (guard) return guard; + + const { searchParams } = new URL(request.url); + const pageUrl = (searchParams.get('url') || '').trim(); + + if (!pageUrl) { + return NextResponse.json({ error: 'url parameter is required' }, { status: 400 }); + } + + try { + return await withDb(async (client: PoolClient) => { + const { rows } = await client.query( + 'SELECT data FROM keyword_data ORDER BY id DESC LIMIT 1', + ); + if (!rows.length) { + return NextResponse.json({ keywords: [], cannibalisation: [] }); + } + + const data = parseJsonField(rows[0].data) || {}; + const allRows = Array.isArray(data.rows) ? (data.rows as KeywordRow[]) : []; + + const normalizedTarget = pageUrl.toLowerCase().replace(/\/$/, ''); + const pageKeywords = allRows.filter((r) => { + const u = (r.gsc_url || '').toLowerCase().replace(/\/$/, ''); + return u === normalizedTarget || u.includes(normalizedTarget) || normalizedTarget.includes(u); + }); + + const cannibRaw = Array.isArray(data.cannibalisation) ? data.cannibalisation : []; + const cannib = (cannibRaw as CannibalisationEntry[]).filter((c) => + (c.pages || []).some((p) => { + const u = (p.url || '').toLowerCase().replace(/\/$/, ''); + return u === normalizedTarget; + }), + ); + + return NextResponse.json({ + url: pageUrl, + keyword_count: pageKeywords.length, + keywords: pageKeywords, + cannibalisation: cannib, + fetched_at: data.fetched_at, + }); + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/integrations/google/keywords/expand/route.js b/web/app/api/integrations/google/keywords/expand/route.ts similarity index 60% rename from web/app/api/integrations/google/keywords/expand/route.js rename to web/app/api/integrations/google/keywords/expand/route.ts index 726b0ad2..fb4363b5 100644 --- a/web/app/api/integrations/google/keywords/expand/route.js +++ b/web/app/api/integrations/google/keywords/expand/route.ts @@ -1,46 +1,45 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { spawn } from 'child_process'; import path from 'path'; import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; +import type { ApiRouteHandler, KeywordExpandPostBody } from '@/types/api'; export const runtime = 'nodejs'; const WEB_CWD = process.cwd(); const DEFAULT_REPO_ROOT = process.env.WEBSITE_PROFILING_ROOT || path.resolve(WEB_CWD, '..'); -const DEFAULT_PYTHON = process.env.PYTHON || 'python'; /** * POST /api/integrations/google/keywords/expand * Body: { seeds: string[], sources?: string[] } - * - * Spawns Python to run Google Suggest expansion and returns a live preview. - * Used by the bulk seed textarea in the Keywords Explorer UI. */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const guard = forbiddenIfNotLocal(request); if (guard) return guard; - let body; + let body: KeywordExpandPostBody; try { - body = await request.json(); + body = (await request.json()) as KeywordExpandPostBody; } catch { return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); } const seeds = Array.isArray(body?.seeds) - ? body.seeds.filter((s) => typeof s === 'string' && s.trim()).slice(0, 30) + ? body.seeds.filter((s): s is string => typeof s === 'string' && Boolean(s.trim())).slice(0, 30) : []; if (seeds.length === 0) { return NextResponse.json({ error: 'No seeds provided' }, { status: 400 }); } - const sources = Array.isArray(body?.sources) ? body.sources : ['web', 'youtube', 'questions']; + const sources = Array.isArray(body?.sources) + ? body.sources.filter((s): s is string => typeof s === 'string') + : ['web', 'youtube', 'questions']; const repoRoot = DEFAULT_REPO_ROOT; - const pythonExe = String(process.env.PYTHON || DEFAULT_PYTHON).trim() || DEFAULT_PYTHON; + const pythonExe = resolvePythonExecutable(null, repoRoot); - // Build inline Python that imports suggest.py and returns JSON const pyScript = [ 'import json, sys', "sys.path.insert(0, '.')", @@ -51,7 +50,7 @@ export async function POST(request) { 'print(json.dumps(result, ensure_ascii=False))', ].join('\n'); - return new Promise((resolve) => { + return new Promise((resolve) => { const proc = spawn(pythonExe, ['-c', pyScript], { cwd: repoRoot, env: { ...process.env }, @@ -60,30 +59,33 @@ export async function POST(request) { let stdout = ''; let stderr = ''; - proc.stdout?.on('data', (d) => (stdout += d.toString())); - proc.stderr?.on('data', (d) => (stderr += d.toString())); + proc.stdout?.on('data', (d: Buffer | string) => { stdout += d.toString(); }); + proc.stderr?.on('data', (d: Buffer | string) => { stderr += d.toString(); }); - proc.on('error', (err) => { - resolve(NextResponse.json({ error: err.message }, { status: 500 })); + proc.on('error', (err: Error) => { + resolve( + NextResponse.json({ error: formatPythonSpawnError(err, pythonExe, repoRoot) }, { status: 500 }), + ); }); const timer = setTimeout(() => { - try { proc.kill(); } catch {} + try { proc.kill(); } catch { /* ignore */ } resolve(NextResponse.json({ error: 'Suggest expansion timed out (45s)' }, { status: 504 })); }, 45_000); - proc.on('close', (code) => { + proc.on('close', (code: number | null) => { clearTimeout(timer); if (code !== 0) { - return resolve( + resolve( NextResponse.json( { error: 'Python error', detail: stderr.slice(0, 500) }, { status: 500 }, ), ); + return; } try { - const result = JSON.parse(stdout.trim()); + const result: unknown = JSON.parse(stdout.trim()); resolve(NextResponse.json({ results: result })); } catch { resolve( @@ -95,4 +97,4 @@ export async function POST(request) { } }); }); -} +}; diff --git a/web/app/api/integrations/google/keywords/history/batch/route.js b/web/app/api/integrations/google/keywords/history/batch/route.js deleted file mode 100644 index f528bec2..00000000 --- a/web/app/api/integrations/google/keywords/history/batch/route.js +++ /dev/null @@ -1,105 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { getReportDbPath } from '@/server/reportSqlite'; -import initSqlJs from 'sql.js'; -import fs from 'fs'; - -export const runtime = 'nodejs'; - -const MAX_KEYWORDS = 100; -const MAX_LIMIT_PER_KEYWORD = 90; - -/** - * POST /api/integrations/google/keywords/history/batch - * Body: { keywords: string[], limit?: number } - * Returns: { histories: { [keyword]: HistoryPoint[] } } - */ -export async function POST(request) { - const guard = forbiddenIfNotLocal(request); - if (guard) return guard; - - let body; - try { - body = await request.json(); - } catch { - return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); - } - - const rawKeywords = Array.isArray(body.keywords) ? body.keywords : []; - const keywords = [...new Set(rawKeywords.map((k) => String(k || '').trim()).filter(Boolean))].slice( - 0, - MAX_KEYWORDS, - ); - const limit = Math.min( - Math.max(parseInt(String(body.limit ?? '30'), 10) || 30, 1), - MAX_LIMIT_PER_KEYWORD, - ); - - if (!keywords.length) { - return NextResponse.json({ histories: {} }); - } - - const dbPath = getReportDbPath(); - if (!dbPath || !fs.existsSync(dbPath)) { - return NextResponse.json({ histories: {} }); - } - - try { - const SQL = await initSqlJs(); - const buf = fs.readFileSync(dbPath); - const db = new SQL.Database(new Uint8Array(buf)); - - try { - const histories = {}; - for (const keyword of keywords) { - histories[keyword] = []; - } - - try { - const placeholders = keywords.map(() => '?').join(', '); - const res = db.exec( - `SELECT keyword, fetched_at, position, clicks, impressions, ctr - FROM keyword_history - WHERE keyword IN (${placeholders}) - ORDER BY keyword, id DESC`, - keywords, - ); - - if (res.length && res[0].values.length) { - const cols = res[0].columns; - const kwIdx = cols.indexOf('keyword'); - const buckets = {}; - for (const kw of keywords) { - buckets[kw] = []; - } - - for (const vals of res[0].values) { - const row = Object.fromEntries(cols.map((c, i) => [c, vals[i]])); - const kw = String(row.keyword ?? ''); - if (!buckets[kw]) continue; - if (buckets[kw].length >= limit) continue; - buckets[kw].push({ - fetched_at: row.fetched_at, - position: row.position, - clicks: row.clicks, - impressions: row.impressions, - ctr: row.ctr, - }); - } - - for (const kw of keywords) { - histories[kw] = (buckets[kw] || []).reverse(); - } - } - } catch { - // keyword_history table may not exist yet - } - - return NextResponse.json({ histories }); - } finally { - db.close(); - } - } catch (err) { - return NextResponse.json({ error: String(err) }, { status: 500 }); - } -} diff --git a/web/app/api/integrations/google/keywords/history/batch/route.ts b/web/app/api/integrations/google/keywords/history/batch/route.ts new file mode 100644 index 00000000..cbd24239 --- /dev/null +++ b/web/app/api/integrations/google/keywords/history/batch/route.ts @@ -0,0 +1,88 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { withDb } from '@/server/db'; +import type { ApiRouteHandler, KeywordHistoryBatchBody, KeywordHistoryRow } from '@/types/api'; +import type { PoolClient } from 'pg'; + +export const runtime = 'nodejs'; + +const MAX_KEYWORDS = 100; +const MAX_LIMIT_PER_KEYWORD = 90; + +/** + * POST /api/integrations/google/keywords/history/batch + * Body: { keywords: string[], limit?: number } + */ +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { + const guard = forbiddenIfNotLocal(request); + if (guard) return guard; + + let body: KeywordHistoryBatchBody; + try { + body = (await request.json()) as KeywordHistoryBatchBody; + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const rawKeywords: unknown[] = Array.isArray(body?.keywords) ? body.keywords : []; + const keywords = Array.from( + new Set( + rawKeywords + .map((k) => String(k || '').trim()) + .filter((k): k is string => Boolean(k)), + ), + ).slice(0, MAX_KEYWORDS); + const limit = Math.min( + Math.max(parseInt(String(body.limit ?? '30'), 10) || 30, 1), + MAX_LIMIT_PER_KEYWORD, + ); + + if (!keywords.length) { + return NextResponse.json({ histories: {} }); + } + + try { + return await withDb(async (client: PoolClient) => { + const histories: Record = Object.fromEntries( + keywords.map((k) => [k, []]), + ); + + try { + const res = await client.query( + `SELECT keyword, fetched_at, position, clicks, impressions, ctr + FROM keyword_history + WHERE keyword = ANY($1::text[]) + ORDER BY keyword, id DESC`, + [keywords], + ); + + const buckets: Record = Object.fromEntries( + keywords.map((k) => [k, []]), + ); + for (const row of res.rows) { + const kw = String(row.keyword ?? ''); + if (!buckets[kw]) continue; + if (buckets[kw].length >= limit) continue; + buckets[kw].push({ + fetched_at: row.fetched_at != null ? String(row.fetched_at) : null, + position: row.position != null ? Number(row.position) : null, + clicks: row.clicks != null ? Number(row.clicks) : null, + impressions: row.impressions != null ? Number(row.impressions) : null, + ctr: row.ctr != null ? Number(row.ctr) : null, + }); + } + + for (const kw of keywords) { + histories[kw] = (buckets[kw] || []).reverse(); + } + } catch { + /* keyword_history table may not exist yet */ + } + + return NextResponse.json({ histories }); + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/integrations/google/keywords/history/route.js b/web/app/api/integrations/google/keywords/history/route.js deleted file mode 100644 index b01ff829..00000000 --- a/web/app/api/integrations/google/keywords/history/route.js +++ /dev/null @@ -1,66 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { getReportDbPath } from '@/server/reportSqlite'; -import initSqlJs from 'sql.js'; -import fs from 'fs'; - -export const runtime = 'nodejs'; - -/** - * GET /api/integrations/google/keywords/history?keyword=seo+audit&limit=30 - * - * Returns position history for a single keyword (for sparkline rendering). - */ -export async function GET(request) { - const guard = forbiddenIfNotLocal(request); - if (guard) return guard; - - const { searchParams } = new URL(request.url); - const keyword = (searchParams.get('keyword') || '').trim(); - const limit = Math.min(parseInt(searchParams.get('limit') || '30', 10), 90); - - if (!keyword) { - return NextResponse.json({ error: 'keyword parameter is required' }, { status: 400 }); - } - - const dbPath = getReportDbPath(); - if (!dbPath || !fs.existsSync(dbPath)) { - return NextResponse.json({ keyword, history: [] }); - } - - try { - const SQL = await initSqlJs(); - const buf = fs.readFileSync(dbPath); - const db = new SQL.Database(new Uint8Array(buf)); - - try { - // keyword_history table may not exist yet if no enrichment has run - let rows = []; - try { - const res = db.exec( - `SELECT fetched_at, position, clicks, impressions, ctr - FROM keyword_history - WHERE keyword = ? - ORDER BY id DESC - LIMIT ${limit}`, - [keyword], - ); - if (res.length && res[0].values.length) { - const cols = res[0].columns; - rows = res[0].values.map((vals) => - Object.fromEntries(cols.map((c, i) => [c, vals[i]])), - ); - rows.reverse(); // oldest first for chart rendering - } - } catch { - // Table doesn't exist yet - } - - return NextResponse.json({ keyword, history: rows }); - } finally { - db.close(); - } - } catch (err) { - return NextResponse.json({ error: String(err) }, { status: 500 }); - } -} diff --git a/web/app/api/integrations/google/keywords/history/route.ts b/web/app/api/integrations/google/keywords/history/route.ts new file mode 100644 index 00000000..a80d9d0f --- /dev/null +++ b/web/app/api/integrations/google/keywords/history/route.ts @@ -0,0 +1,48 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { withDb } from '@/server/db'; +import type { ApiRouteHandler } from '@/types/api'; +import type { KeywordHistoryRow } from '@/types/api'; +import type { PoolClient } from 'pg'; + +export const runtime = 'nodejs'; + +/** + * GET /api/integrations/google/keywords/history?keyword=seo+audit&limit=30 + */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const guard = forbiddenIfNotLocal(request); + if (guard) return guard; + + const { searchParams } = new URL(request.url); + const keyword = (searchParams.get('keyword') || '').trim(); + const limit = Math.min(parseInt(searchParams.get('limit') || '30', 10), 90); + + if (!keyword) { + return NextResponse.json({ error: 'keyword parameter is required' }, { status: 400 }); + } + + try { + return await withDb(async (client: PoolClient) => { + let rows: KeywordHistoryRow[] = []; + try { + const res = await client.query( + `SELECT fetched_at, position, clicks, impressions, ctr + FROM keyword_history + WHERE keyword = $1 + ORDER BY id DESC + LIMIT $2`, + [keyword, limit], + ); + rows = res.rows.reverse() as KeywordHistoryRow[]; + } catch { + /* table may not exist yet */ + } + + return NextResponse.json({ keyword, history: rows }); + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/integrations/google/page-data/route.js b/web/app/api/integrations/google/page-data/route.js deleted file mode 100644 index 9889550c..00000000 --- a/web/app/api/integrations/google/page-data/route.js +++ /dev/null @@ -1,59 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { getReportDbPath } from '@/server/reportSqlite'; -import initSqlJs from 'sql.js'; -import fs from 'fs'; - -export const runtime = 'nodejs'; - -/** - * GET /api/integrations/google/page-data?url=https://example.com/path - * Lazy-loads per-URL Google metrics from google_data SQLite table. - * Used by Link Explorer to avoid bloating the initial payload. - */ -export async function GET(request) { - const denied = forbiddenIfNotLocal(request); - if (denied) return denied; - - const url = request.nextUrl.searchParams.get('url'); - if (!url) { - return NextResponse.json({ error: 'url parameter required' }, { status: 400 }); - } - - const dbPath = getReportDbPath(); - if (!dbPath || !fs.existsSync(dbPath)) { - return NextResponse.json({ gsc: null, ga4: null }); - } - - try { - const SQL = await initSqlJs(); - const buf = fs.readFileSync(dbPath); - const db = new SQL.Database(new Uint8Array(buf)); - try { - const res = db.exec( - 'SELECT data FROM google_data ORDER BY id DESC LIMIT 1' - ); - if (!res.length || !res[0].values.length) { - return NextResponse.json({ gsc: null, ga4: null }); - } - const raw = JSON.parse(res[0].values[0][0]); - const byPage = raw?.gsc?.by_page || {}; - const byPath = raw?.ga4?.by_path || {}; - - // Normalize URL to path for GA4 lookup - let urlPath = url; - try { - urlPath = new URL(url).pathname; - } catch {} - - return NextResponse.json({ - gsc: byPage[url] || null, - ga4: byPath[urlPath] || byPath[url] || null, - }); - } finally { - db.close(); - } - } catch (e) { - return NextResponse.json({ error: e.message }, { status: 500 }); - } -} diff --git a/web/app/api/integrations/google/page-data/route.ts b/web/app/api/integrations/google/page-data/route.ts new file mode 100644 index 00000000..7a927630 --- /dev/null +++ b/web/app/api/integrations/google/page-data/route.ts @@ -0,0 +1,63 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { withDb } from '@/server/db'; +import type { ApiRouteHandler } from '@/types/api'; +import type { PoolClient } from 'pg'; + +export const runtime = 'nodejs'; + +function parseJsonField(val: unknown): Record | null { + if (val == null) return null; + if (typeof val === 'object' && !Array.isArray(val)) return val as Record; + try { + const parsed: unknown = JSON.parse(String(val)); + if (parsed != null && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } + return null; + } catch { + return null; + } +} + +/** + * GET /api/integrations/google/page-data?url=https://example.com/path + */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + const url = request.nextUrl.searchParams.get('url'); + if (!url) { + return NextResponse.json({ error: 'url parameter required' }, { status: 400 }); + } + + try { + return await withDb(async (client: PoolClient) => { + const { rows } = await client.query( + 'SELECT data FROM google_data ORDER BY id DESC LIMIT 1', + ); + if (!rows.length) { + return NextResponse.json({ gsc: null, ga4: null }); + } + const raw = parseJsonField(rows[0].data); + const gsc = raw?.gsc as Record | undefined; + const ga4 = raw?.ga4 as Record | undefined; + const byPage = (gsc?.by_page as Record | undefined) || {}; + const byPath = (ga4?.by_path as Record | undefined) || {}; + + let urlPath = url; + try { + urlPath = new URL(url).pathname; + } catch { /* keep original url */ } + + return NextResponse.json({ + gsc: byPage[url] ?? null, + ga4: byPath[urlPath] ?? byPath[url] ?? null, + }); + }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/integrations/google/properties/route.js b/web/app/api/integrations/google/properties/route.ts similarity index 55% rename from web/app/api/integrations/google/properties/route.js rename to web/app/api/integrations/google/properties/route.ts index 3b91e976..88ddd974 100644 --- a/web/app/api/integrations/google/properties/route.js +++ b/web/app/api/integrations/google/properties/route.ts @@ -1,65 +1,65 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { spawn } from 'child_process'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { getPipelineSpawnEnv, getRepoRoot } from '@/server/pipelineSpawnEnv'; +import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; +import type { ApiRouteHandler } from '@/types/api'; export const runtime = 'nodejs'; -const DEFAULT_PYTHON = process.env.PYTHON || 'python'; - /** * GET /api/integrations/google/properties - * Spawns `python -m src google --list-properties` (config from report.db). + * Spawns `python -m src google --list-properties` (config from PostgreSQL). */ -export async function GET(request) { +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; const repoRoot = getRepoRoot(); - const pythonExe = String(process.env.PYTHON || DEFAULT_PYTHON).trim() || DEFAULT_PYTHON; + const pythonExe = resolvePythonExecutable(null, repoRoot); - return new Promise((resolve) => { + return new Promise((resolve) => { let stdout = ''; let stderr = ''; const proc = spawn( pythonExe, ['-m', 'src', 'google', '--list-properties'], - { cwd: repoRoot, env: getPipelineSpawnEnv(), shell: false } + { cwd: repoRoot, env: getPipelineSpawnEnv(), shell: false }, ); - proc.stdout?.on('data', (c) => { stdout += c.toString(); }); - proc.stderr?.on('data', (c) => { stderr += c.toString(); }); + proc.stdout?.on('data', (c: Buffer | string) => { stdout += c.toString(); }); + proc.stderr?.on('data', (c: Buffer | string) => { stderr += c.toString(); }); - proc.on('error', (err) => { - resolve(NextResponse.json({ error: err.message }, { status: 500 })); + proc.on('error', (err: Error) => { + resolve(NextResponse.json({ error: formatPythonSpawnError(err, pythonExe, repoRoot) }, { status: 500 })); }); - proc.on('close', (code) => { + proc.on('close', (code: number | null) => { if (code !== 0) { resolve( NextResponse.json( { error: stderr.trim() || 'Failed to list properties', exitCode: code }, - { status: 500 } - ) + { status: 500 }, + ), ); return; } try { - const data = JSON.parse(stdout.trim()); + const data: unknown = JSON.parse(stdout.trim()); resolve(NextResponse.json(data)); } catch { resolve( NextResponse.json( { error: 'Could not parse properties response from Python' }, - { status: 500 } - ) + { status: 500 }, + ), ); } }); setTimeout(() => { - try { proc.kill(); } catch {} + try { proc.kill(); } catch { /* ignore */ } resolve(NextResponse.json({ error: 'Timed out listing properties' }, { status: 504 })); }, 30_000); }); -} +}; diff --git a/web/app/api/integrations/google/status/route.js b/web/app/api/integrations/google/status/route.js deleted file mode 100644 index af8abb1f..00000000 --- a/web/app/api/integrations/google/status/route.js +++ /dev/null @@ -1,41 +0,0 @@ -import { NextResponse } from 'next/server'; -import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { getPublicStatus } from '@/server/googleSecrets'; -import { getReportDbPath } from '@/server/reportSqlite'; -import initSqlJs from 'sql.js'; -import fs from 'fs'; - -export const runtime = 'nodejs'; - -async function getLastFetchedAt() { - const dbPath = getReportDbPath(); - if (!dbPath || !fs.existsSync(dbPath)) return null; - try { - const SQL = await initSqlJs(); - const buf = fs.readFileSync(dbPath); - const db = new SQL.Database(new Uint8Array(buf)); - try { - const res = db.exec( - 'SELECT fetched_at FROM google_data ORDER BY id DESC LIMIT 1' - ); - if (res.length > 0 && res[0].values.length > 0) { - return res[0].values[0][0]; - } - return null; - } finally { - db.close(); - } - } catch { - return null; - } -} - -export async function GET(request) { - const denied = forbiddenIfNotLocal(request); - if (denied) return denied; - - const status = getPublicStatus(); - const lastFetchedAt = await getLastFetchedAt(); - - return NextResponse.json({ ...status, lastFetchedAt }); -} diff --git a/web/app/api/integrations/google/status/route.ts b/web/app/api/integrations/google/status/route.ts new file mode 100644 index 00000000..e38b654c --- /dev/null +++ b/web/app/api/integrations/google/status/route.ts @@ -0,0 +1,31 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { getPublicStatus } from '@/server/googleSecrets'; +import { withDb } from '@/server/db'; +import type { ApiRouteHandler } from '@/types/api'; +import type { PoolClient } from 'pg'; + +export const runtime = 'nodejs'; + +async function getLastFetchedAt(): Promise { + try { + return await withDb(async (client: PoolClient) => { + const { rows } = await client.query( + 'SELECT fetched_at FROM google_data ORDER BY id DESC LIMIT 1', + ); + return rows.length ? String(rows[0].fetched_at) : null; + }); + } catch { + return null; + } +} + +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + const status = getPublicStatus(); + const lastFetchedAt = await getLastFetchedAt(); + + return NextResponse.json({ ...status, lastFetchedAt }); +}; diff --git a/web/app/api/integrations/google/test/route.js b/web/app/api/integrations/google/test/route.ts similarity index 53% rename from web/app/api/integrations/google/test/route.js rename to web/app/api/integrations/google/test/route.ts index d1be0f41..e5bd3a2a 100644 --- a/web/app/api/integrations/google/test/route.js +++ b/web/app/api/integrations/google/test/route.ts @@ -1,24 +1,24 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { spawn } from 'child_process'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { getPipelineSpawnEnv, getRepoRoot } from '@/server/pipelineSpawnEnv'; +import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; +import type { ApiRouteHandler } from '@/types/api'; export const runtime = 'nodejs'; -const DEFAULT_PYTHON = process.env.PYTHON || 'python'; - /** * POST /api/integrations/google/test - * Spawns `python -m src google --test` (config from report.db pipeline_config). + * Spawns `python -m src google --test` (config from PostgreSQL pipeline_config). */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; const repoRoot = getRepoRoot(); - const pythonExe = String(process.env.PYTHON || DEFAULT_PYTHON).trim() || DEFAULT_PYTHON; + const pythonExe = resolvePythonExecutable(null, repoRoot); - return new Promise((resolve) => { + return new Promise((resolve) => { let log = ''; const proc = spawn(pythonExe, ['-m', 'src', 'google', '--test'], { cwd: repoRoot, @@ -26,29 +26,30 @@ export async function POST(request) { shell: false, }); - const append = (chunk) => { + const append = (chunk: Buffer | string): void => { log += chunk.toString(); if (log.length > 32_000) log = log.slice(-28_000); }; proc.stdout?.on('data', append); proc.stderr?.on('data', append); - proc.on('error', (err) => { + proc.on('error', (err: Error) => { + const message = formatPythonSpawnError(err, pythonExe, repoRoot); resolve( - NextResponse.json({ ok: false, log, error: err.message }, { status: 500 }) + NextResponse.json({ ok: false, log, error: message }, { status: 500 }), ); }); - proc.on('close', (code) => { + proc.on('close', (code: number | null) => { resolve(NextResponse.json({ ok: code === 0, log, exitCode: code })); }); // Safety timeout: 30s setTimeout(() => { - try { proc.kill(); } catch {} + try { proc.kill(); } catch { /* ignore */ } resolve( - NextResponse.json({ ok: false, log, error: 'Test timed out after 30s' }, { status: 504 }) + NextResponse.json({ ok: false, log, error: 'Test timed out after 30s' }, { status: 504 }), ); }, 30_000); }); -} +}; diff --git a/web/app/api/jobs/[id]/route.js b/web/app/api/jobs/[id]/route.ts similarity index 64% rename from web/app/api/jobs/[id]/route.js rename to web/app/api/jobs/[id]/route.ts index 38773e43..f1e372cb 100644 --- a/web/app/api/jobs/[id]/route.js +++ b/web/app/api/jobs/[id]/route.ts @@ -1,9 +1,10 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { getJob } from '@/server/pipelineJobs'; +import type { ApiRouteHandlerWithParams } from '@/types/api'; export const runtime = 'nodejs'; -function forbiddenIfNotLocal(request) { +function forbiddenIfNotLocal(request: NextRequest): NextResponse | null { const host = (request.headers.get('host') || '').split(':')[0]; if (host !== '127.0.0.1' && host !== 'localhost') { return NextResponse.json({ error: 'Only available on localhost' }, { status: 403 }); @@ -11,7 +12,10 @@ function forbiddenIfNotLocal(request) { return null; } -export async function GET(request, { params }) { +export const GET: ApiRouteHandlerWithParams<{ id: string }> = async ( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; const { id } = await params; @@ -25,4 +29,4 @@ export async function GET(request, { params }) { log: job.log, error: job.error ?? null, }); -} +}; diff --git a/web/app/api/llm-config/route.js b/web/app/api/llm-config/route.ts similarity index 72% rename from web/app/api/llm-config/route.js rename to web/app/api/llm-config/route.ts index 15aaabb6..ba085610 100644 --- a/web/app/api/llm-config/route.js +++ b/web/app/api/llm-config/route.ts @@ -1,14 +1,13 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { loadLlmConfig, saveLlmConfig } from '@/server/llmConfig'; import { ALL_LLM_SCHEMA_KEYS, getLlmFieldByKey } from '@/lib/llmConfigSchema'; +import type { ApiRouteHandler, LlmConfigPutBody, LlmConfigState } from '@/types/api'; export const runtime = 'nodejs'; -/** - * GET /api/llm-config — LLM settings from report.db only (secrets masked). - */ -export async function GET(request) { +/** GET /api/llm-config — LLM settings from PostgreSQL only (secrets masked). */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; @@ -19,18 +18,16 @@ export async function GET(request) { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; -/** - * PUT /api/llm-config — Body: { state: Record } - */ -export async function PUT(request) { +/** PUT /api/llm-config — Body: { state: Record } */ +export const PUT: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; - let body; + let body: LlmConfigPutBody; try { - body = await request.json(); + body = (await request.json()) as LlmConfigPutBody; } catch { return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); } @@ -40,7 +37,7 @@ export async function PUT(request) { return NextResponse.json({ error: 'Missing state object' }, { status: 400 }); } - const state = {}; + const state: LlmConfigState = {}; for (const [key, rawValue] of Object.entries(rawState)) { if (!ALL_LLM_SCHEMA_KEYS.has(key)) continue; if (key.endsWith('_masked')) continue; @@ -64,4 +61,4 @@ export async function PUT(request) { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/pipeline-config/route.js b/web/app/api/pipeline-config/route.ts similarity index 63% rename from web/app/api/pipeline-config/route.js rename to web/app/api/pipeline-config/route.ts index 52a50755..b7d147e5 100644 --- a/web/app/api/pipeline-config/route.js +++ b/web/app/api/pipeline-config/route.ts @@ -1,16 +1,31 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; import { loadPipelineConfig, savePipelineConfig } from '@/server/pipelineConfig'; -import { ALL_SCHEMA_KEYS, getFieldByKey } from '@/lib/pipelineConfigSchema'; +import { ALL_SCHEMA_KEYS, getFieldByKey, validateRequiredPipelineFields } from '@/lib/pipelineConfigSchema'; +import type { + ApiRouteHandler, + PipelineConfigPutBody, + PipelineConfigState, + PipelineUnknownKey, +} from '@/types/api'; export const runtime = 'nodejs'; +function isUnknownKeyEntry(value: unknown): value is PipelineUnknownKey { + return ( + value != null && + typeof value === 'object' && + typeof (value as PipelineUnknownKey).key === 'string' && + typeof (value as PipelineUnknownKey).value === 'string' + ); +} + /** * GET /api/pipeline-config * Returns { state, unknownKeys, source, dbPath }. * Localhost-only. */ -export async function GET(request) { +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; @@ -21,23 +36,19 @@ export async function GET(request) { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; /** * PUT /api/pipeline-config * Body: { state: Record, unknownKeys?: Array<{key,value}> } - * Validates + coerces per field type, saves to report.db pipeline_config table - * and writes shadow pipeline-config.txt. - * Returns { ok: true, dbPath }. - * Localhost-only. */ -export async function PUT(request) { +export const PUT: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; - let body; + let body: PipelineConfigPutBody; try { - body = await request.json(); + body = (await request.json()) as PipelineConfigPutBody; } catch { return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); } @@ -47,9 +58,7 @@ export async function PUT(request) { return NextResponse.json({ error: 'Missing state object' }, { status: 400 }); } - // Coerce each key to its declared type; ignore keys not in schema. - // LLM keys are UI-only (llm_config table) — never persist via pipeline-config. - const state = {}; + const state: PipelineConfigState = {}; for (const [key, rawValue] of Object.entries(rawState)) { if (key.startsWith('llm_')) continue; if (!ALL_SCHEMA_KEYS.has(key)) continue; @@ -68,23 +77,24 @@ export async function PUT(request) { } } - // Validate unknownKeys shape; drop llm_* (UI-only) and legacy ml_* keys - const safeUnknownKeys = Array.isArray(unknownKeys) + const safeUnknownKeys: PipelineUnknownKey[] = Array.isArray(unknownKeys) ? unknownKeys.filter( (u) => - u && - typeof u.key === 'string' && - typeof u.value === 'string' && + isUnknownKeyEntry(u) && !u.key.startsWith('llm_') && !u.key.startsWith('ml_'), ) : []; + const requiredErrors = validateRequiredPipelineFields(state); + if (requiredErrors.length > 0) { + return NextResponse.json({ error: requiredErrors.join(' ') }, { status: 400 }); + } + try { - const dbPath = await savePipelineConfig(state, { unknownKeys: safeUnknownKeys }); return NextResponse.json({ ok: true, dbPath }); } catch (e) { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/report-db/route.js b/web/app/api/report-db/route.js deleted file mode 100644 index 6e0043b6..00000000 --- a/web/app/api/report-db/route.js +++ /dev/null @@ -1,29 +0,0 @@ -import { NextResponse } from 'next/server'; -import fs from 'fs'; -import path from 'path'; - -const REPO_ROOT = process.env.WEBSITE_PROFILING_ROOT || path.resolve(process.cwd(), '..'); - -function forbiddenIfNotLocal(request) { - const host = (request.headers.get('host') || '').split(':')[0]; - if (host !== '127.0.0.1' && host !== 'localhost') { - return NextResponse.json({ error: 'Only available on localhost' }, { status: 403 }); - } - return null; -} - -export async function GET(request) { - const denied = forbiddenIfNotLocal(request); - if (denied) return denied; - const dbPath = process.env.REPORT_DB_PATH || path.join(REPO_ROOT, 'report.db'); - if (!fs.existsSync(dbPath)) { - return NextResponse.json({ error: 'report.db not found' }, { status: 404 }); - } - const buf = fs.readFileSync(dbPath); - return new NextResponse(buf, { - headers: { - 'Content-Type': 'application/x-sqlite3', - 'Cache-Control': 'no-store', - }, - }); -} diff --git a/web/app/api/report/crawl-payload/route.ts b/web/app/api/report/crawl-payload/route.ts new file mode 100644 index 00000000..2b248f26 --- /dev/null +++ b/web/app/api/report/crawl-payload/route.ts @@ -0,0 +1,21 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { getCrawlPreviewPayload } from '@/server/reportDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const dynamic = 'force-dynamic'; + +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const raw = request.nextUrl.searchParams.get('crawlRunId'); + const crawlRunId = raw != null && raw !== '' ? Number(raw) : null; + if (crawlRunId == null || !Number.isFinite(crawlRunId) || crawlRunId <= 0) { + return NextResponse.json({ error: 'Invalid crawlRunId' }, { status: 400 }); + } + try { + const payload = await getCrawlPreviewPayload(crawlRunId); + return NextResponse.json({ payload }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + const status = msg === 'Crawl run not found' ? 404 : 500; + return NextResponse.json({ error: msg }, { status }); + } +}; diff --git a/web/app/api/report/meta/route.js b/web/app/api/report/meta/route.ts similarity index 64% rename from web/app/api/report/meta/route.js rename to web/app/api/report/meta/route.ts index 9f9be1d9..90512435 100644 --- a/web/app/api/report/meta/route.js +++ b/web/app/api/report/meta/route.ts @@ -1,9 +1,10 @@ import { NextResponse } from 'next/server'; -import { getReportMeta } from '@/server/reportSqlite'; +import { getReportMeta } from '@/server/reportDb'; +import type { ApiRouteHandler } from '@/types/api'; export const dynamic = 'force-dynamic'; -export async function GET() { +export const GET: ApiRouteHandler = async (): Promise => { try { const data = await getReportMeta(); return NextResponse.json(data); @@ -11,4 +12,4 @@ export async function GET() { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 500 }); } -} +}; diff --git a/web/app/api/report/payload/route.js b/web/app/api/report/payload/route.ts similarity index 74% rename from web/app/api/report/payload/route.js rename to web/app/api/report/payload/route.ts index 56fe9e61..1a9b1f0f 100644 --- a/web/app/api/report/payload/route.js +++ b/web/app/api/report/payload/route.ts @@ -1,10 +1,11 @@ -import { NextResponse } from 'next/server'; -import { withReportDb } from '@/server/reportSqlite'; +import { NextResponse, type NextRequest } from 'next/server'; +import { withReportDb } from '@/server/reportDb'; import { readReportPayloadFromDatabase } from '@/lib/loadReportDb'; +import type { ApiRouteHandler } from '@/types/api'; export const dynamic = 'force-dynamic'; -export async function GET(request) { +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { const raw = request.nextUrl.searchParams.get('reportId'); const reportId = raw != null && raw !== '' ? Number(raw) : null; if (raw != null && raw !== '' && !Number.isFinite(reportId)) { @@ -18,4 +19,4 @@ export async function GET(request) { const status = msg === 'Report not found' ? 404 : msg.includes('not found') ? 404 : 500; return NextResponse.json({ error: msg }, { status }); } -} +}; diff --git a/web/app/api/report/portfolio/route.js b/web/app/api/report/portfolio/route.js deleted file mode 100644 index 792306fd..00000000 --- a/web/app/api/report/portfolio/route.js +++ /dev/null @@ -1,44 +0,0 @@ -import { NextResponse } from 'next/server'; -import { withReportDb } from '@/server/reportSqlite'; -import { - listReportsFromDatabase, - readReportPayloadFromDatabase, - getCrawlRunsRows, -} from '@/lib/loadReportDb'; -import { computeDomainGroups } from '@/lib/homePortfolio'; -import { strings } from '@/lib/strings'; - -export const dynamic = 'force-dynamic'; - -export async function GET(request) { - const idsParam = request.nextUrl.searchParams.get('ids'); - const ids = idsParam - ? idsParam - .split(',') - .map((s) => Number(String(s).trim())) - .filter((n) => Number.isFinite(n) && n > 0) - : []; - - try { - const groups = await withReportDb((db) => { - const all = listReportsFromDatabase(db); - const idSet = new Set(ids); - const reportList = ids.length ? all.filter((r) => idSet.has(r.id)) : all; - const crawlRows = getCrawlRunsRows(db); - const startUrlByRunId = new Map(crawlRows.map((cr) => [cr.id, cr.start_url])); - const runCreatedAtByRunId = new Map(crawlRows.map((cr) => [cr.id, cr.created_at])); - return computeDomainGroups( - reportList, - startUrlByRunId, - runCreatedAtByRunId, - strings.views.home.unknownBrand, - strings.common.emDash, - (id) => readReportPayloadFromDatabase(db, id) - ); - }); - return NextResponse.json({ groups }); - } catch (e) { - const msg = e instanceof Error ? e.message : String(e); - return NextResponse.json({ error: msg, groups: [] }, { status: 500 }); - } -} diff --git a/web/app/api/report/portfolio/route.ts b/web/app/api/report/portfolio/route.ts new file mode 100644 index 00000000..e4d8dab6 --- /dev/null +++ b/web/app/api/report/portfolio/route.ts @@ -0,0 +1,61 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { withReportDb } from '@/server/reportDb'; +import { + listReportsFromDatabase, + readReportPayloadFromDatabase, + getCrawlRunsRows, + getCrawlRunSummaries, +} from '@/lib/loadReportDb'; +import { + computeDomainGroups, + computeCrawlOnlyGroups, + mergePortfolioGroups, +} from '@/lib/homePortfolio'; +import { strings } from '@/lib/strings'; +import type { ApiRouteHandler } from '@/types/api'; +import type { StringsCatalog } from '@/types/strings'; + +export const dynamic = 'force-dynamic'; + +const catalog = strings as StringsCatalog; + +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const idsParam = request.nextUrl.searchParams.get('ids'); + const ids = idsParam + ? idsParam + .split(',') + .map((s: string) => Number(String(s).trim())) + .filter((n: number) => Number.isFinite(n) && n > 0) + : []; + + try { + const groups = await withReportDb(async (client) => { + const all = await listReportsFromDatabase(client); + const idSet = new Set(ids); + const reportList = ids.length ? all.filter((r) => idSet.has(r.id)) : all; + const crawlRows = await getCrawlRunsRows(client); + const startUrlByRunId = new Map(crawlRows.map((cr) => [cr.id, cr.start_url])); + const runCreatedAtByRunId = new Map(crawlRows.map((cr) => [cr.id, cr.created_at])); + const reportGroups = await computeDomainGroups( + reportList, + startUrlByRunId, + runCreatedAtByRunId, + catalog.views.home.unknownBrand, + catalog.common.emDash, + (id: number) => readReportPayloadFromDatabase(client, id), + ); + const crawlSummaries = await getCrawlRunSummaries(client); + const crawlOnlyGroups = computeCrawlOnlyGroups( + crawlSummaries, + reportGroups, + catalog.views.home.unknownBrand, + catalog.common.emDash, + ); + return mergePortfolioGroups(reportGroups, crawlOnlyGroups); + }); + return NextResponse.json({ groups }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg, groups: [] }, { status: 500 }); + } +}; diff --git a/web/app/api/run/route.js b/web/app/api/run/route.ts similarity index 76% rename from web/app/api/run/route.js rename to web/app/api/run/route.ts index 8a4d390a..f559f229 100644 --- a/web/app/api/run/route.js +++ b/web/app/api/run/route.ts @@ -1,13 +1,20 @@ -import { NextResponse } from 'next/server'; +import { NextResponse, type NextRequest } from 'next/server'; import { startPipelineJob } from '@/server/pipelineJobs'; import { loadPipelineConfig, savePipelineConfig } from '@/server/pipelineConfig'; import { saveLlmConfig } from '@/server/llmConfig'; import { ALL_LLM_SCHEMA_KEYS, getLlmFieldByKey } from '@/lib/llmConfigSchema'; import { ALL_SCHEMA_KEYS, getFieldByKey, validatePipelineRun } from '@/lib/pipelineConfigSchema'; +import type { + ApiRouteHandler, + LlmConfigState, + PipelineConfigState, + PipelineUnknownKey, + RunPostBody, +} from '@/types/api'; export const runtime = 'nodejs'; -function forbiddenIfNotLocal(request) { +function forbiddenIfNotLocal(request: NextRequest): NextResponse | null { const host = (request.headers.get('host') || '').split(':')[0]; if (host !== '127.0.0.1' && host !== 'localhost') { return NextResponse.json({ error: 'Only available on localhost' }, { status: 403 }); @@ -15,21 +22,30 @@ function forbiddenIfNotLocal(request) { return null; } +function isUnknownKeyEntry(value: unknown): value is PipelineUnknownKey { + return ( + value != null && + typeof value === 'object' && + typeof (value as PipelineUnknownKey).key === 'string' && + typeof (value as PipelineUnknownKey).value === 'string' + ); +} + /** * POST /api/run * Body: { command?: string, state: Record, * unknownKeys?: Array<{key,value}>, python?: string, repoRoot?: string } * - * Saves state to report.db (pipeline_config table) + shadow pipeline-config.txt, - * then spawns `python -m src` (Python reads config from DB via REPORT_DB_PATH). + * Saves state to PostgreSQL (pipeline_config table) + shadow pipeline-config.txt, + * then spawns `python -m src` (Python reads config via DATABASE_URL). */ -export async function POST(request) { +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { const denied = forbiddenIfNotLocal(request); if (denied) return denied; - let body; + let body: RunPostBody; try { - body = await request.json().catch(() => ({})); + body = (await request.json().catch(() => ({}))) as RunPostBody; } catch { body = {}; } @@ -56,7 +72,7 @@ export async function POST(request) { } // Coerce state per field type - const state = {}; + const state: PipelineConfigState = {}; for (const [key, rawValue] of Object.entries(resolvedState)) { if (key.startsWith('llm_')) continue; if (!ALL_SCHEMA_KEYS.has(key)) continue; @@ -75,12 +91,10 @@ export async function POST(request) { } } - const safeUnknownKeys = Array.isArray(resolvedUnknownKeys) + const safeUnknownKeys: PipelineUnknownKey[] = Array.isArray(resolvedUnknownKeys) ? resolvedUnknownKeys.filter( (u) => - u && - typeof u.key === 'string' && - typeof u.value === 'string' && + isUnknownKeyEntry(u) && !u.key.startsWith('llm_') && !u.key.startsWith('ml_'), ) @@ -99,7 +113,7 @@ export async function POST(request) { } if (rawLlmState && typeof rawLlmState === 'object') { - const llmCoerced = {}; + const llmCoerced: LlmConfigState = {}; for (const [key, rawValue] of Object.entries(rawLlmState)) { if (!ALL_LLM_SCHEMA_KEYS.has(key)) continue; if (key.endsWith('_masked')) continue; @@ -123,11 +137,11 @@ export async function POST(request) { } try { - // No config path needed — Python reads from DB via REPORT_DB_PATH env. - const id = startPipelineJob(command, null, { python, repoRoot }); + // No config path needed — Python reads from PostgreSQL via DATABASE_URL. + const id = startPipelineJob(command ?? null, null, { python, repoRoot }); return NextResponse.json({ jobId: id }); } catch (e) { const msg = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: msg }, { status: 400 }); } -} +}; diff --git a/web/app/client-providers.jsx b/web/app/client-providers.tsx similarity index 51% rename from web/app/client-providers.jsx rename to web/app/client-providers.tsx index dddb2875..61ba579b 100644 --- a/web/app/client-providers.jsx +++ b/web/app/client-providers.tsx @@ -1,9 +1,10 @@ 'use client'; -import { Suspense } from 'react'; +import { Suspense, type ReactNode } from 'react'; import '@/patchConsole'; import { ThemeProvider } from '@/context/ThemeProvider'; -import { ReportAppClient } from '@/ReportShell'; +import { PipelineProvider } from '@/context/PipelineContext'; +import PipelineRunnerFab from '@/components/pipeline/PipelineRunnerFab'; function LoadingFallback() { return ( @@ -13,11 +14,14 @@ function LoadingFallback() { ); } -export default function ClientProviders({ children }) { +export default function ClientProviders({ children }: { children: ReactNode }): ReactNode { return ( }> - {children} + + {children} + + ); diff --git a/web/app/layout.jsx b/web/app/layout.tsx similarity index 90% rename from web/app/layout.jsx rename to web/app/layout.tsx index 03fd315b..9de9325e 100644 --- a/web/app/layout.jsx +++ b/web/app/layout.tsx @@ -1,5 +1,6 @@ import { DM_Sans } from 'next/font/google'; import Script from 'next/script'; +import type { ReactNode } from 'react'; import './globals.css'; import ClientProviders from './client-providers'; @@ -18,7 +19,7 @@ export const metadata = { const themeInit = `(function(){try{var v=localStorage.getItem('wp-theme');var d=window.matchMedia('(prefers-color-scheme: dark)').matches;var dark=v==='dark'?true:v==='light'?false:d;if(dark)document.documentElement.classList.add('dark');else document.documentElement.classList.remove('dark');document.documentElement.style.colorScheme=dark?'dark':'light';}catch(e){}})()`; -export default function RootLayout({ children }) { +export default function RootLayout({ children }: { children: ReactNode }): ReactNode { return ( >; +}) { + const params = await searchParams; + const sp = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (typeof value === 'string') { + sp.set(key, value); + } else if (Array.isArray(value)) { + for (const v of value) { + sp.append(key, v); + } + } + } + const q = sp.toString(); + redirect(q ? `/home?${q}` : '/home'); +} diff --git a/web/app/pipeline/page.tsx b/web/app/pipeline/page.tsx new file mode 100644 index 00000000..4c8cb17f --- /dev/null +++ b/web/app/pipeline/page.tsx @@ -0,0 +1,7 @@ +import PipelinePage from '@/views/Pipeline'; + +export const dynamic = 'force-dynamic'; + +export default function PipelineRoutePage() { + return ; +} diff --git a/web/global.d.ts b/web/global.d.ts new file mode 100644 index 00000000..561f3fab --- /dev/null +++ b/web/global.d.ts @@ -0,0 +1,21 @@ +declare module '*.css'; + +declare module '@/patchConsole'; + +declare module 'react-chartjs-2' { + import type { ChartProps } from 'react-chartjs-2/dist/types'; + import type { ChartData, ChartOptions, DefaultDataPoint } from 'chart.js'; + import type { ComponentType } from 'react'; + + type ChartComponentProps = + ChartProps, unknown>; + + export const Bar: ComponentType>; + export const Line: ComponentType>; + export const Doughnut: ComponentType>; + export const Pie: ComponentType>; + export const Scatter: ComponentType>; + export const Bubble: ComponentType>; + export const Radar: ComponentType>; + export const PolarArea: ComponentType>; +} diff --git a/web/jsconfig.json b/web/jsconfig.json deleted file mode 100644 index abe04df5..00000000 --- a/web/jsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./src/*"] - } - } -} diff --git a/web/next.config.mjs b/web/next.config.mjs index c79585cb..f41b30cf 100644 --- a/web/next.config.mjs +++ b/web/next.config.mjs @@ -5,8 +5,6 @@ const nextConfig = { env: { NEXT_PUBLIC_BASE_PATH: '', }, - /** sql.js loads native wasm from its package on the server */ - serverExternalPackages: ['sql.js'], }; export default nextConfig; diff --git a/web/package-lock.json b/web/package-lock.json index 076f99a1..a69ca1b0 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,21 +13,26 @@ "chart.js": "^4.5.1", "lucide-react": "^0.577.0", "next": "15.5.14", + "pg": "^8.21.0", "react": "19.1.0", "react-chartjs-2": "^5.3.1", "react-dom": "19.1.0", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "remark-breaks": "^4.0.0", - "remark-gfm": "^4.0.1", - "sql.js": "^1.14.1" + "remark-gfm": "^4.0.1" }, "devDependencies": { "@eslint/eslintrc": "^3", "@tailwindcss/postcss": "^4", + "@types/node": "^25.9.1", + "@types/pg": "^8.20.0", + "@types/react": "^19.2.16", + "@types/react-dom": "^19.2.3", "eslint": "^9", "eslint-config-next": "15.5.14", - "tailwindcss": "^4" + "tailwindcss": "^4", + "typescript": "^6.0.3" } }, "node_modules/@alloc/quick-lru": { @@ -1408,6 +1413,28 @@ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", + "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/prismjs": { "version": "1.26.6", "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.6.tgz", @@ -1415,15 +1442,24 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "version": "19.2.16", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.16.tgz", + "integrity": "sha512-esJiCAnl0kfpNdE69f3So4WJUXy95dLZydX0KwK46riIHDzHM7O9Vtf9xCHW0PXIqvgqNrswl522kA/5yx+F4w==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -1431,20 +1467,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.2.tgz", - "integrity": "sha512-NZZgp0Fm2IkD+La5PR81sd+g+8oS6JwJje+aRWsDocxHkjyRw0J5L5ZTlN3LI1LlOcGL7ph3eaIUmTXMIjLk0w==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.60.1.tgz", + "integrity": "sha512-JQ4S5GB0tfjO8BuJ4fcX+HodkzJjYBV+7OJ+wLygaX7OGQ7FudyHL4NSCA6ob+w3Yn+5MkKIozOwQhXeM7opVg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/type-utils": "8.57.2", - "@typescript-eslint/utils": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.60.1", + "@typescript-eslint/type-utils": "8.60.1", + "@typescript-eslint/utils": "8.60.1", + "@typescript-eslint/visitor-keys": "8.60.1", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1454,9 +1490,9 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.2", + "@typescript-eslint/parser": "^8.60.1", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { @@ -1470,16 +1506,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.2.tgz", - "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.60.1.tgz", + "integrity": "sha512-A0M6ua6H252bVjPvvtSgl2QA4+ET9S5Mtkb2GDyTxIhH/C4qDItT7RQNO5PhMC6NXGYXOR9dIalcDDgBKT7oFA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.60.1", + "@typescript-eslint/types": "8.60.1", + "@typescript-eslint/typescript-estree": "8.60.1", + "@typescript-eslint/visitor-keys": "8.60.1", "debug": "^4.4.3" }, "engines": { @@ -1491,18 +1527,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.2.tgz", - "integrity": "sha512-FuH0wipFywXRTHf+bTTjNyuNQQsQC3qh/dYzaM4I4W0jrCqjCVuUh99+xd9KamUfmCGPvbO8NDngo/vsnNVqgw==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.60.1.tgz", + "integrity": "sha512-eXkTH2bxmXlqD1RnOPmLZ9ZM9D3VwSx04JOwBnP9RQ+yUA5a2Mu7SfW8uaV2Aon53NJzZlZYuX7tn91Izf+xaw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.2", - "@typescript-eslint/types": "^8.57.2", + "@typescript-eslint/tsconfig-utils": "^8.60.1", + "@typescript-eslint/types": "^8.60.1", "debug": "^4.4.3" }, "engines": { @@ -1513,18 +1549,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.2.tgz", - "integrity": "sha512-snZKH+W4WbWkrBqj4gUNRIGb/jipDW3qMqVJ4C9rzdFc+wLwruxk+2a5D+uoFcKPAqyqEnSb4l2ULuZf95eSkw==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.60.1.tgz", + "integrity": "sha512-gvI5OQoptnxQnchOirukCuQ55svJSTuD/4k5+pC267xyBtYry748R9/c3tYUzb/iE6RZfllRz2lVulLCHkTm4w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2" + "@typescript-eslint/types": "8.60.1", + "@typescript-eslint/visitor-keys": "8.60.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1535,9 +1571,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.2.tgz", - "integrity": "sha512-3Lm5DSM+DCowsUOJC+YqHHnKEfFh5CoGkj5Z31NQSNF4l5wdOwqGn99wmwN/LImhfY3KJnmordBq/4+VDe2eKw==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.60.1.tgz", + "integrity": "sha512-nh8w4qAteiKuZu3pSSzG/yGKpw0OlkrKnzFmbVRenKaD4qc+7i1GrmZaLVkr8rk4uipiPGMOW4YsM6WmKZ5CvA==", "dev": true, "license": "MIT", "engines": { @@ -1548,21 +1584,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.2.tgz", - "integrity": "sha512-Co6ZCShm6kIbAM/s+oYVpKFfW7LBc6FXoPXjTRQ449PPNBY8U0KZXuevz5IFuuUj2H9ss40atTaf9dlGLzbWZg==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.60.1.tgz", + "integrity": "sha512-sdwTrpjosW7ANQYJ39ZBF1ZyEMEGVB2UsikrserVM/30a/F1dTLnu9bGxEdosugyu5caigjLrR2qiD11asjI1A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2", + "@typescript-eslint/types": "8.60.1", + "@typescript-eslint/typescript-estree": "8.60.1", + "@typescript-eslint/utils": "8.60.1", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1573,13 +1609,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.2.tgz", - "integrity": "sha512-/iZM6FnM4tnx9csuTxspMW4BOSegshwX5oBDznJ7S4WggL7Vczz5d2W11ecc4vRrQMQHXRSxzrCsyG5EsPPTbA==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.60.1.tgz", + "integrity": "sha512-4h0tY8ppCkdCzcrl2YM5M3my0xsE1Tf8om3owEu5oPWmXwkKRmk0j0LGDzYBGUcAlesEbxBhazqu/K4cu3Ug7w==", "dev": true, "license": "MIT", "engines": { @@ -1591,21 +1627,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.2.tgz", - "integrity": "sha512-2MKM+I6g8tJxfSmFKOnHv2t8Sk3T6rF20A1Puk0svLK+uVapDZB/4pfAeB7nE83uAZrU6OxW+HmOd5wHVdXwXA==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.60.1.tgz", + "integrity": "sha512-alpRkfG8hlVE5kdJW2GkfgDgXxold3e8e4l6EnmhRmRLbekgAPCCGDVD++sABy9FcgPFroq+uFcCSM1vR57Cew==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.2", - "@typescript-eslint/tsconfig-utils": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/project-service": "8.60.1", + "@typescript-eslint/tsconfig-utils": "8.60.1", + "@typescript-eslint/types": "8.60.1", + "@typescript-eslint/visitor-keys": "8.60.1", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1615,7 +1651,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { @@ -1629,9 +1665,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", "dev": true, "license": "MIT", "dependencies": { @@ -1642,13 +1678,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "brace-expansion": "^5.0.5" }, "engines": { "node": "18 || 20 || >=22" @@ -1658,16 +1694,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.2.tgz", - "integrity": "sha512-krRIbvPK1ju1WBKIefiX+bngPs+odIQUtR7kymzPfo1POVw3jlF+nLkmexdSSd4UCbDcQn+wMBATOOmpBbqgKg==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.60.1.tgz", + "integrity": "sha512-h2MPBLoNtjc3qZWfY3Tl51yPorQ2McHn8pJfcMNTcIvrrZrr90Ykffit0yjrPFWQcRcUxzH20+6OcVdW4yHtUg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2" + "@typescript-eslint/scope-manager": "8.60.1", + "@typescript-eslint/types": "8.60.1", + "@typescript-eslint/typescript-estree": "8.60.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1678,17 +1714,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.2.tgz", - "integrity": "sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==", + "version": "8.60.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.60.1.tgz", + "integrity": "sha512-EbGRQg4FhrmwLodl+t3JNAnXHWVr9Vp+Zl1QBZVPY4ByfkzIT8cX3K6QWODHtkIZqqJVEWvhHSx3v5PDHsaQag==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", + "@typescript-eslint/types": "8.60.1", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2560,8 +2596,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/d3-array": { "version": "3.2.4", @@ -6551,6 +6586,95 @@ "dev": true, "license": "MIT" }, + "node_modules/pg": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.21.0.tgz", + "integrity": "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.13.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.14.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.13.0.tgz", + "integrity": "sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.14.0.tgz", + "integrity": "sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -6621,6 +6745,45 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/preact": { "version": "10.29.0", "resolved": "https://registry.npmjs.org/preact/-/preact-10.29.0.tgz", @@ -7291,11 +7454,14 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/sql.js": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz", - "integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==", - "license": "MIT" + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } }, "node_modules/stable-hash": { "version": "0.0.5", @@ -7816,12 +7982,11 @@ } }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7849,6 +8014,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "node_modules/unified": { "version": "11.0.5", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", @@ -8124,6 +8296,15 @@ "node": ">=0.10.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/web/package.json b/web/package.json index 3fd15cc8..260e50d9 100644 --- a/web/package.json +++ b/web/package.json @@ -6,7 +6,9 @@ "dev": "next dev", "build": "next build", "start": "next start", - "lint": "eslint" + "lint": "eslint", + "typecheck": "tsc --noEmit", + "typecheck:strict": "tsc --noEmit -p tsconfig.strict.json" }, "dependencies": { "@tanstack/react-virtual": "^3.13.23", @@ -14,20 +16,25 @@ "chart.js": "^4.5.1", "lucide-react": "^0.577.0", "next": "15.5.14", + "pg": "^8.21.0", "react": "19.1.0", "react-chartjs-2": "^5.3.1", "react-dom": "19.1.0", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "remark-breaks": "^4.0.0", - "remark-gfm": "^4.0.1", - "sql.js": "^1.14.1" + "remark-gfm": "^4.0.1" }, "devDependencies": { "@eslint/eslintrc": "^3", "@tailwindcss/postcss": "^4", + "@types/node": "^25.9.1", + "@types/pg": "^8.20.0", + "@types/react": "^19.2.16", + "@types/react-dom": "^19.2.3", "eslint": "^9", "eslint-config-next": "15.5.14", - "tailwindcss": "^4" + "tailwindcss": "^4", + "typescript": "^6.0.3" } } diff --git a/web/src/ReportShell.jsx b/web/src/ReportShell.jsx deleted file mode 100644 index 920bd3ce..00000000 --- a/web/src/ReportShell.jsx +++ /dev/null @@ -1,464 +0,0 @@ -'use client'; - -import { useState, useEffect } from 'react'; -import dynamic from 'next/dynamic'; -import Link from 'next/link'; -import { useRouter, usePathname, useSearchParams } from 'next/navigation'; -import { - Radar, - Home as HomeIcon, - LayoutDashboard, - AlertOctagon, - Link as LinkIcon, - Repeat, - FileText, - ShieldAlert, - Gauge, - PieChart, - Share2, - Search, - BarChart2, - Cpu, - Menu, - X, - ExternalLink, - Images, - FolderTree, - Settings2, - TrendingUp, - Key, -} from 'lucide-react'; -import IntegrationsModal from './components/IntegrationsModal.jsx'; -import { ReportProvider } from './context/ReportContext.jsx'; -import { useReport } from './context/useReport'; -import { strings, format } from './lib/strings'; -import { canonicalDomainFromPayload, slugifyDomain } from './lib/domainSlug'; -import { pathSlugToViewId, viewIdToPathSlug } from './routes.js'; -import { Badge, ReportSelector } from './components'; -import PipelineRunnerFab from './components/PipelineRunnerFab.jsx'; -import ThemeToggle from './components/ThemeToggle.jsx'; -import ReportShellSkeleton from './components/ReportShellSkeleton.jsx'; -import Overview from './views/Overview'; -import Home from './views/Home'; -import Issues from './views/Issues'; -import Links from './views/Links'; -import SiteStructure from './views/SiteStructure'; -import Redirects from './views/Redirects'; -import Content from './views/Content'; -import Security from './views/Security'; -import Lighthouse from './views/Lighthouse'; -import Charts from './views/Charts'; -const Network = dynamic(() => import('./views/Network'), { - ssr: false, - loading: () => ( -
- Loading network graph… -
-
-
- ), -}); -import ContentAnalytics from './views/ContentAnalytics'; -import TechStack from './views/TechStack'; -import Gallery from './views/Gallery'; -import SearchPerformance from './views/SearchPerformance'; -import Traffic from './views/Traffic'; -import KeywordsExplorer from './views/KeywordsExplorer'; - -const VIEW_CONFIG = [ - { id: 'home', component: Home, icon: HomeIcon }, - { id: 'overview', component: Overview, icon: LayoutDashboard }, - { id: 'issues', component: Issues, icon: AlertOctagon }, - { id: 'links', component: Links, icon: LinkIcon }, - { id: 'site-structure', component: SiteStructure, icon: FolderTree }, - { id: 'redirects', component: Redirects, icon: Repeat }, - { id: 'content', component: Content, icon: FileText }, - { id: 'lighthouse', component: Lighthouse, icon: Gauge }, - { id: 'security', component: Security, icon: ShieldAlert }, - { id: 'content-analytics', component: ContentAnalytics, icon: BarChart2 }, - { id: 'tech-stack', component: TechStack, icon: Cpu }, - { id: 'charts', component: Charts, icon: PieChart }, - { id: 'network', component: Network, icon: Share2 }, - { id: 'gallery', component: Gallery, icon: Images }, - { id: 'search-performance', component: SearchPerformance, icon: TrendingUp }, - { id: 'traffic', component: Traffic, icon: BarChart2 }, - { id: 'keywords-explorer', component: KeywordsExplorer, icon: Key }, -]; - -const VIEWS = VIEW_CONFIG.map((v) => ({ - ...v, - label: strings.nav[v.id].label, - section: strings.nav[v.id].section, -})); - -/** @param {{ slug: string }} props */ -function BrandUrlSync({ slug }) { - const { data, loading, error, startUrlByRunId } = useReport(); - const searchParams = useSearchParams(); - const router = useRouter(); - const pathname = usePathname(); - const searchStr = searchParams.toString(); - - useEffect(() => { - if (slug !== 'home') return; - if (!searchParams.get('domain') && !searchParams.get('brand')) return; - const next = new URLSearchParams(searchStr); - next.delete('domain'); - next.delete('brand'); - const q = next.toString(); - router.replace(q ? `${pathname}?${q}` : pathname); - }, [slug, searchStr, searchParams, router, pathname]); - - useEffect(() => { - if (slug === 'home') return; - if (loading || error || !data) return; - if (searchParams.get('domain') || searchParams.get('brand')) return; - const host = canonicalDomainFromPayload(data, startUrlByRunId); - const fallback = slugifyDomain(data.site_name || ''); - const value = host || fallback; - if (!value) return; - const next = new URLSearchParams(searchStr); - next.set('domain', value); - const q = next.toString(); - router.replace(q ? `${pathname}?${q}` : pathname); - }, [slug, loading, error, data, searchStr, searchParams, router, pathname, startUrlByRunId]); - - return null; -} - -/** @param {{ slug: string }} props */ -function AppContent({ slug }) { - const router = useRouter(); - const pathname = usePathname(); - const searchParams = useSearchParams(); - const [searchQuery, setSearchQuery] = useState(''); - const [sidebarOpen, setSidebarOpen] = useState(false); - const [integrationsOpen, setIntegrationsOpen] = useState(false); - const [integrationsToast, setIntegrationsToast] = useState(null); - const { data, loading, error, setSelectedReportId, startUrlByRunId } = useReport(); - - // Auto-open Integrations modal after OAuth callback (?integrations=open&auth=...) - useEffect(() => { - const intParam = searchParams.get('integrations'); - const authParam = searchParams.get('auth'); - const reasonParam = searchParams.get('reason'); - if (intParam === 'open') { - setIntegrationsOpen(true); - if (authParam === 'success') { - setIntegrationsToast({ type: 'success', message: 'Google account connected successfully.' }); - } else if (authParam === 'error') { - setIntegrationsToast({ - type: 'error', - message: reasonParam ? decodeURIComponent(reasonParam) : 'Google connection failed.', - }); - } - // Clean up the query params - const next = new URLSearchParams(searchParams.toString()); - next.delete('integrations'); - next.delete('auth'); - next.delete('reason'); - const q = next.toString(); - router.replace(q ? `${pathname}?${q}` : pathname); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - const view = pathSlugToViewId(slug ?? ''); - const closeSidebar = () => setSidebarOpen(false); - const trailing = searchParams.toString() ? `?${searchParams.toString()}` : ''; - - useEffect(() => { - if (!pathSlugToViewId(slug ?? '')) { - router.replace('/home'); - } - }, [slug, router]); - - /** - * @param {string} id - view id - * @param {{ domain?: string, reportId?: number }} [opts] - */ - const selectView = (id, opts) => { - if (opts?.reportId != null) { - setSelectedReportId(opts.reportId); - } - closeSidebar(); - const path = `/${viewIdToPathSlug(id)}`; - if (id === 'home') { - router.push('/home'); - return; - } - if (opts?.domain != null && opts.domain !== '') { - const p = new URLSearchParams(searchParams.toString()); - p.set('domain', opts.domain); - const q = p.toString(); - router.push(q ? `${path}?${q}` : path); - return; - } - const q = searchParams.toString(); - router.push(q ? `${path}?${q}` : path); - }; - - if (!view) { - return ( -
-

{strings.app.loading}

-
- ); - } - - const CurrentView = VIEWS.find((v) => v.id === view)?.component || Home; - const showSidebar = view !== 'home'; - const openIntegrations = () => { - setIntegrationsToast(null); - setIntegrationsOpen(true); - }; - const issueCount = data?.categories?.reduce((n, c) => n + (c.issues?.length || 0), 0) ?? 0; - const securityCount = data?.security_findings?.length ?? 0; - - if (loading) { - return ; - } - - if (error) { - const isDomainError = error === strings.app.noReportForDomain; - return ( -
-
- - -
- setIntegrationsOpen(false)} - initialToast={integrationsToast} - /> -
-
-

- {isDomainError ? strings.app.noReportForDomainTitle : strings.app.failedTitle} -

-

{error}

- {!isDomainError ? ( -

{strings.app.failedHint}

- ) : null} - -
-
-
- ); - } - - const auditedHost = - canonicalDomainFromPayload(data, startUrlByRunId) || strings.app.defaultSiteName; - const runId = data?.crawl_run_id != null ? Number(data.crawl_run_id) : null; - const auditedStartUrl = - (runId != null ? startUrlByRunId?.get(runId) : null) || - (auditedHost ? `https://${auditedHost}` : ''); - const auditedInitials = auditedHost.charAt(0).toUpperCase() || 'S'; - const crawlSummary = data?.summary; - const lastCrawlText = - crawlSummary?.crawl_time_s != null - ? format(strings.app.crawlCompletedSeconds, { seconds: crawlSummary.crawl_time_s }) - : strings.app.crawlCompleted; - - const sections = [...new Set(VIEWS.map((v) => v.section))]; - - return ( -
- {showSidebar && sidebarOpen && ( - -
- - - - )} - -
- {showSidebar ? ( -
- -
- - setSearchQuery(e.target.value)} - placeholder={strings.app.searchPlaceholder} - className="w-full bg-brand-900 border border-default focus:border-blue-500 rounded-lg pl-10 pr-4 py-2 text-sm outline-none text-foreground transition-all" - /> -
-
- - - -
-
- ) : null} - - setIntegrationsOpen(false)} - initialToast={integrationsToast} - /> - -
-
- -
-
-
-
- ); -} - -/** @param {{ slug: string }} props */ -function RoutedShell({ slug }) { - return ( - <> - - - - - ); -} - -/** @param {{ slug: string }} props */ -export default function ReportShell({ slug }) { - return ; -} - -/** Wraps children with ReportProvider (db + domain from URL). */ -export function ReportAppClient({ children }) { - const searchParams = useSearchParams(); - const domainRaw = searchParams.get('domain') ?? searchParams.get('brand'); - const domainSlug = domainRaw != null && domainRaw !== '' ? domainRaw : null; - - return ( - - {children} - - ); -} diff --git a/web/src/ReportShell.tsx b/web/src/ReportShell.tsx new file mode 100644 index 00000000..4c565562 --- /dev/null +++ b/web/src/ReportShell.tsx @@ -0,0 +1,263 @@ +'use client'; + +import { useState, useEffect, type ComponentType, type ReactNode } from 'react'; +import dynamic from 'next/dynamic'; +import Link from 'next/link'; +import { useRouter, usePathname, useSearchParams } from 'next/navigation'; +import { + Home as HomeIcon, + LayoutDashboard, + AlertOctagon, + Link as LinkIcon, + Repeat, + FileText, + ShieldAlert, + Gauge, + PieChart, + Share2, + BarChart2, + Cpu, + Images, + FolderTree, + TrendingUp, + Key, + ArrowLeftRight, +} from 'lucide-react'; +import AppShell from './components/AppShell'; +import { useReport } from './context/useReport'; +import { strings } from './lib/strings'; +import { canonicalDomainFromPayload, slugifyDomain } from './lib/domainSlug'; +import { pathSlugToViewId, viewIdToPathSlug, type ViewId } from './routes'; +import { dispatchOpenIntegrations } from './lib/pipelineJobEvents'; +import ReportShellSkeleton from './components/ReportShellSkeleton'; +import { ReportProvider as ReportProviderBase } from './context/ReportContext'; +import type { ReportPayload } from '@/types'; +function viewLoading(label = 'Loading view…') { + return ( +
+ {label} +
+
+
+ ); +} + +const Home = dynamic(() => import('./views/Home'), { loading: () => viewLoading() }); +const Overview = dynamic(() => import('./views/Overview'), { loading: () => viewLoading() }); +const CompareReports = dynamic(() => import('./views/CompareReports'), { loading: () => viewLoading() }); +const Issues = dynamic(() => import('./views/Issues'), { loading: () => viewLoading() }); +const Links = dynamic(() => import('./views/Links'), { loading: () => viewLoading() }); +const SiteStructure = dynamic(() => import('./views/SiteStructure'), { loading: () => viewLoading() }); +const Redirects = dynamic(() => import('./views/Redirects'), { loading: () => viewLoading() }); +const Content = dynamic(() => import('./views/Content'), { loading: () => viewLoading() }); +const Security = dynamic(() => import('./views/Security'), { loading: () => viewLoading() }); +const Lighthouse = dynamic(() => import('./views/Lighthouse'), { loading: () => viewLoading() }); +const Charts = dynamic(() => import('./views/Charts'), { loading: () => viewLoading() }); +const Network = dynamic(() => import('./views/Network'), { + ssr: false, + loading: () => viewLoading('Loading network graph…'), +}); +const ContentAnalytics = dynamic(() => import('./views/ContentAnalytics'), { loading: () => viewLoading() }); +const TechStack = dynamic(() => import('./views/TechStack'), { loading: () => viewLoading() }); +const Gallery = dynamic(() => import('./views/Gallery'), { loading: () => viewLoading() }); +const SearchPerformance = dynamic(() => import('./views/SearchPerformance'), { loading: () => viewLoading() }); +const Traffic = dynamic(() => import('./views/Traffic'), { loading: () => viewLoading() }); +const KeywordsExplorer = dynamic(() => import('./views/KeywordsExplorer'), { loading: () => viewLoading() }); + +interface ReportShellReportContext { + data: ReportPayload | null; + loading: boolean; + error: string | null; + setSelectedReportId: (id: number | null) => void; + startUrlByRunId: Map; +} + +interface CurrentViewProps { + searchQuery: string; + onNavigate: (id: ViewId | string, opts?: { domain?: string; reportId?: number }) => void; + onOpenIntegrations: () => void; +} + +interface ViewConfigEntry { + id: ViewId; + component: ComponentType; + icon: ComponentType<{ className?: string }>; +} + +interface SlugProps { + slug: string; +} + +const ReportProvider = ReportProviderBase as ComponentType<{ + children: ReactNode; + domainSlug?: string | null; +}>; + +const VIEW_CONFIG: ViewConfigEntry[] = [ + { id: 'home', component: Home as ComponentType, icon: HomeIcon }, + { id: 'overview', component: Overview as ComponentType, icon: LayoutDashboard }, + { id: 'compare', component: CompareReports as ComponentType, icon: ArrowLeftRight }, + { id: 'issues', component: Issues as ComponentType, icon: AlertOctagon }, + { id: 'links', component: Links as ComponentType, icon: LinkIcon }, + { id: 'site-structure', component: SiteStructure as ComponentType, icon: FolderTree }, + { id: 'redirects', component: Redirects as ComponentType, icon: Repeat }, + { id: 'content', component: Content as ComponentType, icon: FileText }, + { id: 'lighthouse', component: Lighthouse as ComponentType, icon: Gauge }, + { id: 'security', component: Security as ComponentType, icon: ShieldAlert }, + { id: 'content-analytics', component: ContentAnalytics as ComponentType, icon: BarChart2 }, + { id: 'tech-stack', component: TechStack as ComponentType, icon: Cpu }, + { id: 'charts', component: Charts as ComponentType, icon: PieChart }, + { id: 'network', component: Network as ComponentType, icon: Share2 }, + { id: 'gallery', component: Gallery as ComponentType, icon: Images }, + { id: 'search-performance', component: SearchPerformance as ComponentType, icon: TrendingUp }, + { id: 'traffic', component: Traffic as ComponentType, icon: BarChart2 }, + { id: 'keywords-explorer', component: KeywordsExplorer as ComponentType, icon: Key }, +]; + +const VIEWS = VIEW_CONFIG.map((v) => ({ + ...v, + label: strings.nav[v.id].label, + section: strings.nav[v.id].section, +})); + +/** Sync `?domain=` query param with the active report payload. */ +function BrandUrlSync({ slug }: SlugProps): null { + const { data, loading, error, startUrlByRunId } = useReport() as ReportShellReportContext; + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + const searchStr = searchParams.toString(); + + useEffect(() => { + if (slug !== 'home') return; + if (!searchParams.get('domain') && !searchParams.get('brand')) return; + const next = new URLSearchParams(searchStr); + next.delete('domain'); + next.delete('brand'); + const q = next.toString(); + router.replace(q ? `${pathname}?${q}` : pathname); + }, [slug, searchStr, searchParams, router, pathname]); + + useEffect(() => { + if (slug === 'home') return; + if (loading || error || !data) return; + if (searchParams.get('domain') || searchParams.get('brand')) return; + const host = canonicalDomainFromPayload(data, startUrlByRunId); + const fallback = slugifyDomain(data.site_name || ''); + const value = host || fallback; + if (!value) return; + const next = new URLSearchParams(searchStr); + next.set('domain', value); + const q = next.toString(); + router.replace(q ? `${pathname}?${q}` : pathname); + }, [slug, loading, error, data, searchStr, searchParams, router, pathname, startUrlByRunId]); + + return null; +} + +/** Main report shell layout and navigation. */ +function AppContent({ slug }: SlugProps): ReactNode { + const router = useRouter(); + const searchParams = useSearchParams(); + const [searchQuery, setSearchQuery] = useState(''); + const { loading, error, setSelectedReportId } = useReport() as ReportShellReportContext; + + const view = pathSlugToViewId(slug ?? ''); + + const selectView = (id: ViewId | string, opts?: { domain?: string; reportId?: number }): void => { + if (opts?.reportId != null) { + setSelectedReportId(opts.reportId); + } + const path = `/${viewIdToPathSlug(id)}`; + if (id === 'home') { + router.push('/home'); + return; + } + if (opts?.domain != null && opts.domain !== '') { + const p = new URLSearchParams(searchParams.toString()); + p.set('domain', opts.domain); + const q = p.toString(); + router.push(q ? `${path}?${q}` : path); + return; + } + const q = searchParams.toString(); + router.push(q ? `${path}?${q}` : path); + }; + + if (!view) { + return null; + } + + const CurrentView = VIEWS.find((v) => v.id === view)?.component || Home; + const showSidebar = view !== 'home'; + + if (loading) { + return ; + } + + if (error && view !== 'home') { + const isDomainError = error === strings.app.noReportForDomain; + return ( + +
+
+

+ {isDomainError ? strings.app.noReportForDomainTitle : strings.app.failedTitle} +

+

{error}

+ {!isDomainError ? ( +

{strings.app.failedHint}

+ ) : null} + + Open Pipeline + +
+
+ + ); + } + + return ( + + + + ); +} + +function RoutedShell({ slug }: SlugProps): ReactNode { + return ( + <> + + + + ); +} + +export default function ReportShell({ slug }: SlugProps): ReactNode { + return ; +} + +/** Wraps children with ReportProvider (db + domain from URL). */ +export function ReportAppClient({ children }: { children: ReactNode }): ReactNode { + const searchParams = useSearchParams(); + const domainRaw = searchParams.get('domain') ?? searchParams.get('brand'); + const domainSlug = domainRaw != null && domainRaw !== '' ? domainRaw : null; + + return ( + + {children} + + ); +} diff --git a/web/src/components/AppShell.tsx b/web/src/components/AppShell.tsx new file mode 100644 index 00000000..bd83c1a0 --- /dev/null +++ b/web/src/components/AppShell.tsx @@ -0,0 +1,280 @@ +'use client'; + +import { useEffect, useState, type ReactNode } from 'react'; +import Link from 'next/link'; +import { usePathname, useRouter, useSearchParams } from 'next/navigation'; +import { + ExternalLink, + Menu, + Radar, + Search, + Settings2, + X, +} from 'lucide-react'; +import IntegrationsModal from '@/components/IntegrationsModal'; +import { Badge, ReportSelector } from '@/components'; +import ThemeToggle from '@/components/ThemeToggle'; +import { useReport } from '@/context/useReport'; +import { strings, format } from '@/lib/strings'; +import { canonicalDomainFromPayload } from '@/lib/domainSlug'; +import { OPEN_INTEGRATIONS } from '@/lib/pipelineJobEvents'; +import { + APP_NAV_ITEMS, + APP_NAV_SECTIONS, + isNavItemActive, + navHref, + type NavItemId, +} from '@/lib/appNav'; +import type { ReportPayload } from '@/types'; + +interface IntegrationsToast { + type: 'success' | 'error'; + message: string; +} + +interface ReportCategoryWithIssues { + issues?: unknown[]; +} + +export interface AppShellProps { + children: ReactNode; + showSidebar?: boolean; + showSearch?: boolean; + searchQuery?: string; + onSearchChange?: (value: string) => void; + headerExtra?: ReactNode; +} + +export default function AppShell({ + children, + showSidebar = true, + showSearch = true, + searchQuery = '', + onSearchChange, + headerExtra, +}: AppShellProps) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const [sidebarOpen, setSidebarOpen] = useState(false); + const [integrationsOpen, setIntegrationsOpen] = useState(false); + const [integrationsToast, setIntegrationsToast] = useState(null); + const { data, startUrlByRunId } = useReport(); + + const trailing = searchParams.toString() ? `?${searchParams.toString()}` : ''; + const closeSidebar = () => setSidebarOpen(false); + + useEffect(() => { + const intParam = searchParams.get('integrations'); + const authParam = searchParams.get('auth'); + const reasonParam = searchParams.get('reason'); + if (intParam === 'open') { + setIntegrationsOpen(true); + if (authParam === 'success') { + setIntegrationsToast({ type: 'success', message: 'Google account connected successfully.' }); + } else if (authParam === 'error') { + setIntegrationsToast({ + type: 'error', + message: reasonParam ? decodeURIComponent(reasonParam) : 'Google connection failed.', + }); + } + const next = new URLSearchParams(searchParams.toString()); + next.delete('integrations'); + next.delete('auth'); + next.delete('reason'); + const q = next.toString(); + router.replace(q ? `${pathname}?${q}` : pathname); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + const onOpen = () => { + setIntegrationsToast(null); + setIntegrationsOpen(true); + }; + window.addEventListener(OPEN_INTEGRATIONS, onOpen); + return () => window.removeEventListener(OPEN_INTEGRATIONS, onOpen); + }, []); + + const openIntegrations = () => { + setIntegrationsToast(null); + setIntegrationsOpen(true); + }; + + const issueCount = + (data?.categories as ReportCategoryWithIssues[] | undefined)?.reduce( + (n: number, c: ReportCategoryWithIssues) => n + (c.issues?.length ?? 0), + 0, + ) ?? 0; + const securityFindings = data?.security_findings; + const securityCount = Array.isArray(securityFindings) ? securityFindings.length : 0; + + const auditedHost = + canonicalDomainFromPayload(data, startUrlByRunId) || strings.app.defaultSiteName; + const runId = data?.crawl_run_id != null ? Number(data.crawl_run_id) : null; + const auditedStartUrl = + (runId != null ? startUrlByRunId?.get(runId) : null) || + (auditedHost ? `https://${auditedHost}` : ''); + const auditedInitials = auditedHost.charAt(0).toUpperCase() || 'S'; + const crawlSummary = data?.summary as (ReportPayload['summary'] & { crawl_time_s?: number }) | undefined; + const lastCrawlText = + crawlSummary?.crawl_time_s != null + ? format(strings.app.crawlCompletedSeconds, { seconds: crawlSummary.crawl_time_s }) + : strings.app.crawlCompleted; + + return ( +
+ {showSidebar && sidebarOpen ? ( + +
+ + + +
+
+
+ {auditedInitials} +
+
+
{auditedHost}
+
{lastCrawlText}
+
+
+ {auditedStartUrl ? ( + + + {strings.app.viewSiteLabel} + + ) : null} +
+ + ) : null} + +
+ {showSidebar ? ( +
+ + {showSearch && onSearchChange ? ( +
+ + onSearchChange(e.target.value)} + placeholder={strings.app.searchPlaceholder} + className="w-full bg-brand-900 border border-default focus:border-blue-500 rounded-lg pl-10 pr-4 py-2 text-sm outline-none text-foreground transition-all" + /> +
+ ) : ( +
+ )} +
+ {headerExtra} + + + +
+
+ ) : null} + + setIntegrationsOpen(false)} + initialToast={integrationsToast} + /> + +
+
{children}
+
+
+
+ ); +} + +export type { NavItemId }; diff --git a/web/src/components/Badge.jsx b/web/src/components/Badge.tsx similarity index 80% rename from web/src/components/Badge.jsx rename to web/src/components/Badge.tsx index 6ef2c36a..4051f52e 100644 --- a/web/src/components/Badge.jsx +++ b/web/src/components/Badge.tsx @@ -1,10 +1,11 @@ +import type { ReactNode } from 'react'; import { getBadgeVariant } from '../lib/badges'; /** * Unified severity/priority/status badge. Variants: critical, high, medium, low, info, success. * Single size: text-xs, py-1, px-2. Normalize display value via optional `label` prop. */ -const VARIANT_CLASSES = { +const VARIANT_CLASSES: Record = { critical: 'bg-red-500 text-white', high: 'bg-red-500/20 text-red-700 dark:text-red-400 border border-red-500/30', medium: 'bg-yellow-500/20 text-yellow-800 dark:text-yellow-400 border border-yellow-500/30', @@ -13,7 +14,17 @@ const VARIANT_CLASSES = { success: 'bg-green-500/20 text-green-700 dark:text-green-400 border border-green-500/30', }; -export default function Badge({ variant, value, label, className = '' }) { +export default function Badge({ + variant, + value, + label, + className = '', +}: { + variant?: string; + value?: string | number | null; + label?: string; + className?: string; +}) { const v = variant || getBadgeVariant(value); const display = label != null ? label : (value != null && value !== '' ? String(value) : '—'); const classes = VARIANT_CLASSES[v] || VARIANT_CLASSES.info; diff --git a/web/src/components/Button.jsx b/web/src/components/Button.tsx similarity index 73% rename from web/src/components/Button.jsx rename to web/src/components/Button.tsx index 61dd0c3b..aa4dae91 100644 --- a/web/src/components/Button.jsx +++ b/web/src/components/Button.tsx @@ -1,3 +1,13 @@ +import type { ButtonHTMLAttributes, ReactNode } from 'react'; + +type ButtonVariant = 'primary' | 'secondary' | 'ghost'; + +type ButtonProps = { + children?: ReactNode; + variant?: ButtonVariant; + className?: string; +} & ButtonHTMLAttributes; + /** * Shared button: primary (Export style), secondary (border), ghost. * Same size: px-4 py-2 rounded-lg text-sm font-medium/bold for primary. @@ -10,9 +20,9 @@ export default function Button({ onClick, disabled, ...rest -}) { +}: ButtonProps) { const base = 'inline-flex items-center justify-center gap-2 px-4 py-2 rounded-lg text-sm font-medium transition-colors disabled:opacity-50 disabled:pointer-events-none'; - const variants = { + const variants: Record = { primary: 'bg-blue-600 hover:bg-blue-500 text-white font-bold', secondary: 'border border-default text-foreground hover:bg-brand-700/80', ghost: 'text-muted-foreground hover:text-foreground hover:bg-brand-800/80', diff --git a/web/src/components/Card.jsx b/web/src/components/Card.tsx similarity index 67% rename from web/src/components/Card.jsx rename to web/src/components/Card.tsx index 7d978772..e73af0cd 100644 --- a/web/src/components/Card.jsx +++ b/web/src/components/Card.tsx @@ -1,3 +1,14 @@ +import type { MouseEventHandler, ReactNode } from 'react'; + +type CardProps = { + children?: ReactNode; + className?: string; + padding?: 'default' | 'tight' | 'none'; + shadow?: boolean; + overflowHidden?: boolean; + onClick?: MouseEventHandler; +}; + /** * Standard card container: bg-brand-800, border, rounded-xl, padding. * Use shadow for stat cards, overflowHidden for table wrappers. @@ -8,12 +19,14 @@ export default function Card({ padding = 'default', shadow = false, overflowHidden = false, -}) { + onClick, +}: CardProps) { const paddingClass = padding === 'none' ? '' : padding === 'tight' ? 'p-4' : 'p-5'; const shadowClass = shadow ? 'shadow-sm' : ''; const overflowClass = overflowHidden ? 'overflow-hidden' : ''; return (
{children} diff --git a/web/src/components/GoogleIntegrationsPanel.tsx b/web/src/components/GoogleIntegrationsPanel.tsx new file mode 100644 index 00000000..cfb297d9 --- /dev/null +++ b/web/src/components/GoogleIntegrationsPanel.tsx @@ -0,0 +1,744 @@ +'use client'; + +import { useState, useEffect, useCallback, useRef, type ReactNode } from 'react'; +import { + CheckCircle2, + AlertCircle, + Loader2, + ChevronDown, + ChevronUp, + ExternalLink, + KeyRound, + Link2, + BarChart3, +} from 'lucide-react'; +import type { GooglePropertiesResponse, GoogleStatusResponse, IntegrationToast } from '@/types/api'; +import { apiUrl } from '@/lib/publicBase'; +import { dispatchPipelineJobStarted, pollPipelineJob } from '@/lib/pipelineJobEvents'; +import { useOptionalReport } from '@/context/useReport'; +import Button from '@/components/Button'; + +const GCP_GUIDE_URL = + 'https://developers.google.com/workspace/guides/get-started'; + +function StatusPill({ connected }: { connected?: boolean }) { + if (connected) { + return ( + + + Connected + + ); + } + return ( + + + Not connected + + ); +} + +function GoogleMark({ className = 'h-4 w-4' }: { className?: string }) { + return ( + + ); +} + +function SetupStep({ + step, + title, + description, + done, + icon: Icon, + children, +}: { + step: number; + title: string; + description?: string; + done?: boolean; + icon: typeof KeyRound; + children: ReactNode; +}) { + return ( +
+
+ + {done ? : step} + +
+

{title}

+ {description ?

{description}

: null} +
+ +
+
{children}
+
+ ); +} + +function selectClassName() { + return 'w-full rounded-lg border border-default bg-brand-900 px-3 py-2.5 text-sm text-foreground focus:border-blue-500/50 focus:outline-none focus:ring-2 focus:ring-blue-500/20'; +} + +function InputField({ + label, + id, + type = 'text', + value, + onChange, + placeholder, + helper, + disabled, +}: { + label: string; + id: string; + type?: string; + value: string; + onChange: (v: string) => void; + placeholder?: string; + helper?: string; + disabled?: boolean; +}) { + return ( +
+ + onChange(e.target.value)} + placeholder={placeholder} + disabled={disabled} + autoComplete="off" + className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2.5 text-sm text-foreground font-mono focus:border-blue-500/50 focus:outline-none focus:ring-2 focus:ring-blue-500/20 disabled:opacity-50" + /> + {helper &&

{helper}

} +
+ ); +} + +export interface GoogleIntegrationsPanelProps { + initialToast?: IntegrationToast | null; + showTitle?: boolean; +} + +/** + * Inline Google Search Console + GA4 setup (credentials, connect, properties, fetch). + */ +export default function GoogleIntegrationsPanel({ + initialToast, + showTitle = true, +}: GoogleIntegrationsPanelProps) { + const report = useOptionalReport(); + const [status, setStatus] = useState(null); + const [loadingStatus, setLoadingStatus] = useState(false); + + // Phase 1 fields + const [clientId, setClientId] = useState(''); + const [clientSecret, setClientSecret] = useState(''); + const [savingCreds, setSavingCreds] = useState(false); + + // Phase 2 fields + const [gscSiteUrl, setGscSiteUrl] = useState(''); + const [ga4PropertyId, setGa4PropertyId] = useState(''); + const [dateRangeDays, setDateRangeDays] = useState('28'); + const [savingProps, setSavingProps] = useState(false); + + // Properties dropdowns + const [properties, setProperties] = useState(null); + const [loadingProps, setLoadingProps] = useState(false); + + // Test / Fetch + const [testLog, setTestLog] = useState(''); + const [testing, setTesting] = useState(false); + const [fetching, setFetching] = useState(false); + const [fetchLog, setFetchLog] = useState(''); + const [fetchJobStatus, setFetchJobStatus] = useState(''); + const fetchPollStopRef = useRef<(() => void) | null>(null); + + // Advanced accordion (paste refresh token) + const [showAdvanced, setShowAdvanced] = useState(false); + const [refreshToken, setRefreshToken] = useState(''); + const [savingToken, setSavingToken] = useState(false); + + // Toast from OAuth callback + const [toast, setToast] = useState(initialToast || null); + + const fetchStatus = useCallback(async () => { + setLoadingStatus(true); + try { + const res = await fetch(apiUrl('/integrations/google/status')); + if (res.ok) { + const data = (await res.json()) as GoogleStatusResponse; + setStatus(data); + if (data.gscSiteUrl) setGscSiteUrl(data.gscSiteUrl); + if (data.ga4PropertyId) setGa4PropertyId(data.ga4PropertyId); + if (data.dateRangeDays) setDateRangeDays(String(data.dateRangeDays)); + } + } catch { + // ignore + } finally { + setLoadingStatus(false); + } + }, []); + + useEffect(() => { + void fetchStatus(); + }, [fetchStatus]); + + useEffect(() => { + if (initialToast) setToast(initialToast); + }, [initialToast]); + + // Auto-dismiss toast after 6s + useEffect(() => { + if (!toast) return; + const t = setTimeout(() => setToast(null), 6000); + return () => clearTimeout(t); + }, [toast]); + + const loadProperties = async () => { + setLoadingProps(true); + try { + const res = await fetch(apiUrl('/integrations/google/properties')); + if (res.ok) { + const data = (await res.json()) as GooglePropertiesResponse; + setProperties(data); + if (data.ga4ListError) { + setToast({ type: 'error', message: data.ga4ListError }); + } + } + } catch { + // ignore + } finally { + setLoadingProps(false); + } + }; + + const handleSaveClientCreds = async () => { + if (!clientId.trim() || !clientSecret.trim()) return; + setSavingCreds(true); + try { + const res = await fetch(apiUrl('/integrations/google/credentials'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ clientId: clientId.trim(), clientSecret: clientSecret.trim() }), + }); + const data = await res.json(); + if (!res.ok) { + setToast({ type: 'error', message: data.error || 'Save failed' }); + } else { + setStatus(data.status); + setClientSecret(''); // clear secret from UI after save + setToast({ type: 'success', message: 'Client credentials saved.' }); + } + } catch (e) { + setToast({ type: 'error', message: e instanceof Error ? e.message : String(e) }); + } finally { + setSavingCreds(false); + } + }; + + const handleSaveProperties = async () => { + if (ga4PropertyId && !/^\d+$/.test(ga4PropertyId.trim())) { + setToast({ + type: 'error', + message: + 'Analytics property ID must be a numeric ID (e.g. 123456789). The G-XXXXXXX code is a Measurement ID -- find the numeric ID in GA4 Admin > Property Settings.', + }); + return; + } + setSavingProps(true); + try { + const res = await fetch(apiUrl('/integrations/google/credentials'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + gscSiteUrl: gscSiteUrl.trim() || null, + ga4PropertyId: ga4PropertyId.trim() || null, + dateRangeDays: Number(dateRangeDays) || 28, + }), + }); + const data = await res.json(); + if (!res.ok) { + setToast({ type: 'error', message: data.error || 'Save failed' }); + } else { + setStatus(data.status); + setToast({ type: 'success', message: 'Settings saved.' }); + } + } catch (e) { + setToast({ type: 'error', message: e instanceof Error ? e.message : String(e) }); + } finally { + setSavingProps(false); + } + }; + + const handleSaveRefreshToken = async () => { + if (!refreshToken.trim()) return; + setSavingToken(true); + try { + const res = await fetch(apiUrl('/integrations/google/credentials'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ refreshToken: refreshToken.trim() }), + }); + const data = await res.json(); + if (!res.ok) { + setToast({ type: 'error', message: data.error || 'Save failed' }); + } else { + setStatus(data.status); + setRefreshToken(''); + setToast({ type: 'success', message: 'Connection token saved.' }); + } + } catch (e) { + setToast({ type: 'error', message: e instanceof Error ? e.message : String(e) }); + } finally { + setSavingToken(false); + } + }; + + const handleTest = async () => { + setTesting(true); + setTestLog(''); + try { + const res = await fetch(apiUrl('/integrations/google/test'), { method: 'POST' }); + const data = await res.json(); + const log = data.log || (data.ok ? 'Test passed.' : 'Test failed.'); + setTestLog(log); + const hasIssues = log.includes('Google test completed with issues:'); + if (!data.ok) { + setToast({ + type: 'error', + message: hasIssues + ? 'Connection test found configuration issues — see log below.' + : 'Connection test failed — see log below.', + }); + } else { + setToast({ type: 'success', message: 'Connection test passed — GSC and GA4 are reachable.' }); + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + setTestLog(msg); + setToast({ type: 'error', message: msg }); + } finally { + setTesting(false); + } + }; + + useEffect(() => { + return () => { + fetchPollStopRef.current?.(); + fetchPollStopRef.current = null; + }; + }, []); + + const handleFetch = async () => { + setFetching(true); + setFetchLog('Starting Google data fetch…'); + setFetchJobStatus('running'); + fetchPollStopRef.current?.(); + fetchPollStopRef.current = null; + try { + const res = await fetch(apiUrl('/run'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: 'google' }), + }); + const data = await res.json(); + if (!res.ok) { + setFetchLog(`Error: ${data.error}`); + setFetchJobStatus('error'); + setToast({ type: 'error', message: data.error || 'Fetch failed' }); + } else { + const jobId = data.jobId; + setFetchLog(`Job ${jobId}\nStatus: running\n\nWaiting for output…`); + setToast({ + type: 'success', + message: 'Google fetch started — live log below and on the Pipeline page.', + }); + dispatchPipelineJobStarted(jobId, { command: 'google', openRunner: true }); + fetchPollStopRef.current = pollPipelineJob(jobId, (job) => { + const header = `Job ${jobId}\nStatus: ${job.status}\n`; + setFetchJobStatus(job.status); + setFetchLog(job.log ? `${header}\n${job.log}` : `${header}\nWaiting for output…`); + if (job.status === 'success') { + setToast({ type: 'success', message: 'Google data fetch completed.' }); + fetchStatus(); + report?.loadReport(); + } else if (job.status === 'error') { + setToast({ type: 'error', message: 'Google data fetch failed — see log below.' }); + } + }); + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + setFetchLog(msg); + setFetchJobStatus('error'); + setToast({ type: 'error', message: msg }); + } finally { + setFetching(false); + } + }; + + const handleDisconnect = async () => { + try { + await fetch(apiUrl('/integrations/google/disconnect'), { method: 'POST' }); + await fetchStatus(); + setToast({ type: 'success', message: 'Disconnected.' }); + } catch (e) { + setToast({ type: 'error', message: e instanceof Error ? e.message : String(e) }); + } + }; + + const hasClientId = status?.hasClientId; + const connected = status?.connected; + const step1Done = Boolean(hasClientId); + const step2Done = Boolean(connected); + + return ( +
+ {showTitle ? ( +
+
+ + + +
+

Google Integrations

+

+ Connect Search Console and GA4, then choose properties to sync with your reports. +

+
+
+ +
+ ) : null} + + {toast ? ( +
+ {toast.type === 'success' ? ( + + ) : ( + + )} + {toast.message} +
+ ) : null} + + {loadingStatus ? ( +
+ + Loading connection status… +
+ ) : ( + <> + +

+ Need a project?{' '} + + Google Cloud guide + +

+
+ + +
+
+ +
+ + + + {connected ? ( +
+ +
+

Account connected

+

You can configure properties in the next step.

+
+ +
+ ) : ( +
+ {!hasClientId ? ( +

Complete step 1 to enable sign-in.

+ ) : ( +
+ )} + + + {connected ? ( + +
+

Load sites from your connected account.

+ +
+ +
+
+ + {properties?.gscSites && properties.gscSites.length > 0 ? ( + + ) : ( + setGscSiteUrl(e.target.value)} + placeholder="https://www.example.com/" + className={`${selectClassName()} font-mono`} + /> + )} +
+ +
+ + {properties?.ga4Properties && properties.ga4Properties.length > 0 ? ( + + ) : ( + setGa4PropertyId(e.target.value)} + placeholder="123456789" + className={`${selectClassName()} font-mono`} + /> + )} + {properties?.ga4ListError ? ( +

{properties.ga4ListError}

+ ) : ( +

Numeric ID from GA4 Admin → Property settings.

+ )} +
+ +
+ + +
+
+ + {status?.lastFetchedAt ? ( +

+ Last fetched: {new Date(status.lastFetchedAt).toLocaleString()} +

+ ) : null} + +
+ + + +
+ + {testLog ? ( + 
+                  {testLog}
+
+ ) : null} + + {fetchLog ? ( +
+ {fetchJobStatus ? ( +

+ Fetch status:{' '} + + {fetchJobStatus} + + {fetchJobStatus === 'running' ? ( + + ) : null} +

+ ) : null} + 
+                    {fetchLog}
+
+
+ ) : null} + + ) : null} + +
+ + {showAdvanced ? ( +
+ + +
+ ) : null} +
+ + )} +
+ ); +} diff --git a/web/src/components/IntegrationsModal.jsx b/web/src/components/IntegrationsModal.jsx deleted file mode 100644 index b78c125e..00000000 --- a/web/src/components/IntegrationsModal.jsx +++ /dev/null @@ -1,731 +0,0 @@ -'use client'; - -import { useState, useEffect, useCallback, useRef } from 'react'; -import { - X, - Settings2, - CheckCircle2, - AlertCircle, - Loader2, - ChevronDown, - ChevronUp, - ExternalLink, -} from 'lucide-react'; -import { apiUrl } from '@/lib/publicBase'; -import { dispatchPipelineJobStarted, pollPipelineJob } from '@/lib/pipelineJobEvents'; -import { useReport } from '@/context/useReport'; - -const GCP_GUIDE_URL = - 'https://developers.google.com/workspace/guides/get-started'; - -function StatusPill({ connected }) { - if (connected) { - return ( - - - Connected - - ); - } - return ( - - - Not connected - - ); -} - -function InputField({ label, id, type = 'text', value, onChange, placeholder, helper, disabled }) { - return ( -
- - onChange(e.target.value)} - placeholder={placeholder} - disabled={disabled} - autoComplete="off" - className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground font-mono disabled:opacity-50" - /> - {helper &&

{helper}

} -
- ); -} - -/** - * Integrations modal for Google Search Console + GA4. - * Two-phase wizard: - * Phase 1: Paste GCP Client ID + Client Secret (one-time) - * Phase 2: Connect with Google button + property pickers - */ -export default function IntegrationsModal({ open, onClose, initialToast }) { - const { loadReport } = useReport(); - const [status, setStatus] = useState(null); - const [loadingStatus, setLoadingStatus] = useState(false); - - // Phase 1 fields - const [clientId, setClientId] = useState(''); - const [clientSecret, setClientSecret] = useState(''); - const [savingCreds, setSavingCreds] = useState(false); - const [credsSaved, setCredsSaved] = useState(false); - - // Phase 2 fields - const [gscSiteUrl, setGscSiteUrl] = useState(''); - const [ga4PropertyId, setGa4PropertyId] = useState(''); - const [dateRangeDays, setDateRangeDays] = useState('28'); - const [savingProps, setSavingProps] = useState(false); - - // Properties dropdowns - const [properties, setProperties] = useState(null); - const [loadingProps, setLoadingProps] = useState(false); - - // Test / Fetch - const [testLog, setTestLog] = useState(''); - const [testing, setTesting] = useState(false); - const [fetching, setFetching] = useState(false); - const [fetchLog, setFetchLog] = useState(''); - const [fetchJobStatus, setFetchJobStatus] = useState(''); - const fetchPollStopRef = useRef(null); - - // Advanced accordion (paste refresh token) - const [showAdvanced, setShowAdvanced] = useState(false); - const [refreshToken, setRefreshToken] = useState(''); - const [savingToken, setSavingToken] = useState(false); - - // Toast from OAuth callback - const [toast, setToast] = useState(initialToast || null); - - const fetchStatus = useCallback(async () => { - setLoadingStatus(true); - try { - const res = await fetch(apiUrl('/integrations/google/status')); - if (res.ok) { - const data = await res.json(); - setStatus(data); - if (data.gscSiteUrl) setGscSiteUrl(data.gscSiteUrl); - if (data.ga4PropertyId) setGa4PropertyId(data.ga4PropertyId); - if (data.dateRangeDays) setDateRangeDays(String(data.dateRangeDays)); - } - } catch { - // ignore - } finally { - setLoadingStatus(false); - } - }, []); - - useEffect(() => { - if (open) { - fetchStatus(); - } - }, [open, fetchStatus]); - - useEffect(() => { - if (initialToast) setToast(initialToast); - }, [initialToast]); - - // Auto-dismiss toast after 6s - useEffect(() => { - if (!toast) return; - const t = setTimeout(() => setToast(null), 6000); - return () => clearTimeout(t); - }, [toast]); - - const loadProperties = async () => { - setLoadingProps(true); - try { - const res = await fetch(apiUrl('/integrations/google/properties')); - if (res.ok) { - const data = await res.json(); - setProperties(data); - if (data.ga4ListError) { - setToast({ type: 'error', message: data.ga4ListError }); - } - } - } catch { - // ignore - } finally { - setLoadingProps(false); - } - }; - - const handleSaveClientCreds = async () => { - if (!clientId.trim() || !clientSecret.trim()) return; - setSavingCreds(true); - try { - const res = await fetch(apiUrl('/integrations/google/credentials'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ clientId: clientId.trim(), clientSecret: clientSecret.trim() }), - }); - const data = await res.json(); - if (!res.ok) { - setToast({ type: 'error', message: data.error || 'Save failed' }); - } else { - setStatus(data.status); - setCredsSaved(true); - setClientSecret(''); // clear secret from UI after save - setToast({ type: 'success', message: 'Client credentials saved.' }); - } - } catch (e) { - setToast({ type: 'error', message: e.message }); - } finally { - setSavingCreds(false); - } - }; - - const handleSaveProperties = async () => { - if (ga4PropertyId && !/^\d+$/.test(ga4PropertyId.trim())) { - setToast({ - type: 'error', - message: - 'Analytics property ID must be a numeric ID (e.g. 123456789). The G-XXXXXXX code is a Measurement ID -- find the numeric ID in GA4 Admin > Property Settings.', - }); - return; - } - setSavingProps(true); - try { - const res = await fetch(apiUrl('/integrations/google/credentials'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - gscSiteUrl: gscSiteUrl.trim() || null, - ga4PropertyId: ga4PropertyId.trim() || null, - dateRangeDays: Number(dateRangeDays) || 28, - }), - }); - const data = await res.json(); - if (!res.ok) { - setToast({ type: 'error', message: data.error || 'Save failed' }); - } else { - setStatus(data.status); - setToast({ type: 'success', message: 'Settings saved.' }); - } - } catch (e) { - setToast({ type: 'error', message: e.message }); - } finally { - setSavingProps(false); - } - }; - - const handleSaveRefreshToken = async () => { - if (!refreshToken.trim()) return; - setSavingToken(true); - try { - const res = await fetch(apiUrl('/integrations/google/credentials'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ refreshToken: refreshToken.trim() }), - }); - const data = await res.json(); - if (!res.ok) { - setToast({ type: 'error', message: data.error || 'Save failed' }); - } else { - setStatus(data.status); - setRefreshToken(''); - setToast({ type: 'success', message: 'Connection token saved.' }); - } - } catch (e) { - setToast({ type: 'error', message: e.message }); - } finally { - setSavingToken(false); - } - }; - - const handleFileUpload = async (e) => { - const file = e.target.files?.[0]; - if (!file) return; - try { - const text = await file.text(); - const res = await fetch(apiUrl('/integrations/google/credentials/upload'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ fileContent: text }), - }); - const data = await res.json(); - if (!res.ok) { - setToast({ type: 'error', message: data.error || 'Upload failed' }); - } else { - setStatus(data.status); - setToast({ type: 'success', message: 'Service account key uploaded.' }); - } - } catch (e) { - setToast({ type: 'error', message: e.message }); - } - e.target.value = ''; - }; - - const handleTest = async () => { - setTesting(true); - setTestLog(''); - try { - const res = await fetch(apiUrl('/integrations/google/test'), { method: 'POST' }); - const data = await res.json(); - const log = data.log || (data.ok ? 'Test passed.' : 'Test failed.'); - setTestLog(log); - const hasIssues = log.includes('Google test completed with issues:'); - if (!data.ok) { - setToast({ - type: 'error', - message: hasIssues - ? 'Connection test found configuration issues — see log below.' - : 'Connection test failed — see log below.', - }); - } else { - setToast({ type: 'success', message: 'Connection test passed — GSC and GA4 are reachable.' }); - } - } catch (e) { - setTestLog(e.message); - setToast({ type: 'error', message: e.message }); - } finally { - setTesting(false); - } - }; - - useEffect(() => { - return () => { - fetchPollStopRef.current?.(); - fetchPollStopRef.current = null; - }; - }, []); - - const handleFetch = async () => { - setFetching(true); - setFetchLog('Starting Google data fetch…'); - setFetchJobStatus('running'); - fetchPollStopRef.current?.(); - fetchPollStopRef.current = null; - try { - const res = await fetch(apiUrl('/run'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: 'google' }), - }); - const data = await res.json(); - if (!res.ok) { - setFetchLog(`Error: ${data.error}`); - setFetchJobStatus('error'); - setToast({ type: 'error', message: data.error || 'Fetch failed' }); - } else { - const jobId = data.jobId; - setFetchLog(`Job ${jobId}\nStatus: running\n\nWaiting for output…`); - setToast({ - type: 'success', - message: 'Google fetch started — live log below and in Pipeline Runner (bottom-right).', - }); - dispatchPipelineJobStarted(jobId, { command: 'google', openRunner: true }); - fetchPollStopRef.current = pollPipelineJob(jobId, (job) => { - const header = `Job ${jobId}\nStatus: ${job.status}\n`; - setFetchJobStatus(job.status); - setFetchLog(job.log ? `${header}\n${job.log}` : `${header}\nWaiting for output…`); - if (job.status === 'success') { - setToast({ type: 'success', message: 'Google data fetch completed.' }); - fetchStatus(); - loadReport(); - } else if (job.status === 'error') { - setToast({ type: 'error', message: 'Google data fetch failed — see log below.' }); - } - }); - } - } catch (e) { - setFetchLog(e.message); - setFetchJobStatus('error'); - setToast({ type: 'error', message: e.message }); - } finally { - setFetching(false); - } - }; - - const handleDisconnect = async () => { - try { - await fetch(apiUrl('/integrations/google/disconnect'), { method: 'POST' }); - await fetchStatus(); - setToast({ type: 'success', message: 'Disconnected.' }); - } catch (e) { - setToast({ type: 'error', message: e.message }); - } - }; - - if (!open) return null; - - const hasClientId = status?.hasClientId; - const connected = status?.connected; - - return ( -
-
- {/* Header */} -
-
- -

Google Integrations

-
-
- - -
-
- - {/* Toast */} - {toast && ( -
- {toast.type === 'success' ? ( - - ) : ( - - )} - {toast.message} -
- )} - - {/* Body */} -
- {loadingStatus ? ( -
- - Loading... -
- ) : ( - <> - {/* ── Phase 1: GCP client credentials ── */} -
-
-

- Step 1: Google Cloud credentials (one time) -

- {hasClientId && ( - - Saved - - )} -
-

- You need a{' '} - - Google Cloud project - {' '} - with Search Console API and Analytics Data API enabled. Ask your developer or follow the guide. -

-
- - -
-
- - OR - -
-
- - {/* ── Phase 2: Connect + properties ── */} -
-

- Step 2: Connect your Google account -

- {connected ? ( -
- - Connected - -
- ) : ( - - - Connect with Google - - )} - {!hasClientId && ( -

- Complete Step 1 first to enable this button. -

- )} -
- - {/* Properties + date range (shown when connected) */} - {connected && ( -
-
-

Properties

- -
- - {/* GSC site */} -
- - {properties?.gscSites?.length > 0 ? ( - - ) : ( - setGscSiteUrl(e.target.value)} - placeholder="https://www.example.com/" - className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground font-mono" - /> - )} -
- - {/* GA4 property */} -
- - {properties?.ga4Properties?.length > 0 ? ( - - ) : ( - setGa4PropertyId(e.target.value)} - placeholder="123456789" - className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground font-mono" - /> - )} - {properties?.ga4ListError && ( -

- {properties.ga4ListError} -

- )} -

- Numeric ID from GA4 Admin > Property Settings (not the G-XXXXXXX Measurement ID). -

-
- - {/* Date range */} -
- - -
- - - - {/* Actions */} -
- - -
- - {testLog && ( - 
-                      {testLog}
-
- )} - {fetchLog && ( -
- {fetchJobStatus && ( -

- Status:{' '} - - {fetchJobStatus} - - {fetchJobStatus === 'running' && ( - - - running - - )} -

- )} - 
-                        {fetchLog}
-
-

- Full live log also opens in Pipeline Runner (blue terminal button, bottom-right). -

-
- )} -
- )} - - {/* Last fetched */} - {status?.lastFetchedAt && ( -

- Last fetched: {new Date(status.lastFetchedAt).toLocaleString()} -

- )} - - {/* ── Advanced: paste refresh token ── */} -
- - {showAdvanced && ( -
- - -
- )} -
- - )} -
-
-
- ); -} diff --git a/web/src/components/IntegrationsModal.tsx b/web/src/components/IntegrationsModal.tsx new file mode 100644 index 00000000..ae36fec9 --- /dev/null +++ b/web/src/components/IntegrationsModal.tsx @@ -0,0 +1,43 @@ +'use client'; + +import { Settings2, X } from 'lucide-react'; +import type { IntegrationToast } from '@/types/api'; +import GoogleIntegrationsPanel from './GoogleIntegrationsPanel'; + +export interface IntegrationsModalProps { + open: boolean; + onClose: () => void; + initialToast?: IntegrationToast | null; +} + +/** Modal wrapper around {@link GoogleIntegrationsPanel} for report views. */ +export default function IntegrationsModal({ open, onClose, initialToast }: IntegrationsModalProps) { + if (!open) return null; + + return ( +
+
+
+
+ +

Google Integrations

+
+ +
+
+ +
+
+
+ ); +} diff --git a/web/src/components/PageHeader.jsx b/web/src/components/PageHeader.jsx deleted file mode 100644 index f5005fb1..00000000 --- a/web/src/components/PageHeader.jsx +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Consistent page title and optional subtitle. - */ -export default function PageHeader({ title, subtitle }) { - return ( -
-

{title}

- {subtitle &&

{subtitle}

} -
- ); -} diff --git a/web/src/components/PageHeader.tsx b/web/src/components/PageHeader.tsx new file mode 100644 index 00000000..ca047d0a --- /dev/null +++ b/web/src/components/PageHeader.tsx @@ -0,0 +1,19 @@ +/** + * Consistent page title and optional subtitle. + */ +import type { ReactNode } from 'react'; + +export default function PageHeader({ + title, + subtitle, +}: { + title: string; + subtitle?: ReactNode; +}) { + return ( +
+

{title}

+ {subtitle ?
{subtitle}
: null} +
+ ); +} diff --git a/web/src/components/PageLayout.jsx b/web/src/components/PageLayout.tsx similarity index 60% rename from web/src/components/PageLayout.jsx rename to web/src/components/PageLayout.tsx index 65bb02a9..d04073af 100644 --- a/web/src/components/PageLayout.jsx +++ b/web/src/components/PageLayout.tsx @@ -1,7 +1,17 @@ +import type { ReactNode } from 'react'; + /** * Page wrapper with consistent padding. Optional max-width for focused views (e.g. Lighthouse). */ -export default function PageLayout({ children, className = '', maxWidth = false }) { +export default function PageLayout({ + children, + className = '', + maxWidth = false, +}: { + children?: ReactNode; + className?: string; + maxWidth?: boolean; +}) { const maxWidthClass = maxWidth ? 'max-w-6xl mx-auto' : ''; return (
diff --git a/web/src/components/PipelineRunnerFab.jsx b/web/src/components/PipelineRunnerFab.jsx deleted file mode 100644 index f286d6d5..00000000 --- a/web/src/components/PipelineRunnerFab.jsx +++ /dev/null @@ -1,871 +0,0 @@ -'use client'; - -import { useState, useRef, useEffect, useCallback, useMemo } from 'react'; -import { Loader2, Maximize2, Minimize2, Terminal, X, Save } from 'lucide-react'; -import { apiUrl } from '@/lib/publicBase'; -import { PIPELINE_JOB_STARTED, pollPipelineJob } from '@/lib/pipelineJobEvents'; -import { useReport } from '@/context/useReport'; -import { strings } from '@/lib/strings'; -import { - PIPELINE_CONFIG_SECTIONS, - buildInitialPipelineConfigState, - validatePipelineRun, -} from '@/lib/pipelineConfigSchema'; -import { - LLM_CONFIG_SECTIONS, - buildInitialLlmConfigState, -} from '@/lib/llmConfigSchema'; - -const COMMANDS = [ - { value: '', label: 'Full pipeline (per form config)' }, - { value: 'crawl', label: 'crawl' }, - { value: 'report', label: 'report' }, - { value: 'plot', label: 'plot' }, - { value: 'lighthouse', label: 'lighthouse' }, - { value: 'keywords', label: 'keywords' }, - { value: 'warnings', label: 'warnings' }, - { value: 'enrich', label: 'enrich (analysis + AI)' }, - { value: 'google', label: 'google (fetch GSC & GA4)' }, - { value: 'keywords --enrich-google', label: 'keywords --enrich-google (Keywords Explorer)' }, -]; - -const TAB_RUN = 'run'; -const TAB_AI = 'ai'; -const TAB_OTHER = 'other'; - -// Build tab list: de-dupe section ids (safety), AI tab, then Run -function buildMainTabs(unknownKeys) { - const seen = new Set(); - const tabs = []; - for (const s of PIPELINE_CONFIG_SECTIONS) { - if (!seen.has(s.id)) { - seen.add(s.id); - tabs.push({ id: s.id, label: s.label }); - } - } - tabs.push({ id: TAB_AI, label: 'AI' }); - tabs.push({ id: TAB_RUN, label: 'Run' }); - if (unknownKeys.length > 0) { - tabs.push({ id: TAB_OTHER, label: 'Other' }); - } - return tabs; -} - -// ─── ConfigField ───────────────────────────────────────────────────────────── - -/** Single config row for any field type. */ -function ConfigField({ field: f, value, disabled, onChange }) { - const id = `pipe-cfg-${f.key}`; - - const helpEl = f.help ? ( -

{f.help}

- ) : null; - - if (f.type === 'select') { - const strVal = value == null ? String(f.defaultValue ?? '') : String(value); - return ( -
- - - {helpEl} -
- ); - } - - if (f.type === 'secret') { - const strVal = value == null ? '' : String(value); - const placeholder = strVal.startsWith('••••') ? strVal : 'Paste API key (optional if env var set)'; - return ( -
- - onChange(e.target.value)} - className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground font-mono" - /> - {helpEl} -
- ); - } - - if (f.type === 'bool') { - const checked = value === true; - return ( -
- - {helpEl} -
- ); - } - - if (f.type === 'tristate') { - const strVal = value == null ? 'auto' : String(value); - return ( -
- - - {helpEl} -
- ); - } - - if (f.type === 'textarea') { - const strVal = value == null ? '' : String(value); - return ( -
- -