diff --git a/.coverage b/.coverage
index 91ad01af..c3abd146 100644
Binary files a/.coverage and b/.coverage differ
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0bed9188..4901e7f4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,15 +39,17 @@ jobs:
run: |
pytest tests/test_categories_roadmap.py tests/test_report_categories_golden.py \
tests/test_categories_coverage.py tests/test_indexation_coverage.py tests/test_crawl_segments.py \
- tests/test_terminology.py \
+ tests/test_terminology.py tests/test_compare_payload.py \
--cov=website_profiling.reporting --cov-config=.coveragerc.reporting \
--cov-report=term-missing --cov-fail-under=100 -q -o addopts=
- name: Pytest (tools coverage gate)
run: |
pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \
- tests/test_export_audit_coverage.py \
+ tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \
+ tests/test_audit_tools_coverage.py \
+ tests/test_mcp_registry.py tests/test_mcp_resources.py \
--cov=website_profiling.tools --cov-config=.coveragerc.tools \
- --cov-report=term-missing --cov-fail-under=100 -q -o addopts=
+ --cov-report=term-missing --cov-fail-under=95 -q -o addopts=
- name: CLI smoke
run: python -m src --help
diff --git a/AGENT.md b/AGENT.md
index b4b3264e..55c5d8da 100644
--- a/AGENT.md
+++ b/AGENT.md
@@ -19,12 +19,14 @@
**Run / APIs**
- Run audit (CLI): `python -m src` — reads config from PostgreSQL (`pipeline_config`); shadow `DATA_DIR/pipeline-config.txt` if table empty. CLI override: `python -m src --config path`
-- Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` | `google`
+- Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` | `google` | `chat`
- **`preserve_crawl_history`** (default true): append crawls; `false` truncates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs`
- **`DATABASE_URL`** env: PostgreSQL connection string (required). **`DATA_DIR`**: secrets + shadow config (Docker: `/data`).
- **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline.
- **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch.
-- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run
+- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run
+- **MCP:** `python -m website_profiling.mcp` (stdio, **121 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements-mcp.txt`.
+- **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`).
- **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only).
- **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
@@ -36,7 +38,8 @@
| Report | `reporting/builder.py`, `reporting/categories.py` |
| DB schema | `alembic/versions/` |
| Local analysis | `analysis/local.py`, `requirements.txt` |
-| AI insights (LLM) | `llm/enrich.py`, `llm_config.py`, `requirements-llm.txt` |
+| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements-llm.txt` |
+| Audit query tools (MCP + chat) | `tools/audit_tools/`, `mcp/server.py`, `commands/chat_cmd.py` |
| Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` |
| UI pipeline schema | `web/src/lib/pipelineConfigSchema.ts` |
| UI LLM schema | `web/src/lib/llmConfigSchema.ts` |
diff --git a/README.md b/README.md
index 377e7826..3bdf5895 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,15 @@ Google Search Console / Analytics: connect via **Integrations** (gear icon) in t
**JavaScript crawl (optional):** In Audit settings, set **Crawl rendering** to `javascript` (always headless Chromium) or `auto` (static first, browser when SPA heuristics match). Install locally: `pip install -r requirements-browser.txt` and Chromium on `PATH` or `CHROME_PATH` (included in Docker). The UI preflights via `GET /api/crawl/browser-status` before runs when JS/auto is selected.
+**AI Chat (optional):** Ask questions about your audit data at [http://localhost:3000/chat](http://localhost:3000/chat). Enable a provider under **Run audit → AI settings** (`llm_enabled`, provider, model). `./local-run setup` installs `requirements-llm.txt` (`httpx`, OpenAI, Anthropic SDKs).
+
+| Provider | Notes |
+|----------|--------|
+| **Ollama** | Local daemon at `http://127.0.0.1:11434`. Chat UI lists installed models plus the live Ollama cloud catalog (billing: free local, account free tier, Pro). Native tool calling when supported; otherwise ReAct fallback. Pick the model in-chat without leaving the page. |
+| **OpenAI** / **Anthropic** | API key in AI settings; native tool calling with streaming. |
+
+The agent uses the same **121 read-only audit tools** as the MCP server (`docs/MCP.md`). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`).
+
Production: `docker-compose.prod.yml` (set `POSTGRES_PASSWORD`, `AUTH_SECRET`).
## License
diff --git a/alembic/versions/012_chat_sessions.py b/alembic/versions/012_chat_sessions.py
new file mode 100644
index 00000000..368b7551
--- /dev/null
+++ b/alembic/versions/012_chat_sessions.py
@@ -0,0 +1,50 @@
+"""Chat sessions and messages for in-app AI chat.
+
+Revision ID: 012_chat_sessions
+Revises: 011_roadmap_foundation
+"""
+from __future__ import annotations
+
+from alembic import op
+
+revision = "012_chat_sessions"
+down_revision = "011_roadmap_foundation"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.execute("""
+ CREATE TABLE chat_sessions (
+ id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ property_id BIGINT NOT NULL REFERENCES properties(id) ON DELETE CASCADE,
+ title TEXT NOT NULL DEFAULT 'New chat',
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+ )
+ """)
+ op.execute("""
+ CREATE TABLE chat_messages (
+ id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ session_id BIGINT NOT NULL REFERENCES chat_sessions(id) ON DELETE CASCADE,
+ role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'tool')),
+ content TEXT NOT NULL DEFAULT '',
+ tool_name TEXT,
+ tool_args JSONB,
+ tool_result JSONB,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+ )
+ """)
+ op.execute("""
+ CREATE INDEX chat_sessions_property_updated_idx
+ ON chat_sessions(property_id, updated_at DESC)
+ """)
+ op.execute("""
+ CREATE INDEX chat_messages_session_created_idx
+ ON chat_messages(session_id, created_at)
+ """)
+
+
+def downgrade() -> None:
+ op.execute("DROP TABLE IF EXISTS chat_messages")
+ op.execute("DROP TABLE IF EXISTS chat_sessions")
diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md
index d62b907d..64ae9695 100644
--- a/docs/GLOSSARY.md
+++ b/docs/GLOSSARY.md
@@ -40,6 +40,8 @@ UI terms agencies recognize, mapped to internal keys and data sources.
| Property alerts | `alert_webhook_url`, `/api/alerts/check` | Health snapshot rules | Ops notifications |
| Content brief | Keywords Brief button, `/api/keywords/content-brief` | LLM or deterministic | Content planning |
| AI issue fix | `llm_recommendation`, `/api/issues/fix-suggestion` | LLM on demand + report build | Actionable remediation |
+| AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational site audit queries |
+| MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | Cursor / Claude Desktop integration — see [MCP.md](MCP.md) |
| Read-only session | `AUTH_DEFAULT_ROLE=client-readonly`, `/api/auth/session` | Session cookie | Client view-only access |
| Export executive summary | `export_audit_html/pdf/csv`, `executive_summary` | Report payload + optional AI | Client deliverable |
diff --git a/docs/MCP.md b/docs/MCP.md
new file mode 100644
index 00000000..c499d43d
--- /dev/null
+++ b/docs/MCP.md
@@ -0,0 +1,120 @@
+# Site Audit MCP server
+
+Read-only [Model Context Protocol](https://modelcontextprotocol.io) tools for querying audit data from Cursor, Claude Desktop, or other MCP clients.
+
+## Install
+
+```bash
+pip install -r requirements-mcp.txt
+export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling
+export PYTHONPATH=src
+```
+
+## Cursor configuration
+
+Add to `.cursor/mcp.json` (or Cursor MCP settings):
+
+```json
+{
+ "mcpServers": {
+ "site-audit": {
+ "command": "python",
+ "args": ["-m", "website_profiling.mcp"],
+ "env": {
+ "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
+ "PYTHONPATH": "src",
+ "WP_PROPERTY_ID": "1"
+ }
+ }
+ }
+}
+```
+
+`WP_PROPERTY_ID` sets the default property when tools omit `property_id`.
+
+## MCP resources
+
+| URI | Content |
+|-----|---------|
+| `audit://properties` | JSON list of properties |
+| `audit://property/{id}` | Property details + latest report summary |
+| `audit://property/{id}/report/latest` | Payload key index (counts, not full blob) |
+| `audit://property/{id}/report/{report_id}` | Payload key index for a specific report |
+| `audit://glossary` | Excerpt from `docs/GLOSSARY.md` |
+| `audit://tools` | Tool catalog grouped by SEO domain |
+
+## Tools (121 read-only)
+
+### Portfolio and report
+
+`list_properties`, `get_property`, `get_report_summary`, `get_category_scores`, `get_executive_summary`, `get_report_meta`, `get_site_level`, `list_report_history`, `get_audit_recommendations`, `get_ml_errors`, `get_ssl_expiry_info`, `list_audit_categories`, `get_category_recommendations`, `get_crawl_summary`
+
+### Issues and workflow
+
+`list_issues`, `list_issues_by_category`, `get_category_issues`, `list_issue_workflow`, `list_issues_with_ai_fixes`, `list_seo_onpage_issues`
+
+### On-page SEO
+
+`list_content_url_issues`, `list_pages_missing_title`, `list_pages_missing_h1`, `list_pages_multiple_h1`, `list_pages_missing_meta_description`, `list_pages_meta_desc_too_short`, `list_pages_meta_desc_too_long`, `list_pages_noindex`, `get_seo_health`
+
+### Crawl and pages
+
+`search_pages`, `search_pages_advanced`, `get_page_details`, `get_page_analysis`, `get_internal_links`, `list_redirects`, `list_broken_links`, `list_status_4xx_pages`, `list_status_5xx_pages`, `get_status_code_breakdown`, `get_response_time_stats`, `get_depth_distribution`, `get_crawl_segments`, `get_browser_diagnostics_summary`, `list_pages_with_console_errors`, `list_pages_by_fetch_method`, `get_crawl_links_table`, `get_graph_edges_sample`
+
+### Schema and technical
+
+`get_schema_coverage`, `list_pages_without_schema`, `search_pages_by_schema_type`, `get_tech_stack_summary`, `get_security_findings`
+
+### Links and architecture
+
+`list_orphan_pages`, `get_top_linked_pages`, `get_top_crawled_pages`, `get_outbound_link_domains`, `get_link_graph_summary`, `get_url_fingerprints`, `get_mime_type_breakdown`, `get_title_length_distribution`, `get_domain_link_distribution`, `get_outlink_distribution`
+
+### Indexation and international
+
+`get_indexation_coverage`, `list_indexation_gaps`, `get_indexation_url_join`, `get_hreflang_summary`, `get_language_summary`
+
+### Content and social
+
+`get_content_analytics`, `get_content_duplicates`, `get_social_coverage`, `get_keyword_opportunities`, `get_ner_site_summary`, `list_thin_content_pages`
+
+### Keywords
+
+`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`
+
+### Google
+
+`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`
+
+### Backlinks
+
+`get_gsc_links_summary`, `get_gsc_links_import_status`, `get_gsc_sample_links`, `get_gsc_latest_links`, `get_third_party_links_overlay`, `get_backlinks_velocity`, `get_competitor_link_gap`, `get_bing_backlinks_summary`
+
+### Performance
+
+`get_lighthouse_summary`, `get_lighthouse_for_url`, `get_lighthouse_human_summary`, `get_lighthouse_diagnostics`, `get_crux_summary`, `list_slow_pages`, `list_lighthouse_poor_seo_pages`
+
+### Drift, health, and compare
+
+`get_health_history`, `get_category_health_history`, `compare_reports`, `compare_issue_deltas`, `compare_category_deltas`, `compare_seo_health_deltas`, `compare_lighthouse_deltas`, `compare_url_set_diff`, `compare_redirect_deltas`, `compare_link_metric_deltas`
+
+### Ops and logs
+
+`get_integration_alerts`, `get_property_ops`, `list_crawl_runs`, `list_log_uploads`, `get_latest_log_analysis`
+
+## Example prompts
+
+- "What indexation gaps exist between crawl and GSC?"
+- "List pages missing titles or meta descriptions"
+- "Show backlinks velocity and third-party overlay data"
+- "List keyword cannibalisation issues"
+- "Compare the latest audit to report ID 38 — what URLs were added or removed?"
+- "Which pages have JS console errors or poor Lighthouse SEO scores?"
+- "What does the latest access log analysis show?"
+
+## In-app chat
+
+The same tools power **AI Chat** at [http://localhost:3000/chat](http://localhost:3000/chat). Enable AI in Run audit → AI settings.
+
+## Ollama note
+
+When the local Ollama daemon supports native tools (most current models, including Ollama cloud refs like `minimax-m3:cloud`), chat uses Ollama’s `/api/chat` tool format. Older or tool-less models fall back to JSON ReAct parsing. OpenAI and Anthropic always use native tool calling with streaming in the chat UI.
diff --git a/local-run.ps1 b/local-run.ps1
new file mode 100644
index 00000000..ae34f5dd
--- /dev/null
+++ b/local-run.ps1
@@ -0,0 +1 @@
+& "$PSScriptRoot\scripts\local-run.ps1" @args
diff --git a/local-test.ps1 b/local-test.ps1
new file mode 100644
index 00000000..f7180beb
--- /dev/null
+++ b/local-test.ps1
@@ -0,0 +1,2 @@
+# Wrapper — run from repo root: .\local-test.ps1 [command]
+& "$PSScriptRoot\scripts\local-test.ps1" @args
diff --git a/requirements-mcp.txt b/requirements-mcp.txt
new file mode 100644
index 00000000..8a09cd6f
--- /dev/null
+++ b/requirements-mcp.txt
@@ -0,0 +1,2 @@
+# MCP server for Cursor / Claude Desktop (optional)
+mcp>=1.0.0
diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1
new file mode 100644
index 00000000..95b26265
--- /dev/null
+++ b/scripts/local-run.ps1
@@ -0,0 +1,320 @@
+# Local dev: PostgreSQL in Docker (wp-pg), Python venv + Next.js on the host.
+# Usage: .\local-run.ps1 [command]
+# (default) start — ensure DB, migrations, npm run dev
+# setup — DB + venv + deps + migrations (no web server)
+# db — start Postgres container only
+# migrate — alembic upgrade head
+# stop — stop wp-pg container
+# help — show commands
+# Requires: PowerShell 5.1+ (PowerShell 7+ recommended for reliable exit codes)
+
+$ErrorActionPreference = "Stop"
+
+$ROOT = Split-Path -Parent $PSScriptRoot
+Set-Location $ROOT
+
+$PG_CONTAINER = if ($env:WP_PG_CONTAINER) { $env:WP_PG_CONTAINER } else { "wp-pg" }
+$PG_IMAGE = if ($env:WP_PG_IMAGE) { $env:WP_PG_IMAGE } else { "postgres:16-alpine" }
+$PG_PORT = if ($env:WP_PG_PORT) { $env:WP_PG_PORT } else { "5432" }
+$PG_USER = if ($env:WP_PG_USER) { $env:WP_PG_USER } else { "postgres" }
+$PG_PASSWORD = if ($env:WP_PG_PASSWORD) { $env:WP_PG_PASSWORD } else { "dev" }
+$PG_DB = if ($env:WP_PG_DB) { $env:WP_PG_DB } else { "website_profiling" }
+
+if (-not $env:DATABASE_URL) {
+ $env:DATABASE_URL = "postgres://${PG_USER}:${PG_PASSWORD}@127.0.0.1:${PG_PORT}/${PG_DB}"
+}
+if (-not $env:DATA_DIR) {
+ $env:DATA_DIR = Join-Path $ROOT "data"
+}
+
+$VENV = Join-Path $ROOT ".venv"
+$VENV_PYTHON = Join-Path $VENV "Scripts\python.exe"
+$VENV_PIP = Join-Path $VENV "Scripts\pip.exe"
+$VENV_ALEMBIC = Join-Path $VENV "Scripts\alembic.exe"
+$WEB = Join-Path $ROOT "web"
+
+$env:WEBSITE_PROFILING_ROOT = $ROOT
+if ($env:PYTHONPATH) {
+ $env:PYTHONPATH = "$($env:PYTHONPATH);$(Join-Path $ROOT 'src')"
+} else {
+ $env:PYTHONPATH = Join-Path $ROOT "src"
+}
+if (-not $env:PYTHON) {
+ $env:PYTHON = $VENV_PYTHON
+}
+
+function Write-Log([string]$Message) {
+ Write-Host "-> $Message" -ForegroundColor Cyan
+}
+
+function Write-Warn([string]$Message) {
+ Write-Host "! $Message" -ForegroundColor Yellow
+}
+
+function Write-Die([string]$Message) {
+ Write-Host "X $Message" -ForegroundColor Red
+ exit 1
+}
+
+function Assert-LastExitCode([string]$Message) {
+ $failed = $false
+ if ($PSVersionTable.PSVersion.Major -ge 7) {
+ $failed = ($LASTEXITCODE -ne 0)
+ } else {
+ $failed = (-not $?)
+ }
+ if ($failed) {
+ Write-Die $Message
+ }
+}
+
+function Test-Command([string]$Name) {
+ if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) {
+ Write-Die "Missing required command: $Name"
+ }
+}
+
+function Get-PythonLauncher {
+ foreach ($cmd in @("python", "python3", "py")) {
+ if (Get-Command $cmd -ErrorAction SilentlyContinue) {
+ if ($cmd -eq "py") {
+ return ,@("py", "-3")
+ }
+ return ,@($cmd)
+ }
+ }
+ Write-Die "Missing required command: python (install Python 3.11+ and ensure it is on PATH)"
+}
+
+function Invoke-PythonLauncher {
+ param(
+ [Parameter(Mandatory = $true)]
+ [string[]]$Launcher,
+ [Parameter(ValueFromRemainingArguments = $true)]
+ [string[]]$PythonArgs
+ )
+
+ if ($Launcher.Count -gt 1) {
+ & $Launcher[0] $Launcher[1] @PythonArgs
+ } else {
+ & $Launcher[0] @PythonArgs
+ }
+ Assert-LastExitCode "Python command failed: $($Launcher -join ' ') $($PythonArgs -join ' ')"
+}
+
+function Get-DockerContainerNames {
+ param([switch]$All)
+
+ $dockerArgs = if ($All) { @("ps", "-a") } else { @("ps") }
+ $output = & docker @dockerArgs --format "{{.Names}}" 2>$null
+ if (-not $output) {
+ return @()
+ }
+ # Force array: a single container name returns a scalar string; -contains on a
+ # string checks characters, not whole names.
+ return @($output | ForEach-Object { "$_".Trim() } | Where-Object { $_ })
+}
+
+function Test-DockerRunning {
+ Test-Command docker
+ # Docker Desktop writes capability warnings to stderr; avoid treating them as terminating errors.
+ $prevErrorAction = $ErrorActionPreference
+ $ErrorActionPreference = "Continue"
+ try {
+ cmd /c "docker info >nul 2>&1"
+ } finally {
+ $ErrorActionPreference = $prevErrorAction
+ }
+ Assert-LastExitCode "Docker is not running. Start Docker Desktop, then retry."
+}
+
+function Test-ContainerExists([string]$Name) {
+ return (Get-DockerContainerNames -All) -contains $Name
+}
+
+function Test-ContainerRunning([string]$Name) {
+ return (Get-DockerContainerNames) -contains $Name
+}
+
+function Wait-ForPostgres {
+ for ($i = 1; $i -le 30; $i++) {
+ & docker exec $PG_CONTAINER pg_isready -U $PG_USER -d $PG_DB *> $null
+ if ($PSVersionTable.PSVersion.Major -ge 7) {
+ if ($LASTEXITCODE -eq 0) { return }
+ } elseif ($?) {
+ return
+ }
+ Start-Sleep -Seconds 1
+ }
+ Write-Die "Postgres did not become ready in time (container: $PG_CONTAINER)"
+}
+
+function Invoke-Db {
+ Test-DockerRunning
+ if (Test-ContainerExists $PG_CONTAINER) {
+ if (Test-ContainerRunning $PG_CONTAINER) {
+ Write-Log "Postgres already running ($PG_CONTAINER)"
+ } else {
+ Write-Log "Starting existing container $PG_CONTAINER"
+ & docker start $PG_CONTAINER *> $null
+ Assert-LastExitCode "Failed to start container $PG_CONTAINER"
+ }
+ } else {
+ Write-Log "Creating Postgres container $PG_CONTAINER on port $PG_PORT"
+ & docker run -d --name $PG_CONTAINER `
+ -e "POSTGRES_PASSWORD=$PG_PASSWORD" `
+ -e "POSTGRES_DB=$PG_DB" `
+ -p "${PG_PORT}:5432" `
+ $PG_IMAGE *> $null
+ Assert-LastExitCode "Failed to create Postgres container $PG_CONTAINER"
+ }
+ Wait-ForPostgres
+ Write-Log "DATABASE_URL=$($env:DATABASE_URL)"
+}
+
+function Invoke-Venv {
+ $pyLauncher = Get-PythonLauncher
+ if (-not (Test-Path $VENV_PYTHON)) {
+ Write-Log "Creating Python venv at .venv"
+ Invoke-PythonLauncher -Launcher $pyLauncher -PythonArgs @("-m", "venv", $VENV)
+ }
+ Write-Log "Installing Python dependencies"
+ & $VENV_PIP install -q -r (Join-Path $ROOT "requirements.txt")
+ Assert-LastExitCode "Failed to install requirements.txt"
+ & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-browser.txt")
+ Assert-LastExitCode "Failed to install requirements-browser.txt"
+ & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-llm.txt")
+ Assert-LastExitCode "Failed to install requirements-llm.txt"
+}
+
+function Invoke-Migrate {
+ Invoke-Db
+ if (-not (Test-Path $VENV_ALEMBIC)) {
+ Invoke-Venv
+ }
+ Write-Log "Applying database migrations (alembic upgrade head)"
+ & $VENV_ALEMBIC upgrade head
+ Assert-LastExitCode "Database migration failed (alembic upgrade head)"
+}
+
+function Invoke-WebDeps {
+ Test-Command npm
+ $nodeModules = Join-Path $WEB "node_modules"
+ if (-not (Test-Path $nodeModules)) {
+ Write-Log "Installing web dependencies (npm ci)"
+ Push-Location $WEB
+ try {
+ & npm ci
+ Assert-LastExitCode "Failed to install web dependencies (npm ci)"
+ } finally {
+ Pop-Location
+ }
+ }
+}
+
+function Invoke-BrowserDeps {
+ if (-not (Test-Path $VENV_PYTHON)) {
+ Invoke-Venv
+ }
+ Write-Log "Ensuring Playwright + Chromium for JS crawl"
+ $script = @"
+from website_profiling.crawl.fetchers import ensure_browser_deps
+import json, sys
+status = ensure_browser_deps()
+print(json.dumps(status))
+sys.exit(0 if status.get('ok') else 1)
+"@
+ & $VENV_PYTHON -c $script
+ if ($PSVersionTable.PSVersion.Major -ge 7) {
+ $failed = ($LASTEXITCODE -ne 0)
+ } else {
+ $failed = (-not $?)
+ }
+ if ($failed) {
+ Write-Warn "Browser deps unavailable - JS/auto crawl disabled until Playwright + Chromium install successfully"
+ }
+}
+
+function Invoke-Setup {
+ New-Item -ItemType Directory -Force -Path $env:DATA_DIR | Out-Null
+ Invoke-Db
+ Invoke-Venv
+ Invoke-BrowserDeps
+ Invoke-Migrate
+ Invoke-WebDeps
+ Write-Log "Setup complete."
+ Write-Log "Start the UI: .\local-run.ps1 start"
+ Write-Log "Open http://localhost:3000/home (use localhost, not 127.0.0.1 for pipeline APIs)"
+}
+
+function Invoke-Start {
+ New-Item -ItemType Directory -Force -Path $env:DATA_DIR | Out-Null
+ Invoke-Db
+ if (-not (Test-Path $VENV_ALEMBIC)) {
+ Invoke-Venv
+ }
+ Invoke-BrowserDeps
+ Write-Log "Ensuring migrations are up to date"
+ & $VENV_ALEMBIC upgrade head
+ Assert-LastExitCode "Database migration failed (alembic upgrade head)"
+ Invoke-WebDeps
+ Write-Log "Starting Next.js dev server (Ctrl+C to stop)"
+ Write-Log "DATABASE_URL=$($env:DATABASE_URL)"
+ Write-Log "DATA_DIR=$($env:DATA_DIR)"
+ Write-Log "PYTHON=$($env:PYTHON)"
+ Push-Location $WEB
+ try {
+ & npm run dev
+ } finally {
+ Pop-Location
+ }
+}
+
+function Invoke-Stop {
+ Test-DockerRunning
+ if (Test-ContainerRunning $PG_CONTAINER) {
+ Write-Log "Stopping $PG_CONTAINER"
+ & docker stop $PG_CONTAINER *> $null
+ Assert-LastExitCode "Failed to stop container $PG_CONTAINER"
+ } else {
+ Write-Warn "Container $PG_CONTAINER is not running"
+ }
+}
+
+function Show-Help {
+ Write-Host @"
+Local dev runner - Postgres in Docker, app on your machine
+
+ .\local-run.ps1 Same as: start
+ .\local-run.ps1 start DB + migrations + npm run dev
+ .\local-run.ps1 setup One-time setup (no dev server)
+ .\local-run.ps1 db Start Postgres only
+ .\local-run.ps1 migrate Run alembic upgrade head
+ .\local-run.ps1 stop Stop Postgres container
+
+Environment overrides (optional):
+ DATABASE_URL (default: postgres://postgres:dev@127.0.0.1:5432/website_profiling)
+ DATA_DIR (default: /data)
+ PYTHON (default: /.venv/Scripts/python.exe)
+ WP_PG_CONTAINER, WP_PG_PORT, WP_PG_PASSWORD, WP_PG_DB
+
+After start, open: http://localhost:3000/home
+Run audits via sidebar "Run audit" (bottom-right FAB).
+
+Run CI-style tests: .\local-test.ps1 or ./local-test (bash/Git Bash/WSL).
+"@
+}
+
+$cmd = if ($args.Count -gt 0) { $args[0] } else { "start" }
+
+switch ($cmd) {
+ "start" { Invoke-Start }
+ "setup" { Invoke-Setup }
+ "db" { Invoke-Db }
+ "migrate" { Invoke-Migrate }
+ "stop" { Invoke-Stop }
+ "help" { Show-Help }
+ "-h" { Show-Help }
+ "--help" { Show-Help }
+ default { Write-Die "Unknown command: $cmd (try: .\local-run.ps1 help)" }
+}
diff --git a/scripts/local-run.sh b/scripts/local-run.sh
index b32b7410..ca13e8bd 100755
--- a/scripts/local-run.sh
+++ b/scripts/local-run.sh
@@ -84,6 +84,7 @@ cmd_venv() {
log "Installing Python dependencies"
"$VENV/bin/pip" install -q -r "$ROOT/requirements.txt"
"$VENV/bin/pip" install -q -r "$ROOT/requirements-browser.txt"
+ "$VENV/bin/pip" install -q -r "$ROOT/requirements-llm.txt"
}
cmd_migrate() {
diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1
new file mode 100644
index 00000000..a04218d9
--- /dev/null
+++ b/scripts/local-test.ps1
@@ -0,0 +1,379 @@
+# Local test runner — mirrors .github/workflows/ci.yml on Windows.
+# Usage: .\scripts\local-test.ps1 [command] [-NoCov]
+# (default) all — Postgres + migrations + Python + web checks
+# python — DB + pytest + CLI smoke only
+# reporting — reporting module 100% coverage gate (CI step)
+# tools — tools module coverage gate
+# web — typecheck, lint, vitest (no Postgres)
+# quick — pytest -NoCov + web (DB must already be running)
+# help — show commands
+# Requires: PowerShell 5.1+ (PowerShell 7+ recommended for reliable exit codes)
+
+$ErrorActionPreference = "Stop"
+
+$ROOT = Split-Path -Parent $PSScriptRoot
+Set-Location $ROOT
+
+$PG_CONTAINER = if ($env:WP_PG_CONTAINER) { $env:WP_PG_CONTAINER } else { "wp-pg" }
+$PG_IMAGE = if ($env:WP_PG_IMAGE) { $env:WP_PG_IMAGE } else { "postgres:16-alpine" }
+$PG_PORT = if ($env:WP_PG_PORT) { $env:WP_PG_PORT } else { "5432" }
+$PG_USER = if ($env:WP_PG_USER) { $env:WP_PG_USER } else { "postgres" }
+$PG_PASSWORD = if ($env:WP_PG_PASSWORD) { $env:WP_PG_PASSWORD } else { "dev" }
+$PG_DB = if ($env:WP_PG_DB) { $env:WP_PG_DB } else { "website_profiling" }
+
+if (-not $env:DATABASE_URL) {
+ $env:DATABASE_URL = "postgres://${PG_USER}:${PG_PASSWORD}@127.0.0.1:${PG_PORT}/${PG_DB}"
+}
+if (-not $env:DATA_DIR) {
+ $env:DATA_DIR = Join-Path $ROOT "data"
+}
+
+$VENV = Join-Path $ROOT ".venv"
+$VENV_PYTHON = Join-Path $VENV "Scripts\python.exe"
+$VENV_PIP = Join-Path $VENV "Scripts\pip.exe"
+$VENV_PYTEST = Join-Path $VENV "Scripts\pytest.exe"
+$VENV_ALEMBIC = Join-Path $VENV "Scripts\alembic.exe"
+$WEB = Join-Path $ROOT "web"
+
+$env:WEBSITE_PROFILING_ROOT = $ROOT
+if ($env:PYTHONPATH) {
+ $env:PYTHONPATH = "$($env:PYTHONPATH);$(Join-Path $ROOT 'src')"
+} else {
+ $env:PYTHONPATH = Join-Path $ROOT "src"
+}
+
+$PytestNoCov = $false
+
+function Write-Log([string]$Message) {
+ Write-Host "-> $Message" -ForegroundColor Cyan
+}
+
+function Write-Ok([string]$Message) {
+ Write-Host "OK $Message" -ForegroundColor Green
+}
+
+function Write-Warn([string]$Message) {
+ Write-Host "! $Message" -ForegroundColor Yellow
+}
+
+function Write-Die([string]$Message) {
+ Write-Host "X $Message" -ForegroundColor Red
+ exit 1
+}
+
+function Assert-LastExitCode([string]$Message) {
+ $failed = $false
+ if ($PSVersionTable.PSVersion.Major -ge 7) {
+ $failed = ($LASTEXITCODE -ne 0)
+ } else {
+ $failed = (-not $?)
+ }
+ if ($failed) {
+ Write-Die $Message
+ }
+}
+
+function Test-Command([string]$Name) {
+ if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) {
+ Write-Die "Missing required command: $Name"
+ }
+}
+
+function Get-PythonLauncher {
+ foreach ($cmd in @("python", "python3", "py")) {
+ if (Get-Command $cmd -ErrorAction SilentlyContinue) {
+ if ($cmd -eq "py") {
+ return ,@("py", "-3")
+ }
+ return ,@($cmd)
+ }
+ }
+ Write-Die "Missing required command: python (install Python 3.11+ and ensure it is on PATH)"
+}
+
+function Invoke-PythonLauncher {
+ param(
+ [Parameter(Mandatory = $true)]
+ [string[]]$Launcher,
+ [Parameter(ValueFromRemainingArguments = $true)]
+ [string[]]$PythonArgs
+ )
+
+ if ($Launcher.Count -gt 1) {
+ & $Launcher[0] $Launcher[1] @PythonArgs
+ } else {
+ & $Launcher[0] @PythonArgs
+ }
+ Assert-LastExitCode "Python command failed: $($Launcher -join ' ') $($PythonArgs -join ' ')"
+}
+
+function Get-DockerContainerNames {
+ param([switch]$All)
+
+ $dockerArgs = if ($All) { @("ps", "-a") } else { @("ps") }
+ $output = & docker @dockerArgs --format "{{.Names}}" 2>$null
+ if (-not $output) {
+ return @()
+ }
+ return @($output | ForEach-Object { "$_".Trim() } | Where-Object { $_ })
+}
+
+function Test-DockerRunning {
+ Test-Command docker
+ $prevErrorAction = $ErrorActionPreference
+ $ErrorActionPreference = "Continue"
+ try {
+ cmd /c "docker info >nul 2>&1"
+ } finally {
+ $ErrorActionPreference = $prevErrorAction
+ }
+ Assert-LastExitCode "Docker is not running. Start Docker Desktop, then retry."
+}
+
+function Test-ContainerExists([string]$Name) {
+ return (Get-DockerContainerNames -All) -contains $Name
+}
+
+function Test-ContainerRunning([string]$Name) {
+ return (Get-DockerContainerNames) -contains $Name
+}
+
+function Wait-ForPostgres {
+ for ($i = 1; $i -le 30; $i++) {
+ & docker exec $PG_CONTAINER pg_isready -U $PG_USER -d $PG_DB *> $null
+ if ($PSVersionTable.PSVersion.Major -ge 7) {
+ if ($LASTEXITCODE -eq 0) { return }
+ } elseif ($?) {
+ return
+ }
+ Start-Sleep -Seconds 1
+ }
+ Write-Die "Postgres did not become ready in time (container: $PG_CONTAINER)"
+}
+
+function Invoke-Db {
+ Test-DockerRunning
+ if (Test-ContainerExists $PG_CONTAINER) {
+ if (Test-ContainerRunning $PG_CONTAINER) {
+ Write-Log "Postgres already running ($PG_CONTAINER)"
+ } else {
+ Write-Log "Starting existing container $PG_CONTAINER"
+ & docker start $PG_CONTAINER *> $null
+ Assert-LastExitCode "Failed to start container $PG_CONTAINER"
+ }
+ } else {
+ Write-Log "Creating Postgres container $PG_CONTAINER on port $PG_PORT"
+ & docker run -d --name $PG_CONTAINER `
+ -e "POSTGRES_PASSWORD=$PG_PASSWORD" `
+ -e "POSTGRES_DB=$PG_DB" `
+ -p "${PG_PORT}:5432" `
+ $PG_IMAGE *> $null
+ Assert-LastExitCode "Failed to create Postgres container $PG_CONTAINER"
+ }
+ Wait-ForPostgres
+ Write-Log "DATABASE_URL=$($env:DATABASE_URL)"
+}
+
+function Invoke-Venv {
+ $pyLauncher = Get-PythonLauncher
+ if (-not (Test-Path $VENV_PYTHON)) {
+ Write-Log "Creating Python venv at .venv"
+ Invoke-PythonLauncher -Launcher $pyLauncher -PythonArgs @("-m", "venv", $VENV)
+ }
+ if (-not (Test-Path $VENV_PYTEST)) {
+ Write-Log "Installing Python dependencies"
+ & $VENV_PIP install -q -r (Join-Path $ROOT "requirements.txt")
+ Assert-LastExitCode "Failed to install requirements.txt"
+ & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-browser.txt")
+ Assert-LastExitCode "Failed to install requirements-browser.txt"
+ }
+}
+
+function Invoke-Migrate {
+ Invoke-Db
+ if (-not (Test-Path $VENV_ALEMBIC)) {
+ Invoke-Venv
+ }
+ Write-Log "Applying database migrations (alembic upgrade head)"
+ & $VENV_ALEMBIC upgrade head
+ Assert-LastExitCode "Database migration failed (alembic upgrade head)"
+}
+
+function Invoke-WebDeps {
+ Test-Command npm
+ $nodeModules = Join-Path $WEB "node_modules"
+ if (-not (Test-Path $nodeModules)) {
+ Write-Log "Installing web dependencies (npm ci)"
+ Push-Location $WEB
+ try {
+ & npm ci
+ Assert-LastExitCode "Failed to install web dependencies (npm ci)"
+ } finally {
+ Pop-Location
+ }
+ }
+}
+
+function Invoke-PytestCore {
+ if ($PytestNoCov) {
+ Write-Log "Pytest (tests/ -q -m not browser --no-cov)"
+ & $VENV_PYTEST tests/ -q -m "not browser" --no-cov
+ } else {
+ Write-Log "Pytest (tests/ -q -m not browser, 100% coverage gate)"
+ & $VENV_PYTEST tests/ -q -m "not browser"
+ }
+ Assert-LastExitCode "Core pytest failed"
+}
+
+function Invoke-PytestReporting {
+ Write-Log "Pytest (reporting coverage gate, 100%)"
+ & $VENV_PYTEST `
+ tests/test_categories_roadmap.py `
+ tests/test_report_categories_golden.py `
+ tests/test_categories_coverage.py `
+ tests/test_indexation_coverage.py `
+ tests/test_crawl_segments.py `
+ tests/test_terminology.py `
+ tests/test_compare_payload.py `
+ --cov=website_profiling.reporting `
+ --cov-config=.coveragerc.reporting `
+ --cov-report=term-missing `
+ --cov-fail-under=100 `
+ -q `
+ -o addopts=
+ Assert-LastExitCode "Reporting coverage gate failed"
+}
+
+function Invoke-PytestTools {
+ Write-Log "Pytest (tools coverage gate, 95%)"
+ & $VENV_PYTEST `
+ tests/test_alert_checker.py `
+ tests/test_schedule_runner.py `
+ tests/test_export_audit.py `
+ tests/test_export_audit_coverage.py `
+ tests/test_audit_tools.py `
+ tests/test_audit_tools_expanded.py `
+ tests/test_audit_tools_coverage.py `
+ tests/test_mcp_registry.py `
+ tests/test_mcp_resources.py `
+ --cov=website_profiling.tools `
+ --cov-config=.coveragerc.tools `
+ --cov-report=term-missing `
+ --cov-fail-under=95 `
+ -q `
+ -o addopts=
+ Assert-LastExitCode "Tools coverage gate failed"
+}
+
+function Invoke-PythonChecks {
+ Invoke-Db
+ Invoke-Venv
+ Invoke-Migrate
+ Invoke-PytestCore
+ Invoke-PytestReporting
+ Invoke-PytestTools
+ Write-Log "CLI smoke (python -m src --help)"
+ & $VENV_PYTHON -m src --help *> $null
+ Assert-LastExitCode "CLI smoke failed"
+ Write-Ok "Python checks passed"
+}
+
+function Invoke-WebChecks {
+ Invoke-WebDeps
+ Write-Log "Web typecheck"
+ Push-Location $WEB
+ try {
+ & npm run typecheck
+ Assert-LastExitCode "Web typecheck failed"
+ Write-Log "Web lint"
+ & npm run lint
+ Assert-LastExitCode "Web lint failed"
+ Write-Log "Web tests (vitest)"
+ & npm test
+ Assert-LastExitCode "Web tests failed"
+ } finally {
+ Pop-Location
+ }
+ Write-Ok "Web checks passed"
+}
+
+function Invoke-Quick {
+ if (-not $env:DATABASE_URL) {
+ Write-Die "DATABASE_URL is not set. Export it or run .\scripts\local-test.ps1 all"
+ }
+ Invoke-Venv
+ Invoke-WebDeps
+ Write-Warn "quick: assuming Postgres is up and migrated (.\local-run.ps1 db; .\local-run.ps1 migrate)"
+ $PytestNoCov = $true
+ Invoke-PytestCore
+ Write-Log "CLI smoke (python -m src --help)"
+ & $VENV_PYTHON -m src --help *> $null
+ Assert-LastExitCode "CLI smoke failed"
+ Invoke-WebChecks
+ Write-Ok "Quick test run passed"
+}
+
+function Show-Help {
+ Write-Host @"
+Local test runner — mirrors CI (python + web jobs)
+
+ .\scripts\local-test.ps1 Same as: all
+ .\scripts\local-test.ps1 all Postgres + migrations + full pytest + web
+ .\scripts\local-test.ps1 python DB + pytest (core + reporting + tools) + CLI
+ .\scripts\local-test.ps1 reporting Reporting module 100% coverage gate only
+ .\scripts\local-test.ps1 tools Tools module coverage gate only
+ .\scripts\local-test.ps1 web typecheck, lint, vitest (no Docker)
+ .\scripts\local-test.ps1 quick pytest -NoCov + web (DB must be ready)
+
+ .\scripts\local-test.ps1 all -NoCov skip pytest coverage gates (faster)
+
+Environment (same as .\local-run.ps1):
+ DATABASE_URL, DATA_DIR, WP_PG_CONTAINER, WP_PG_PORT, ...
+
+One-time dev setup: .\local-run.ps1 setup
+"@
+}
+
+$cmd = "all"
+$argList = @($args)
+if ($argList.Count -gt 0) {
+ $cmd = $argList[0]
+ $argList = if ($argList.Count -gt 1) { $argList[1..($argList.Count - 1)] } else { @() }
+}
+
+foreach ($arg in $argList) {
+ switch ($arg) {
+ "-NoCov" { $PytestNoCov = $true }
+ "-h" { Show-Help; exit 0 }
+ "--help" { Show-Help; exit 0 }
+ default { Write-Die "Unknown argument: $arg (try: .\scripts\local-test.ps1 help)" }
+ }
+}
+
+switch ($cmd) {
+ "all" {
+ Invoke-PythonChecks
+ Invoke-WebChecks
+ Write-Ok "All local tests passed (CI python + web jobs)"
+ }
+ "python" { Invoke-PythonChecks }
+ "reporting" {
+ Invoke-Venv
+ Invoke-PytestReporting
+ Write-Ok "Reporting coverage gate passed"
+ }
+ "tools" {
+ Invoke-Venv
+ Invoke-PytestTools
+ Write-Ok "Tools coverage gate passed"
+ }
+ "web" { Invoke-WebChecks }
+ "quick" {
+ $PytestNoCov = $true
+ Invoke-Quick
+ }
+ "help" { Show-Help }
+ "-h" { Show-Help }
+ "--help" { Show-Help }
+ default { Write-Die "Unknown command: $cmd (try: .\scripts\local-test.ps1 help)" }
+}
diff --git a/src/website_profiling/cli.py b/src/website_profiling/cli.py
index f492f16d..8b8ce8d3 100644
--- a/src/website_profiling/cli.py
+++ b/src/website_profiling/cli.py
@@ -4,6 +4,7 @@
from __future__ import annotations
from .commands import (
+ chat_cmd,
config_resolve,
enrich_cmd,
google_cmd,
@@ -40,6 +41,8 @@ def main() -> None:
page_live_cmd.run(cfg, cwd, args)
elif args.command == "page-coach":
page_coach_cmd.run(cfg, cwd, args)
+ elif args.command == "chat":
+ chat_cmd.run(cfg, args)
else:
pipeline_cmd.run(cfg, args)
diff --git a/src/website_profiling/commands/chat_cmd.py b/src/website_profiling/commands/chat_cmd.py
new file mode 100644
index 00000000..25c63b96
--- /dev/null
+++ b/src/website_profiling/commands/chat_cmd.py
@@ -0,0 +1,55 @@
+"""CLI: chat --stdin-json — agent turn for in-app chat (NDJSON events on stdout)."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from ..text_sanitize import sanitize_unicode_deep
+from ..tools.audit_tools import AuditToolContext
+from ..llm.agent import run_agent_turn
+
+
+def run(_cfg: dict, args: argparse.Namespace) -> None:
+ if not getattr(args, "stdin_json", False):
+ print("Error: chat requires --stdin-json", file=sys.stderr)
+ sys.exit(1)
+
+ try:
+ payload = json.load(sys.stdin)
+ except json.JSONDecodeError as e:
+ print(json.dumps({"type": "error", "message": f"Invalid stdin JSON: {e}"}))
+ sys.exit(1)
+
+ messages = payload.get("messages") or []
+ if not isinstance(messages, list):
+ messages = []
+
+ property_id = payload.get("property_id")
+ report_id = payload.get("report_id")
+ try:
+ pid = int(property_id) if property_id is not None else None
+ except (TypeError, ValueError):
+ pid = None
+ try:
+ rid = int(report_id) if report_id is not None else None
+ except (TypeError, ValueError):
+ rid = None
+
+ ctx = AuditToolContext(property_id=pid, report_id=rid)
+
+ def on_event(event: dict) -> None:
+ print(json.dumps(sanitize_unicode_deep(event), default=str), flush=True)
+
+ try:
+ result = run_agent_turn(messages, ctx, on_event=on_event)
+ except Exception as e:
+ msg = str(e).strip() or type(e).__name__
+ print(json.dumps({"type": "error", "message": msg}), flush=True)
+ sys.exit(1)
+
+ if not result.get("ok"):
+ err = result.get("error", "Agent failed")
+ print(json.dumps({"type": "error", "message": err}), flush=True)
+ sys.exit(1)
+ sys.exit(0)
diff --git a/src/website_profiling/commands/config_resolve.py b/src/website_profiling/commands/config_resolve.py
index 90aab5be..b6e0226e 100644
--- a/src/website_profiling/commands/config_resolve.py
+++ b/src/website_profiling/commands/config_resolve.py
@@ -232,6 +232,7 @@ def build_parser() -> argparse.ArgumentParser:
"gsc-links-import",
"page-live",
"page-coach",
+ "chat",
],
help="Run only this step (default: run all steps according to config)",
)
@@ -310,4 +311,10 @@ def build_parser() -> argparse.ArgumentParser:
dest="expand_only",
help="For 'keywords' command: only run Suggest expansion and print JSON to stdout.",
)
+ parser.add_argument(
+ "--stdin-json",
+ action="store_true",
+ dest="stdin_json",
+ help="For 'chat' command: read JSON payload from stdin and emit NDJSON events.",
+ )
return parser
diff --git a/src/website_profiling/db/_common.py b/src/website_profiling/db/_common.py
index 860f7c24..635267e3 100644
--- a/src/website_profiling/db/_common.py
+++ b/src/website_profiling/db/_common.py
@@ -10,6 +10,8 @@
from psycopg import Connection
from psycopg.types.json import Json
+from ..text_sanitize import strip_surrogates
+
def _now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
@@ -59,8 +61,10 @@ def _sanitize_for_json(obj: Any) -> Any:
"""Recursively replace NaN/Inf and numpy types so JSON is valid."""
if obj is None:
return None
- if isinstance(obj, (bool, str)):
+ if isinstance(obj, bool):
return obj
+ if isinstance(obj, str):
+ return strip_surrogates(obj)
if isinstance(obj, int):
return int(obj)
if isinstance(obj, float):
diff --git a/src/website_profiling/db/chat_store.py b/src/website_profiling/db/chat_store.py
new file mode 100644
index 00000000..206d4f24
--- /dev/null
+++ b/src/website_profiling/db/chat_store.py
@@ -0,0 +1,157 @@
+"""Chat session and message persistence for in-app AI chat."""
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+from psycopg import Connection
+from psycopg.types.json import Json
+
+from ._common import _row_field
+
+
+def create_session(
+ conn: Connection,
+ property_id: int,
+ title: str = "New chat",
+) -> int:
+ cur = conn.execute(
+ """INSERT INTO chat_sessions (property_id, title, created_at, updated_at)
+ VALUES (%s, %s, now(), now()) RETURNING id""",
+ (property_id, title.strip() or "New chat"),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ rid = _row_field(row, "id", index=0)
+ return int(rid) if rid is not None else 0
+
+
+def list_sessions(conn: Connection, property_id: int, limit: int = 30) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ """SELECT id, property_id, title, created_at, updated_at
+ FROM chat_sessions
+ WHERE property_id = %s
+ ORDER BY updated_at DESC
+ LIMIT %s""",
+ (property_id, max(1, min(limit, 100))),
+ )
+ out: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ created = _row_field(row, "created_at", index=3)
+ updated = _row_field(row, "updated_at", index=4)
+ out.append({
+ "id": int(_row_field(row, "id", index=0)),
+ "property_id": int(_row_field(row, "property_id", index=1)),
+ "title": _row_field(row, "title", index=2),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "updated_at": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""),
+ })
+ return out
+
+
+def get_session(conn: Connection, session_id: int) -> dict[str, Any] | None:
+ cur = conn.execute(
+ """SELECT id, property_id, title, created_at, updated_at
+ FROM chat_sessions WHERE id = %s""",
+ (session_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ created = _row_field(row, "created_at", index=3)
+ updated = _row_field(row, "updated_at", index=4)
+ return {
+ "id": int(_row_field(row, "id", index=0)),
+ "property_id": int(_row_field(row, "property_id", index=1)),
+ "title": _row_field(row, "title", index=2),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "updated_at": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""),
+ }
+
+
+def delete_session(conn: Connection, session_id: int) -> bool:
+ cur = conn.execute("DELETE FROM chat_sessions WHERE id = %s RETURNING id", (session_id,))
+ row = cur.fetchone()
+ conn.commit()
+ return row is not None
+
+
+def get_messages(conn: Connection, session_id: int, limit: int = 200) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ """SELECT id, role, content, tool_name, tool_args, tool_result, created_at
+ FROM chat_messages
+ WHERE session_id = %s
+ ORDER BY created_at ASC
+ LIMIT %s""",
+ (session_id, max(1, min(limit, 500))),
+ )
+ out: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ created = _row_field(row, "created_at", index=6)
+ tool_args = _row_field(row, "tool_args", index=4)
+ tool_result = _row_field(row, "tool_result", index=5)
+ if isinstance(tool_args, str):
+ try:
+ tool_args = json.loads(tool_args)
+ except json.JSONDecodeError:
+ pass
+ if isinstance(tool_result, str):
+ try:
+ tool_result = json.loads(tool_result)
+ except json.JSONDecodeError:
+ pass
+ out.append({
+ "id": int(_row_field(row, "id", index=0)),
+ "role": _row_field(row, "role", index=1),
+ "content": _row_field(row, "content", index=2) or "",
+ "tool_name": _row_field(row, "tool_name", index=3),
+ "tool_args": tool_args,
+ "tool_result": tool_result,
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ })
+ return out
+
+
+def append_message(
+ conn: Connection,
+ session_id: int,
+ role: str,
+ content: str = "",
+ *,
+ tool_name: Optional[str] = None,
+ tool_args: Optional[dict[str, Any]] = None,
+ tool_result: Optional[dict[str, Any]] = None,
+) -> int:
+ cur = conn.execute(
+ """INSERT INTO chat_messages
+ (session_id, role, content, tool_name, tool_args, tool_result, created_at)
+ VALUES (%s, %s, %s, %s, %s, %s, now()) RETURNING id""",
+ (
+ session_id,
+ role,
+ content,
+ tool_name,
+ Json(tool_args) if tool_args is not None else None,
+ Json(tool_result) if tool_result is not None else None,
+ ),
+ )
+ row = cur.fetchone()
+ touch_session(conn, session_id)
+ conn.commit()
+ rid = _row_field(row, "id", index=0)
+ return int(rid) if rid is not None else 0
+
+
+def update_session_title(conn: Connection, session_id: int, title: str) -> None:
+ conn.execute(
+ "UPDATE chat_sessions SET title = %s, updated_at = now() WHERE id = %s",
+ (title.strip() or "New chat", session_id),
+ )
+ conn.commit()
+
+
+def touch_session(conn: Connection, session_id: int) -> None:
+ conn.execute(
+ "UPDATE chat_sessions SET updated_at = now() WHERE id = %s",
+ (session_id,),
+ )
diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py
index f10dc525..443b7bf2 100644
--- a/src/website_profiling/db/storage.py
+++ b/src/website_profiling/db/storage.py
@@ -9,6 +9,16 @@
from __future__ import annotations
from ._common import _parse_json_field, _parse_row_json, _row_field, _sanitize_for_json
+from .chat_store import (
+ append_message,
+ create_session,
+ delete_session,
+ get_messages,
+ get_session,
+ list_sessions,
+ touch_session,
+ update_session_title,
+)
from .config_store import read_llm_config, read_pipeline_config, write_llm_config, write_pipeline_config
from .crawl_store import (
create_crawl_run,
@@ -44,16 +54,22 @@
"_parse_row_json",
"_row_field",
"_sanitize_for_json",
+ "append_message",
"backup_db_if_exists",
"close_db_pool",
"create_crawl_run",
+ "create_session",
+ "delete_session",
"db_session",
"ensure_crawl_tables_cleared",
"get_crawl_run_info",
"get_data_dir",
+ "get_messages",
+ "get_session",
"get_database_url",
"get_latest_crawl_run_id",
"init_schema",
+ "list_sessions",
"read_crawl",
"read_edges",
"read_historical_data",
@@ -70,6 +86,8 @@
"read_pipeline_config",
"read_report_payload",
"restore_historical_data",
+ "touch_session",
+ "update_session_title",
"write_crawl",
"write_crawl_batch",
"write_edges",
diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py
new file mode 100644
index 00000000..236dcc18
--- /dev/null
+++ b/src/website_profiling/llm/agent.py
@@ -0,0 +1,239 @@
+"""Agent loop for in-app chat and MCP — tool calling with streaming events."""
+from __future__ import annotations
+
+import json
+from typing import Any, Callable
+
+from ..llm_config import llm_is_enabled, load_llm_config_from_db
+from ..text_sanitize import sanitize_unicode_deep, strip_surrogates
+from ..tools.audit_tools import AuditToolContext
+from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, openai_tools_schema
+from .base import ChatResult, ToolCall, get_llm_client
+
+MAX_TOOL_ROUNDS = 10
+
+SYSTEM_PROMPT = """You are Site Audit AI, a technical SEO assistant for a self-hosted site audit platform.
+You help users understand crawl results, audit issues, Lighthouse scores, keywords, and Search Console data.
+
+Tool domains (prefer specific tools over generic list_issues):
+- Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history
+- Issues: list_issues, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, list_issue_workflow
+- On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues
+- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, get_status_code_breakdown, get_depth_distribution
+- Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_tech_stack_summary
+- Indexation: get_indexation_coverage, list_indexation_gaps, get_indexation_url_join
+- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_by_position, get_keyword_serp_overlay
+- Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_google_integration_status, get_gsc_page_query_slice, get_ga4_page_metrics
+- Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay
+- Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary
+- Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages
+- Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis
+- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_url_set_diff, get_health_history, get_category_health_history
+
+Visualization playbook (chat UI renders charts and tables from tool JSON automatically):
+- Category scores / health: get_category_scores, list_audit_categories, or get_report_summary
+- Issue breakdown: get_report_summary, get_issue_priority_breakdown (priority chart), and list_issues or get_critical_issues for the table
+- Top critical issues (required trio): get_report_summary, get_issue_priority_breakdown, get_critical_issues — then only write recommendations, never enumerate issues in prose
+- Audit overview / site health recap: get_report_summary (health, crawl, categories, issue counts). Keep prose to interpretation and next steps only — never repeat health score, URL counts, success rate, category scores, or priority counts in markdown; the UI renders those as cards and charts.
+- Distributions: get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_status_code_breakdown, get_depth_distribution
+- Trends over time: get_health_history, get_category_health_history
+- Compare drift: compare_category_deltas, compare_issue_deltas
+- Lighthouse: get_lighthouse_summary
+- Google/GSC: get_google_summary, get_gsc_top_queries
+
+Rules:
+- Use the provided tools to query real audit data. Do not invent URLs, scores, or metrics.
+- When citing issues, include the URL when available.
+- The chat UI automatically renders charts, gauges, and tables from tool results. Never tell the user you cannot show graphs or charts, and never send them to other app pages for data you can fetch with tools.
+- For visual or chart requests, always call the appropriate tools first, then give a short interpretation (2–4 sentences) with recommendations.
+- When tools return issue lists, scores, or breakdowns, keep the narrative short. Do not re-list every issue or duplicate data in markdown tables—the UI renders structured blocks from tool data.
+- Use markdown headings and bullets for structure. Do not emit fake chart JSON or custom visualization blocks.
+- You are read-only: you cannot run crawls or change settings.
+- If data is missing, say what integration or crawl step is needed.
+"""
+
+REACT_PROMPT_SUFFIX = """
+Respond with valid JSON only, one of:
+{"action":"tool","name":"","args":{...}}
+{"action":"answer","text":""}
+"""
+
+
+def _emit(on_event: Callable[[dict], None] | None, event: dict[str, Any]) -> None:
+ if on_event:
+ on_event(sanitize_unicode_deep(event))
+
+
+def _supports_native_tools(client: Any) -> bool:
+ return callable(getattr(client, "chat_with_tools", None))
+
+
+def _uses_ollama_tool_format(client: Any) -> bool:
+ return client.__class__.__name__ == "OllamaClient"
+
+
+def _react_step(
+ client: Any,
+ messages: list[dict[str, Any]],
+ tools_desc: str,
+ on_token: Callable[[str], None] | None,
+) -> ChatResult:
+ """JSON ReAct fallback for providers without native tool calling."""
+ convo = "\n".join(
+ f"{m.get('role')}: {m.get('content')}"
+ for m in messages
+ if m.get("role") in ("user", "assistant", "system")
+ )
+ user = f"Available tools:\n{tools_desc}\n\nConversation:\n{convo}\n\nNext action JSON:"
+ data = client.complete_json(SYSTEM_PROMPT + REACT_PROMPT_SUFFIX, user)
+ action = str(data.get("action") or "").lower()
+ if action == "tool":
+ name = str(data.get("name") or "")
+ args = data.get("args") if isinstance(data.get("args"), dict) else {}
+ return ChatResult(
+ tool_calls=[ToolCall(id="react-0", name=name, arguments=args)],
+ )
+ text = str(data.get("text") or data.get("answer") or data.get("content") or "")
+ if on_token and text:
+ on_token(text)
+ return ChatResult(content=text)
+
+
+def _tools_description(*, compact: bool = False) -> str:
+ lines = []
+ for t in TOOL_DEFINITIONS:
+ if compact:
+ lines.append(f"- {t['name']}")
+ else:
+ lines.append(f"- {t['name']}: {t.get('description', '')}")
+ return "\n".join(lines)
+
+
+def _build_openai_messages(history: list[dict[str, str]]) -> list[dict[str, Any]]:
+ out: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}]
+ for msg in history:
+ role = msg.get("role")
+ content = strip_surrogates(str(msg.get("content") or ""))
+ if role in ("user", "assistant"):
+ out.append({"role": role, "content": content})
+ return out
+
+
+def run_agent_turn(
+ messages: list[dict[str, str]],
+ context: AuditToolContext,
+ *,
+ on_event: Callable[[dict], None] | None = None,
+) -> dict[str, Any]:
+ """
+ Run the agent loop. Emits NDJSON-style events via on_event.
+ Returns final result dict with ok, message, tool_events.
+ """
+ cfg = load_llm_config_from_db()
+ if not llm_is_enabled(cfg):
+ err = "AI is disabled. Enable AI insights in the AI settings tab and configure a provider."
+ _emit(on_event, {"type": "error", "message": err})
+ return {"ok": False, "error": err}
+
+ try:
+ client = get_llm_client(cfg)
+ except ValueError as e:
+ msg = str(e)
+ _emit(on_event, {"type": "error", "message": msg})
+ return {"ok": False, "error": msg}
+
+ openai_messages = _build_openai_messages(messages)
+ tools = openai_tools_schema()
+ tool_events: list[dict[str, Any]] = []
+ final_message = ""
+
+ def on_token(text: str) -> None:
+ _emit(on_event, {"type": "token", "text": strip_surrogates(text)})
+
+ for _round in range(MAX_TOOL_ROUNDS):
+ _emit(on_event, {
+ "type": "status",
+ "phase": "model",
+ "detail": f"Thinking (step {_round + 1}/{MAX_TOOL_ROUNDS})…",
+ })
+ try:
+ llm_messages = sanitize_unicode_deep(openai_messages)
+ if _supports_native_tools(client):
+ result = client.chat_with_tools(llm_messages, tools, on_token=on_token)
+ else:
+ result = _react_step(client, llm_messages, _tools_description(compact=True), on_token)
+ except Exception as e:
+ msg = str(e).strip() or type(e).__name__
+ if "httpx" in msg.lower() or "requirements-llm" in msg.lower():
+ msg = (
+ "LLM dependencies are missing. Run: pip install -r requirements-llm.txt "
+ f"(or restart with ./local-run setup). Details: {msg}"
+ )
+ _emit(on_event, {"type": "error", "message": msg})
+ return {"ok": False, "error": msg, "tool_events": tool_events}
+
+ if result.tool_calls:
+ ollama_format = _uses_ollama_tool_format(client)
+ assistant_tool_calls = []
+ for i, tc in enumerate(result.tool_calls):
+ if ollama_format:
+ assistant_tool_calls.append({
+ "type": "function",
+ "function": {
+ "index": i,
+ "name": tc.name,
+ "arguments": sanitize_unicode_deep(tc.arguments),
+ },
+ })
+ else:
+ assistant_tool_calls.append({
+ "id": tc.id,
+ "type": "function",
+ "function": {"name": tc.name, "arguments": json.dumps(tc.arguments)},
+ })
+
+ if _supports_native_tools(client):
+ openai_messages.append({
+ "role": "assistant",
+ "content": strip_surrogates(result.content or ""),
+ "tool_calls": assistant_tool_calls,
+ })
+ else:
+ openai_messages.append({
+ "role": "assistant",
+ "content": f"Calling tool {result.tool_calls[0].name}",
+ })
+
+ for tc in result.tool_calls:
+ _emit(on_event, {"type": "tool_start", "name": tc.name, "args": tc.arguments})
+ tool_result = sanitize_unicode_deep(
+ dispatch_tool(tc.name, tc.arguments, context=context),
+ )
+ _emit(on_event, {"type": "tool_end", "name": tc.name, "result": tool_result})
+ tool_events.append({"name": tc.name, "args": tc.arguments, "result": tool_result})
+
+ tool_content = json.dumps(tool_result, default=str)
+ if ollama_format:
+ openai_messages.append({
+ "role": "tool",
+ "tool_name": tc.name,
+ "content": tool_content,
+ })
+ else:
+ openai_messages.append({
+ "role": "tool",
+ "tool_call_id": tc.id,
+ "content": tool_content,
+ })
+ continue
+
+ final_message = strip_surrogates(result.content).strip()
+ if final_message:
+ _emit(on_event, {"type": "done", "message": final_message})
+ return {"ok": True, "message": final_message, "tool_events": tool_events}
+
+ break
+
+ err = "Agent stopped after maximum tool rounds without a final answer."
+ _emit(on_event, {"type": "error", "message": err})
+ return {"ok": False, "error": err, "tool_events": tool_events}
diff --git a/src/website_profiling/llm/base.py b/src/website_profiling/llm/base.py
index b27dd5da..d200af46 100644
--- a/src/website_profiling/llm/base.py
+++ b/src/website_profiling/llm/base.py
@@ -3,12 +3,38 @@
import json
import re
-from typing import Any, Protocol
+from dataclasses import dataclass, field
+from typing import Any, Callable, Protocol
+
+
+@dataclass
+class ToolCall:
+ id: str
+ name: str
+ arguments: dict[str, Any]
+
+
+@dataclass
+class ChatResult:
+ content: str = ""
+ tool_calls: list[ToolCall] = field(default_factory=list)
+ finish_reason: str = "stop"
+
+
+TokenCallback = Callable[[str], None]
class LLMClient(Protocol):
def complete_json(self, system: str, user: str) -> dict[str, Any]: ...
+ def chat_with_tools(
+ self,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ *,
+ on_token: TokenCallback | None = None,
+ ) -> ChatResult: ...
+
def parse_json_response(text: str) -> dict[str, Any]:
text = (text or "").strip()
diff --git a/src/website_profiling/llm/providers/anthropic.py b/src/website_profiling/llm/providers/anthropic.py
index ce7e8467..e41b8b24 100644
--- a/src/website_profiling/llm/providers/anthropic.py
+++ b/src/website_profiling/llm/providers/anthropic.py
@@ -1,9 +1,10 @@
"""Anthropic Messages API."""
from __future__ import annotations
+import json
from typing import Any
-from ..base import parse_json_response
+from ..base import ChatResult, TokenCallback, ToolCall, parse_json_response
class AnthropicClient:
@@ -33,3 +34,105 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]:
if getattr(block, "type", None) == "text":
parts.append(block.text)
return parse_json_response("\n".join(parts))
+
+ def chat_with_tools(
+ self,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ *,
+ on_token: TokenCallback | None = None,
+ ) -> ChatResult:
+ if not self._api_key:
+ raise RuntimeError("Anthropic API key missing. Set it in the AI tab or ANTHROPIC_API_KEY.")
+ try:
+ import anthropic
+ except ImportError as e:
+ raise ImportError("pip install anthropic (or requirements-llm.txt)") from e
+
+ system_parts: list[str] = []
+ anthropic_messages: list[dict[str, Any]] = []
+ for msg in messages:
+ role = msg.get("role")
+ if role == "system":
+ system_parts.append(str(msg.get("content") or ""))
+ elif role == "tool":
+ anthropic_messages.append({
+ "role": "user",
+ "content": [{
+ "type": "tool_result",
+ "tool_use_id": str(msg.get("tool_call_id") or ""),
+ "content": str(msg.get("content") or ""),
+ }],
+ })
+ else:
+ anthropic_messages.append({
+ "role": role if role in ("user", "assistant") else "user",
+ "content": str(msg.get("content") or ""),
+ })
+
+ anthropic_tools = []
+ for tool in tools:
+ fn = tool.get("function") or tool
+ anthropic_tools.append({
+ "name": fn.get("name"),
+ "description": fn.get("description") or "",
+ "input_schema": fn.get("parameters") or {"type": "object", "properties": {}},
+ })
+
+ client = anthropic.Anthropic(api_key=self._api_key, timeout=self._timeout)
+ kwargs: dict[str, Any] = {
+ "model": self._model,
+ "max_tokens": 4096,
+ "system": "\n".join(system_parts),
+ "messages": anthropic_messages,
+ "tools": anthropic_tools,
+ }
+
+ if on_token:
+ content_parts: list[str] = []
+ tool_calls: list[ToolCall] = []
+ with client.messages.stream(**kwargs) as stream:
+ for event in stream:
+ if event.type == "content_block_delta" and hasattr(event.delta, "text"):
+ text = event.delta.text
+ content_parts.append(text)
+ on_token(text)
+ if event.type == "content_block_start" and getattr(event.content_block, "type", None) == "tool_use":
+ block = event.content_block
+ tool_calls.append(
+ ToolCall(id=block.id, name=block.name, arguments={}),
+ )
+ if event.type == "content_block_delta" and getattr(event.delta, "type", None) == "input_json_delta":
+ if tool_calls:
+ partial = getattr(event.delta, "partial_json", "") or ""
+ prev = tool_calls[-1].arguments.get("_partial", "")
+ tool_calls[-1].arguments["_partial"] = prev + partial
+ final = stream.get_final_message()
+ for tc in tool_calls:
+ partial = tc.arguments.pop("_partial", "")
+ if partial:
+ try:
+ tc.arguments = json.loads(partial)
+ except json.JSONDecodeError:
+ tc.arguments = {}
+ text_parts = []
+ for block in final.content:
+ if getattr(block, "type", None) == "text":
+ text_parts.append(block.text)
+ return ChatResult(content="".join(content_parts) or "".join(text_parts), tool_calls=tool_calls)
+
+ msg = client.messages.create(**kwargs)
+ content_parts: list[str] = []
+ tool_calls = []
+ for block in msg.content:
+ if getattr(block, "type", None) == "text":
+ content_parts.append(block.text)
+ if getattr(block, "type", None) == "tool_use":
+ tool_calls.append(
+ ToolCall(
+ id=block.id,
+ name=block.name,
+ arguments=dict(block.input) if isinstance(block.input, dict) else {},
+ ),
+ )
+ return ChatResult(content="".join(content_parts), tool_calls=tool_calls)
diff --git a/src/website_profiling/llm/providers/ollama.py b/src/website_profiling/llm/providers/ollama.py
index 47c77cef..ff1d2696 100644
--- a/src/website_profiling/llm/providers/ollama.py
+++ b/src/website_profiling/llm/providers/ollama.py
@@ -1,24 +1,92 @@
-"""Ollama local chat API."""
+"""Ollama local chat API with native tool calling when supported."""
from __future__ import annotations
import json
from typing import Any
-from ..base import parse_json_response
+from ..base import ChatResult, TokenCallback, ToolCall, parse_json_response
+
+
+def normalize_messages_for_ollama(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Convert OpenAI-style tool messages to Ollama's expected chat format."""
+ out: list[dict[str, Any]] = []
+ for msg in messages:
+ role = msg.get("role")
+ if role == "tool":
+ content = msg.get("content")
+ out.append({
+ "role": "tool",
+ "tool_name": msg.get("tool_name") or msg.get("name") or "tool",
+ "content": content if isinstance(content, str) else json.dumps(content, default=str),
+ })
+ continue
+
+ cleaned: dict[str, Any] = {"role": role}
+ content = msg.get("content")
+ if content is not None:
+ cleaned["content"] = content
+
+ tool_calls = msg.get("tool_calls")
+ if tool_calls:
+ ollama_calls = []
+ for i, tc in enumerate(tool_calls):
+ if not isinstance(tc, dict):
+ continue
+ fn = tc.get("function") or {}
+ raw_args = fn.get("arguments", {})
+ if isinstance(raw_args, str):
+ try:
+ args = json.loads(raw_args) if raw_args.strip() else {}
+ except json.JSONDecodeError:
+ args = {}
+ elif isinstance(raw_args, dict):
+ args = raw_args
+ else:
+ args = {}
+ ollama_calls.append({
+ "type": "function",
+ "function": {
+ "index": fn.get("index", i),
+ "name": fn.get("name") or tc.get("name") or "",
+ "arguments": args,
+ },
+ })
+ cleaned["tool_calls"] = ollama_calls
+ if "content" not in cleaned:
+ cleaned["content"] = ""
+
+ out.append(cleaned)
+ return out
class OllamaClient:
def __init__(self, cfg: dict[str, str]) -> None:
self._model = (cfg.get("llm_model") or "llama3.2").strip()
- self._timeout = float(cfg.get("llm_timeout_s") or 120)
+ configured = float(cfg.get("llm_timeout_s") or 120)
+ self._timeout = max(configured, 300)
self._base = (cfg.get("llm_base_url") or "http://127.0.0.1:11434").strip().rstrip("/")
- def complete_json(self, system: str, user: str) -> dict[str, Any]:
+ def _client(self):
try:
import httpx
except ImportError as e:
raise ImportError("pip install httpx (or requirements-llm.txt)") from e
+ return httpx.Client(timeout=self._timeout)
+ def _raise_for_status(self, response: Any) -> None:
+ try:
+ response.raise_for_status()
+ except Exception as e:
+ detail = ""
+ try:
+ detail = response.text.strip()
+ except Exception:
+ pass
+ if detail:
+ raise RuntimeError(f"Ollama API error ({response.status_code}): {detail}") from e
+ raise
+
+ def complete_json(self, system: str, user: str) -> dict[str, Any]:
payload = {
"model": self._model,
"stream": False,
@@ -29,9 +97,102 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]:
],
}
url = f"{self._base}/api/chat"
- with httpx.Client(timeout=self._timeout) as client:
+ with self._client() as client:
r = client.post(url, json=payload)
- r.raise_for_status()
+ self._raise_for_status(r)
data = r.json()
content = (data.get("message") or {}).get("content") or ""
return parse_json_response(content if isinstance(content, str) else json.dumps(content))
+
+ def chat_with_tools(
+ self,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ *,
+ on_token: TokenCallback | None = None,
+ ) -> ChatResult:
+ ollama_messages = normalize_messages_for_ollama(messages)
+ payload: dict[str, Any] = {
+ "model": self._model,
+ "messages": ollama_messages,
+ "tools": tools,
+ "stream": bool(on_token),
+ }
+ url = f"{self._base}/api/chat"
+
+ if on_token:
+ return self._stream_chat(url, payload, on_token)
+
+ with self._client() as client:
+ r = client.post(url, json={**payload, "stream": False})
+ self._raise_for_status(r)
+ data = r.json()
+ return self._parse_chat_response(data)
+
+ def _parse_tool_calls(self, raw_calls: list[Any]) -> list[ToolCall]:
+ tool_calls: list[ToolCall] = []
+ for tc in raw_calls:
+ if not isinstance(tc, dict):
+ continue
+ fn = tc.get("function") or {}
+ raw_args = fn.get("arguments") or tc.get("arguments") or "{}"
+ if isinstance(raw_args, dict):
+ args = raw_args
+ else:
+ try:
+ args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args)
+ except json.JSONDecodeError:
+ args = {}
+ tool_calls.append(
+ ToolCall(
+ id=str(tc.get("id") or f"ollama-{len(tool_calls)}"),
+ name=str(fn.get("name") or tc.get("name") or ""),
+ arguments=args if isinstance(args, dict) else {},
+ ),
+ )
+ return tool_calls
+
+ def _parse_chat_response(self, data: dict[str, Any]) -> ChatResult:
+ msg = data.get("message") or {}
+ content = str(msg.get("content") or "")
+ tool_calls = self._parse_tool_calls(msg.get("tool_calls") or [])
+ return ChatResult(
+ content=content,
+ tool_calls=tool_calls,
+ finish_reason="tool_calls" if tool_calls else "stop",
+ )
+
+ def _stream_chat(
+ self,
+ url: str,
+ payload: dict[str, Any],
+ on_token: TokenCallback,
+ ) -> ChatResult:
+ content_parts: list[str] = []
+ tool_calls: list[ToolCall] = []
+
+ with self._client() as client:
+ with client.stream("POST", url, json=payload) as resp:
+ self._raise_for_status(resp)
+ for line in resp.iter_lines():
+ if not line:
+ continue
+ try:
+ chunk = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ msg = chunk.get("message") or {}
+ if msg.get("content"):
+ text = str(msg["content"])
+ content_parts.append(text)
+ on_token(text)
+ if msg.get("tool_calls"):
+ tool_calls = self._parse_tool_calls(msg["tool_calls"])
+ if chunk.get("done"):
+ break
+
+ return ChatResult(
+ content="".join(content_parts),
+ tool_calls=tool_calls,
+ finish_reason="tool_calls" if tool_calls else "stop",
+ )
diff --git a/src/website_profiling/llm/providers/openai.py b/src/website_profiling/llm/providers/openai.py
index 273b7031..6619de2b 100644
--- a/src/website_profiling/llm/providers/openai.py
+++ b/src/website_profiling/llm/providers/openai.py
@@ -4,7 +4,7 @@
import json
from typing import Any
-from ..base import parse_json_response
+from ..base import ChatResult, TokenCallback, ToolCall, parse_json_response
class OpenAIClient:
@@ -40,3 +40,122 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]:
data = r.json()
content = data["choices"][0]["message"]["content"]
return parse_json_response(content if isinstance(content, str) else json.dumps(content))
+
+ def chat_with_tools(
+ self,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ *,
+ on_token: TokenCallback | None = None,
+ ) -> ChatResult:
+ if not self._api_key:
+ raise RuntimeError("OpenAI API key missing. Set it in the AI tab or OPENAI_API_KEY.")
+ try:
+ import httpx
+ except ImportError as e:
+ raise ImportError("pip install httpx (or requirements-llm.txt)") from e
+
+ payload: dict[str, Any] = {
+ "model": self._model,
+ "messages": messages,
+ "tools": tools,
+ "tool_choice": "auto",
+ "temperature": 0.2,
+ "stream": bool(on_token),
+ }
+ headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"}
+ url = f"{self._base}/chat/completions"
+
+ if on_token:
+ return self._stream_chat(url, headers, payload, on_token)
+
+ with httpx.Client(timeout=self._timeout) as client:
+ r = client.post(url, headers=headers, json={**payload, "stream": False})
+ r.raise_for_status()
+ data = r.json()
+ return self._parse_chat_response(data)
+
+ def _parse_chat_response(self, data: dict[str, Any]) -> ChatResult:
+ choice = (data.get("choices") or [{}])[0]
+ msg = choice.get("message") or {}
+ content = str(msg.get("content") or "")
+ tool_calls: list[ToolCall] = []
+ for tc in msg.get("tool_calls") or []:
+ fn = tc.get("function") or {}
+ raw_args = fn.get("arguments") or "{}"
+ try:
+ args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args)
+ except json.JSONDecodeError:
+ args = {}
+ tool_calls.append(
+ ToolCall(
+ id=str(tc.get("id") or ""),
+ name=str(fn.get("name") or ""),
+ arguments=args if isinstance(args, dict) else {},
+ ),
+ )
+ return ChatResult(
+ content=content,
+ tool_calls=tool_calls,
+ finish_reason=str(choice.get("finish_reason") or "stop"),
+ )
+
+ def _stream_chat(
+ self,
+ url: str,
+ headers: dict[str, str],
+ payload: dict[str, Any],
+ on_token: TokenCallback,
+ ) -> ChatResult:
+ import httpx
+
+ content_parts: list[str] = []
+ tool_calls_acc: dict[int, dict[str, Any]] = {}
+
+ with httpx.Client(timeout=self._timeout) as client:
+ with client.stream("POST", url, headers=headers, json=payload) as resp:
+ resp.raise_for_status()
+ for line in resp.iter_lines():
+ if not line or not line.startswith("data: "):
+ continue
+ chunk_raw = line[6:].strip()
+ if chunk_raw == "[DONE]":
+ break
+ try:
+ chunk = json.loads(chunk_raw)
+ except json.JSONDecodeError:
+ continue
+ delta = ((chunk.get("choices") or [{}])[0]).get("delta") or {}
+ if delta.get("content"):
+ text = str(delta["content"])
+ content_parts.append(text)
+ on_token(text)
+ for tc in delta.get("tool_calls") or []:
+ idx = int(tc.get("index") or 0)
+ acc = tool_calls_acc.setdefault(
+ idx,
+ {"id": "", "name": "", "arguments": ""},
+ )
+ if tc.get("id"):
+ acc["id"] = tc["id"]
+ fn = tc.get("function") or {}
+ if fn.get("name"):
+ acc["name"] = fn["name"]
+ if fn.get("arguments"):
+ acc["arguments"] += fn["arguments"]
+
+ tool_calls: list[ToolCall] = []
+ for acc in tool_calls_acc.values():
+ raw_args = acc.get("arguments") or "{}"
+ try:
+ args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args)
+ except json.JSONDecodeError:
+ args = {}
+ tool_calls.append(
+ ToolCall(
+ id=str(acc.get("id") or ""),
+ name=str(acc.get("name") or ""),
+ arguments=args if isinstance(args, dict) else {},
+ ),
+ )
+ return ChatResult(content="".join(content_parts), tool_calls=tool_calls, finish_reason="stop")
diff --git a/src/website_profiling/mcp/__init__.py b/src/website_profiling/mcp/__init__.py
new file mode 100644
index 00000000..e1ce3837
--- /dev/null
+++ b/src/website_profiling/mcp/__init__.py
@@ -0,0 +1 @@
+"""Model Context Protocol server for Site Audit."""
diff --git a/src/website_profiling/mcp/__main__.py b/src/website_profiling/mcp/__main__.py
new file mode 100644
index 00000000..f5f6e402
--- /dev/null
+++ b/src/website_profiling/mcp/__main__.py
@@ -0,0 +1,4 @@
+from .server import main
+
+if __name__ == "__main__":
+ main()
diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py
new file mode 100644
index 00000000..c185cccd
--- /dev/null
+++ b/src/website_profiling/mcp/server.py
@@ -0,0 +1,231 @@
+"""stdio MCP server exposing read-only Site Audit tools and resources."""
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from ..db.storage import db_session
+from ..tools.audit_tools import AuditToolContext
+from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, tool_handler_names
+
+_URI_PROPERTY = re.compile(r"^audit://property/(\d+)$")
+_URI_REPORT_LATEST = re.compile(r"^audit://property/(\d+)/report/latest$")
+_URI_REPORT_ID = re.compile(r"^audit://property/(\d+)/report/(\d+)$")
+
+
+def _default_property_id() -> int | None:
+ raw = os.environ.get("WP_PROPERTY_ID", "").strip()
+ if not raw:
+ return None
+ try:
+ pid = int(raw)
+ return pid if pid > 0 else None
+ except ValueError:
+ return None
+
+
+def _merge_context(args: dict[str, Any]) -> AuditToolContext:
+ pid = args.get("property_id")
+ rid = args.get("report_id")
+ default_pid = _default_property_id()
+ try:
+ property_id = int(pid) if pid is not None else default_pid
+ except (TypeError, ValueError):
+ property_id = default_pid
+ try:
+ report_id = int(rid) if rid is not None else None
+ except (TypeError, ValueError):
+ report_id = None
+ return AuditToolContext(property_id=property_id, report_id=report_id)
+
+
+def _payload_index(payload: dict[str, Any]) -> dict[str, Any]:
+ """Truncated payload index: keys and list lengths only."""
+ index: dict[str, Any] = {}
+ for key, val in payload.items():
+ if isinstance(val, list):
+ index[key] = {"type": "list", "count": len(val)}
+ elif isinstance(val, dict):
+ index[key] = {"type": "object", "keys": list(val.keys())[:30]}
+ else:
+ index[key] = {"type": type(val).__name__, "preview": str(val)[:120]}
+ return index
+
+
+def _read_glossary_excerpt() -> str:
+ root = Path(__file__).resolve().parents[3]
+ path = root / "docs" / "GLOSSARY.md"
+ if not path.is_file():
+ return "Glossary file not found."
+ text = path.read_text(encoding="utf-8")
+ return text[:12000]
+
+
+def _tools_catalog_json() -> str:
+ domains: dict[str, list[str]] = {
+ "portfolio": [],
+ "issues": [],
+ "crawl": [],
+ "schema": [],
+ "links": [],
+ "indexation": [],
+ "content": [],
+ "keywords": [],
+ "google": [],
+ "backlinks": [],
+ "performance": [],
+ "drift": [],
+ "security": [],
+ }
+ for tool in TOOL_DEFINITIONS:
+ name = tool["name"]
+ if name.startswith(("list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report")):
+ domains["portfolio"].append(name)
+ elif "issue" in name or "category" in name or "workflow" in name:
+ domains["issues"].append(name)
+ elif name in ("search_pages", "get_page_details", "get_internal_links", "list_redirects", "list_broken_links",
+ "get_status_code", "get_response_time", "get_depth", "get_crawl_segments", "get_browser"):
+ domains["crawl"].append(name)
+ elif "schema" in name or name == "get_seo_health":
+ domains["schema"].append(name)
+ elif "orphan" in name or "link" in name or "fingerprint" in name:
+ domains["links"].append(name)
+ elif "indexation" in name or "hreflang" in name or "language" in name:
+ domains["indexation"].append(name)
+ elif "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name:
+ domains["content"].append(name)
+ elif "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name:
+ domains["keywords"].append(name)
+ elif "google" in name or "gsc" in name or "ga4" in name:
+ domains["google"].append(name)
+ elif "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name:
+ domains["backlinks"].append(name)
+ elif "lighthouse" in name or "crux" in name or "slow" in name:
+ domains["performance"].append(name)
+ elif "health" in name or "compare" in name or "alert" in name or "tech_stack" in name:
+ domains["drift"].append(name)
+ elif "security" in name:
+ domains["security"].append(name)
+ else:
+ domains["portfolio"].append(name)
+ return json.dumps({"tool_count": len(TOOL_DEFINITIONS), "handlers": sorted(tool_handler_names()), "domains": domains}, indent=2)
+
+
+def _resolve_resource(uri: str) -> str:
+ if uri == "audit://properties":
+ result = dispatch_tool("list_properties", {})
+ return json.dumps(result, indent=2, default=str)
+
+ if uri == "audit://glossary":
+ return _read_glossary_excerpt()
+
+ if uri == "audit://tools":
+ return _tools_catalog_json()
+
+ m = _URI_PROPERTY.match(uri)
+ if m:
+ pid = int(m.group(1))
+ prop = dispatch_tool("get_property", {"property_id": pid})
+ summary = dispatch_tool("get_report_summary", {"property_id": pid})
+ return json.dumps({"property": prop, "latest_report": summary}, indent=2, default=str)
+
+ m = _URI_REPORT_LATEST.match(uri)
+ if m:
+ pid = int(m.group(1))
+ ctx = AuditToolContext(property_id=pid)
+ with db_session() as conn:
+ payload = ctx.load_payload(conn)
+ if not payload:
+ return json.dumps({"error": "no report found"})
+ return json.dumps(_payload_index(payload), indent=2, default=str)
+
+ m = _URI_REPORT_ID.match(uri)
+ if m:
+ pid = int(m.group(1))
+ rid = int(m.group(2))
+ ctx = AuditToolContext(property_id=pid, report_id=rid)
+ with db_session() as conn:
+ payload = ctx.load_payload(conn)
+ if not payload:
+ return json.dumps({"error": "no report found"})
+ return json.dumps(_payload_index(payload), indent=2, default=str)
+
+ return json.dumps({"error": f"unknown resource: {uri}"})
+
+
+def main() -> None:
+ try:
+ from mcp.server import Server
+ from mcp.server.stdio import stdio_server
+ from mcp.types import Resource, TextContent, Tool
+ except ImportError as e:
+ raise SystemExit(
+ "MCP SDK not installed. Run: pip install -r requirements-mcp.txt",
+ ) from e
+
+ server = Server("site-audit")
+ default_pid = _default_property_id()
+
+ @server.list_tools()
+ async def list_tools() -> list[Tool]:
+ out: list[Tool] = []
+ for spec in TOOL_DEFINITIONS:
+ out.append(
+ Tool(
+ name=spec["name"],
+ description=spec.get("description", ""),
+ inputSchema=spec.get("inputSchema", {"type": "object", "properties": {}}),
+ ),
+ )
+ return out
+
+ @server.call_tool()
+ async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]:
+ args = dict(arguments or {})
+ ctx = _merge_context(args)
+ result = dispatch_tool(name, args, context=ctx)
+ text = json.dumps(result, indent=2, default=str)
+ return [TextContent(type="text", text=text)]
+
+ @server.list_resources()
+ async def list_resources() -> list[Resource]:
+ resources = [
+ Resource(uri="audit://properties", name="Properties", description="All configured site properties", mimeType="application/json"),
+ Resource(uri="audit://glossary", name="Glossary", description="Site Audit field glossary excerpt", mimeType="text/markdown"),
+ Resource(uri="audit://tools", name="Tool catalog", description="MCP tool catalog grouped by domain", mimeType="application/json"),
+ ]
+ if default_pid:
+ resources.extend([
+ Resource(
+ uri=f"audit://property/{default_pid}",
+ name=f"Property {default_pid}",
+ description="Property details and latest report summary",
+ mimeType="application/json",
+ ),
+ Resource(
+ uri=f"audit://property/{default_pid}/report/latest",
+ name=f"Latest report index (property {default_pid})",
+ description="Payload key index for latest audit report",
+ mimeType="application/json",
+ ),
+ ])
+ return resources
+
+ @server.read_resource()
+ async def read_resource(uri: str) -> str:
+ return _resolve_resource(uri)
+
+ async def run() -> None:
+ async with stdio_server() as (read_stream, write_stream):
+ await server.run(read_stream, write_stream, server.create_initialization_options())
+
+ import asyncio
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/website_profiling/reporting/compare_payload.py b/src/website_profiling/reporting/compare_payload.py
new file mode 100644
index 00000000..3ef511a8
--- /dev/null
+++ b/src/website_profiling/reporting/compare_payload.py
@@ -0,0 +1,574 @@
+"""Report payload comparison — parity with web reportCompare.ts / reportCompareExtras.ts."""
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlparse
+
+_PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}
+_LH_DELTA_THRESHOLD = 5
+_ISSUE_DELTA_CAP = 100
+_LINK_METRIC_CAP = 200
+
+_SEO_HEALTH_FIELDS = [
+ ("missing_title", "Missing title", False),
+ ("title_ok", "Title OK", True),
+ ("missing_meta_desc", "Missing meta description", False),
+ ("meta_desc_ok", "Meta description OK", True),
+ ("h1_zero", "Pages with no H1", False),
+ ("h1_one", "Pages with one H1", True),
+ ("h1_multi", "Pages with multiple H1s", False),
+ ("thin_content", "Thin content (flagged)", False),
+]
+
+
+def norm_report_url(url: str) -> str:
+ raw = str(url or "").strip()
+ if not raw:
+ return ""
+ try:
+ p = urlparse(raw)
+ path = (p.path or "/").rstrip("/") or "/"
+ host = (p.hostname or "").lower()
+ if not host:
+ return raw.rstrip("/").lower()
+ return f"{host}{path}"
+ except Exception:
+ return raw.rstrip("/").lower()
+
+
+def _num(v: Any) -> float | None:
+ try:
+ n = float(v)
+ return n if n == n else None # NaN check
+ except (TypeError, ValueError):
+ return None
+
+
+def _score_from_categories(categories: list[Any]) -> int | None:
+ scores = [
+ float(c.get("score"))
+ for c in categories
+ if isinstance(c, dict) and isinstance(c.get("score"), (int, float))
+ ]
+ return round(sum(scores) / len(scores)) if scores else None
+
+
+def _issue_key(url: str, category: str, message: str) -> str:
+ return f"{norm_report_url(url)}|{category}|{message[:120]}"
+
+
+def _flatten_category_issues(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ out: dict[str, dict[str, Any]] = {}
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ category = str(cat.get("name") or cat.get("id") or "")
+ for iss in cat.get("issues") or []:
+ if not isinstance(iss, dict):
+ continue
+ url = str(iss.get("url") or "")
+ message = str(iss.get("message") or iss.get("recommendation") or "").strip()
+ if not url and not message:
+ continue
+ key = _issue_key(url, category, message)
+ out[key] = {
+ "kind": "new",
+ "url": url or "—",
+ "category": category,
+ "priority": str(iss.get("priority") or "Medium"),
+ "message": message or "—",
+ }
+ return out
+
+
+def build_issue_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ cur = _flatten_category_issues(current)
+ base = _flatten_category_issues(baseline)
+ out: list[dict[str, Any]] = []
+ for key, row in cur.items():
+ if key not in base:
+ out.append({**row, "kind": "new"})
+ for key, row in base.items():
+ if key not in cur:
+ out.append({**row, "kind": "resolved"})
+ out.sort(key=lambda x: (
+ _PRIORITY_ORDER.get(x.get("priority", "Low"), 9),
+ 0 if x.get("kind") == "new" else 1,
+ str(x.get("url") or ""),
+ ))
+ return out
+
+
+def build_priority_counts(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ def count_map(payload: dict[str, Any]) -> dict[str, int]:
+ counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ for iss in cat.get("issues") or []:
+ if not isinstance(iss, dict):
+ continue
+ p = str(iss.get("priority") or "Medium")
+ counts[p] = counts.get(p, 0) + 1
+ return counts
+
+ cur = count_map(current)
+ base = count_map(baseline)
+ return [
+ {
+ "priority": p,
+ "current": cur.get(p, 0),
+ "baseline": base.get(p, 0),
+ "delta": cur.get(p, 0) - base.get(p, 0),
+ }
+ for p in ("Critical", "High", "Medium", "Low")
+ ]
+
+
+def _lh_from_payload(payload: dict[str, Any]) -> dict[str, dict[str, float | None]]:
+ out: dict[str, dict[str, float | None]] = {}
+ by_url = payload.get("lighthouse_by_url")
+ if isinstance(by_url, dict):
+ for raw_url, summary in by_url.items():
+ if not isinstance(summary, dict):
+ continue
+ k = norm_report_url(str(raw_url))
+ if not k:
+ continue
+ metrics = summary.get("median_metrics") or summary
+ perf = _num(metrics.get("performance_score") or summary.get("performance"))
+ seo = _num(metrics.get("seo_score") or summary.get("seo"))
+ out[k] = {
+ "perf": round(perf) if perf is not None else None,
+ "seo": round(seo) if seo is not None else None,
+ }
+ for link in payload.get("links") or []:
+ if not isinstance(link, dict):
+ continue
+ k = norm_report_url(str(link.get("url") or ""))
+ if not k or k in out:
+ continue
+ lh = link.get("lighthouse") if isinstance(link.get("lighthouse"), dict) else {}
+ metrics = lh.get("median_metrics") or {}
+ perf = _num(metrics.get("performance_score"))
+ seo = _num(metrics.get("seo_score"))
+ out[k] = {
+ "perf": round(perf) if perf is not None else None,
+ "seo": round(seo) if seo is not None else None,
+ }
+ return out
+
+
+def build_lighthouse_url_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ cur = _lh_from_payload(current)
+ base = _lh_from_payload(baseline)
+ out: list[dict[str, Any]] = []
+ for k, c in cur.items():
+ b = base.get(k)
+ if not b:
+ continue
+ perf_delta = (c["perf"] - b["perf"]) if c["perf"] is not None and b["perf"] is not None else None
+ seo_delta = (c["seo"] - b["seo"]) if c["seo"] is not None and b["seo"] is not None else None
+ if (
+ (perf_delta is not None and abs(perf_delta) >= _LH_DELTA_THRESHOLD)
+ or (seo_delta is not None and abs(seo_delta) >= _LH_DELTA_THRESHOLD)
+ ):
+ out.append({
+ "url": k,
+ "performance_current": c["perf"],
+ "performance_baseline": b["perf"],
+ "performance_delta": perf_delta,
+ "seo_current": c["seo"],
+ "seo_baseline": b["seo"],
+ "seo_delta": seo_delta,
+ })
+ out.sort(key=lambda x: abs(x.get("performance_delta") or 0), reverse=True)
+ return out
+
+
+def build_link_metric_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ specs = [
+ ("inlinks", "inlinks", 1),
+ ("outlinks", "outlinks", 1),
+ ("word_count", "word_count", 25),
+ ("response_time_ms", "response_ms", 150),
+ ]
+ cur_map = {
+ norm_report_url(str(l.get("url") or "")): l
+ for l in (current.get("links") or [])
+ if isinstance(l, dict) and norm_report_url(str(l.get("url") or ""))
+ }
+ out: list[dict[str, Any]] = []
+ for bl in baseline.get("links") or []:
+ if not isinstance(bl, dict):
+ continue
+ k = norm_report_url(str(bl.get("url") or ""))
+ if not k:
+ continue
+ cl = cur_map.get(k)
+ if not cl:
+ continue
+ for key, metric, min_delta in specs:
+ c = _num(cl.get(key))
+ b = _num(bl.get(key))
+ if c is None or b is None:
+ continue
+ delta = round((c - b) * 10) / 10
+ if abs(delta) >= min_delta:
+ out.append({
+ "url": cl.get("url") or bl.get("url"),
+ "metric": metric,
+ "current": c,
+ "baseline": b,
+ "delta": delta,
+ })
+ out.sort(key=lambda x: abs(x.get("delta") or 0), reverse=True)
+ return out[:_LINK_METRIC_CAP]
+
+
+def _redirect_key(r: dict[str, Any]) -> str:
+ return norm_report_url(str(r.get("url") or r.get("from") or ""))
+
+
+def build_redirect_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ def to_map(lst: list[Any]) -> dict[str, dict[str, Any]]:
+ m: dict[str, dict[str, Any]] = {}
+ for r in lst:
+ if not isinstance(r, dict):
+ continue
+ k = _redirect_key(r)
+ if not k:
+ continue
+ m[k] = {
+ "kind": "new",
+ "url": str(r.get("url") or r.get("from") or k),
+ "status": str(r.get("status") or "—"),
+ "final_url": str(r.get("final_url") or r.get("to") or ""),
+ }
+ return m
+
+ cur = to_map(current.get("redirects") or [])
+ base = to_map(baseline.get("redirects") or [])
+ out: list[dict[str, Any]] = []
+ for k, row in cur.items():
+ if k not in base:
+ out.append({**row, "kind": "new"})
+ for k, row in base.items():
+ if k not in cur:
+ out.append({**row, "kind": "removed"})
+ out.sort(key=lambda x: str(x.get("url") or ""))
+ return out
+
+
+def _security_key(f: dict[str, Any]) -> str:
+ return f"{norm_report_url(str(f.get('url') or ''))}|{f.get('finding_type')}|{str(f.get('message') or '')[:80]}"
+
+
+def build_security_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ def to_map(lst: list[Any]) -> dict[str, dict[str, Any]]:
+ m: dict[str, dict[str, Any]] = {}
+ for f in lst:
+ if not isinstance(f, dict):
+ continue
+ k = _security_key(f)
+ m[k] = {
+ "kind": "new",
+ "url": str(f.get("url") or "—"),
+ "severity": str(f.get("severity") or "—"),
+ "finding_type": str(f.get("finding_type") or "—"),
+ "message": str(f.get("message") or "—"),
+ }
+ return m
+
+ cur = to_map(current.get("security_findings") or [])
+ base = to_map(baseline.get("security_findings") or [])
+ out: list[dict[str, Any]] = []
+ for key, row in cur.items():
+ if key not in base:
+ out.append({**row, "kind": "new"})
+ for key, row in base.items():
+ if key not in cur:
+ out.append({**row, "kind": "resolved"})
+ return out
+
+
+def build_duplicate_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ def to_map(lst: list[Any]) -> dict[str, dict[str, Any]]:
+ m: dict[str, dict[str, Any]] = {}
+ for c in lst:
+ if not isinstance(c, dict):
+ continue
+ k = str(c.get("id") or c.get("representative_url") or "").strip()
+ if not k:
+ continue
+ members = c.get("member_count")
+ if members is None:
+ members = len(c.get("member_urls") or [])
+ m[k] = {"rep": c.get("representative_url") or k, "members": int(members or 0)}
+ return m
+
+ cur = to_map(current.get("content_duplicates") or [])
+ base = to_map(baseline.get("content_duplicates") or [])
+ out: list[dict[str, Any]] = []
+ for cid, c in cur.items():
+ b = base.get(cid)
+ if not b:
+ out.append({
+ "kind": "new",
+ "cluster_id": cid,
+ "representative_url": c["rep"],
+ "current_members": c["members"],
+ "baseline_members": 0,
+ })
+ elif c["members"] != b["members"]:
+ out.append({
+ "kind": "changed",
+ "cluster_id": cid,
+ "representative_url": c["rep"],
+ "current_members": c["members"],
+ "baseline_members": b["members"],
+ })
+ for cid, b in base.items():
+ if cid not in cur:
+ out.append({
+ "kind": "removed",
+ "cluster_id": cid,
+ "representative_url": b["rep"],
+ "current_members": 0,
+ "baseline_members": b["members"],
+ })
+ return out
+
+
+def build_tech_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ def to_map(payload: dict[str, Any]) -> dict[str, int]:
+ m: dict[str, int] = {}
+ tech = payload.get("tech_stack_summary") or {}
+ entries = tech.get("technologies") if isinstance(tech, dict) else []
+ for t in entries or []:
+ if not isinstance(t, dict):
+ continue
+ name = str(t.get("name") or t.get("tech") or "").strip()
+ if name:
+ m[name] = int(t.get("count") or 0)
+ return m
+
+ cur = to_map(current)
+ base = to_map(baseline)
+ out: list[dict[str, Any]] = []
+ for name, count in cur.items():
+ if name not in base:
+ out.append({"kind": "added", "name": name, "current_count": count, "baseline_count": 0})
+ for name, count in base.items():
+ if name not in cur:
+ out.append({"kind": "removed", "name": name, "current_count": 0, "baseline_count": count})
+ out.sort(key=lambda x: str(x.get("name") or ""))
+ return out
+
+
+def _metric_row(
+ id_: str,
+ label: str,
+ current: float | None,
+ baseline: float | None,
+ higher_is_better: bool,
+ fmt: str = "count",
+) -> dict[str, Any]:
+ delta = round((current - baseline) * 10) / 10 if current is not None and baseline is not None else None
+ return {
+ "id": id_,
+ "label": label,
+ "current": current,
+ "baseline": baseline,
+ "delta": delta,
+ "higher_is_better": higher_is_better,
+ "format": fmt,
+ }
+
+
+def build_content_metrics(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ cw = (current.get("content_analytics") or {}).get("word_count_stats") or {}
+ bw = (baseline.get("content_analytics") or {}).get("word_count_stats") or {}
+ cur_thin = len((current.get("content_analytics") or {}).get("thin_pages") or [])
+ if not cur_thin:
+ cur_thin = int((current.get("seo_health") or {}).get("thin_content") or 0)
+ base_thin = len((baseline.get("content_analytics") or {}).get("thin_pages") or [])
+ if not base_thin:
+ base_thin = int((baseline.get("seo_health") or {}).get("thin_content") or 0)
+ cs = current.get("social_coverage") or {}
+ bs = baseline.get("social_coverage") or {}
+ rows = [
+ _metric_row("mean_words", "Mean words", _num(cw.get("mean")), _num(bw.get("mean")), True),
+ _metric_row("median_words", "Median words", _num(cw.get("median")), _num(bw.get("median")), True),
+ _metric_row("thin_pages", "Thin pages", float(cur_thin), float(base_thin), False),
+ _metric_row("dup_groups", "Duplicate groups", float(len(current.get("content_duplicates") or [])),
+ float(len(baseline.get("content_duplicates") or [])), False),
+ _metric_row("og_cov", "OG coverage %", _num(cs.get("og_coverage_pct")), _num(bs.get("og_coverage_pct")), True, "percent"),
+ _metric_row("tw_cov", "Twitter coverage %", _num(cs.get("twitter_coverage_pct")), _num(bs.get("twitter_coverage_pct")), True, "percent"),
+ _metric_row("resp_p50", "Response p50 ms", _num((current.get("response_time_stats") or {}).get("p50")),
+ _num((baseline.get("response_time_stats") or {}).get("p50")), False),
+ _metric_row("resp_p95", "Response p95 ms", _num((current.get("response_time_stats") or {}).get("p95")),
+ _num((baseline.get("response_time_stats") or {}).get("p95")), False),
+ _metric_row("crawl_time", "Crawl duration s", _num((current.get("summary") or {}).get("crawl_time_s")),
+ _num((baseline.get("summary") or {}).get("crawl_time_s")), False),
+ _metric_row("count_3xx", "Redirect pages", _num((current.get("summary") or {}).get("count_3xx")),
+ _num((baseline.get("summary") or {}).get("count_3xx")), False),
+ _metric_row("avg_outlinks", "Avg outlinks", _num((current.get("summary") or {}).get("avg_outlinks")),
+ _num((baseline.get("summary") or {}).get("avg_outlinks")), True),
+ ]
+ return [r for r in rows if r["current"] is not None or r["baseline"] is not None]
+
+
+def build_google_metrics(current: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]:
+ cg = ((current.get("google") or {}).get("gsc") or {}).get("summary")
+ bg = ((baseline.get("google") or {}).get("gsc") or {}).get("summary")
+ ca = ((current.get("google") or {}).get("ga4") or {}).get("summary")
+ ba = ((baseline.get("google") or {}).get("ga4") or {}).get("summary")
+ has_gsc = cg is not None or bg is not None
+ has_ga4 = ca is not None or ba is not None
+ if not has_gsc and not has_ga4:
+ return {"available": False, "metrics": []}
+ rows: list[dict[str, Any]] = []
+ if has_gsc:
+ rows.extend([
+ _metric_row("gsc_clicks", "GSC clicks", _num(cg.get("clicks") if isinstance(cg, dict) else None),
+ _num(bg.get("clicks") if isinstance(bg, dict) else None), True),
+ _metric_row("gsc_impr", "GSC impressions", _num(cg.get("impressions") if isinstance(cg, dict) else None),
+ _num(bg.get("impressions") if isinstance(bg, dict) else None), True),
+ _metric_row("gsc_ctr", "GSC CTR", _num(cg.get("ctr") if isinstance(cg, dict) else None),
+ _num(bg.get("ctr") if isinstance(bg, dict) else None), True, "percent"),
+ _metric_row("gsc_pos", "GSC position", _num(cg.get("position") if isinstance(cg, dict) else None),
+ _num(bg.get("position") if isinstance(bg, dict) else None), False),
+ ])
+ if has_ga4:
+ rows.extend([
+ _metric_row("ga4_sessions", "GA4 sessions", _num(ca.get("sessions") if isinstance(ca, dict) else None),
+ _num(ba.get("sessions") if isinstance(ba, dict) else None), True),
+ _metric_row("ga4_users", "GA4 users", _num(ca.get("activeUsers") if isinstance(ca, dict) else None),
+ _num(ba.get("activeUsers") if isinstance(ba, dict) else None), True),
+ _metric_row("ga4_views", "GA4 page views", _num(ca.get("screenPageViews") if isinstance(ca, dict) else None),
+ _num(ba.get("screenPageViews") if isinstance(ba, dict) else None), True),
+ _metric_row("ga4_engagement", "GA4 engagement", _num(ca.get("engagementRate") if isinstance(ca, dict) else None),
+ _num(ba.get("engagementRate") if isinstance(ba, dict) else None), True, "percent"),
+ ])
+ metrics = [r for r in rows if r["current"] is not None or r["baseline"] is not None]
+ return {"available": True, "metrics": metrics}
+
+
+def build_seo_health_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ cur = current.get("seo_health") or {}
+ base = baseline.get("seo_health") or {}
+ out: list[dict[str, Any]] = []
+ for key, label, higher in _SEO_HEALTH_FIELDS:
+ c = int(cur.get(key) or 0)
+ b = int(base.get(key) or 0)
+ if c == b:
+ continue
+ out.append({
+ "id": key,
+ "label": label,
+ "current": c,
+ "baseline": b,
+ "delta": c - b,
+ "higher_is_better": higher,
+ })
+ out.sort(key=lambda x: abs(x.get("delta") or 0), reverse=True)
+ return out
+
+
+def build_category_scores(current: dict[str, Any], baseline: dict[str, Any]) -> list[dict[str, Any]]:
+ base_map = {
+ str(c.get("id") or c.get("name") or "").strip(): c
+ for c in (baseline.get("categories") or [])
+ if isinstance(c, dict) and str(c.get("id") or c.get("name") or "").strip()
+ }
+ rows: list[dict[str, Any]] = []
+ for c in current.get("categories") or []:
+ if not isinstance(c, dict):
+ continue
+ k = str(c.get("id") or c.get("name") or "").strip()
+ if not k:
+ continue
+ b = base_map.get(k)
+ cur_score = _num(c.get("score"))
+ base_score = _num(b.get("score")) if isinstance(b, dict) else None
+ delta = (cur_score - base_score) if cur_score is not None and base_score is not None else None
+ rows.append({
+ "id": k,
+ "name": str(c.get("name") or c.get("id") or k),
+ "current": round(cur_score) if cur_score is not None else None,
+ "baseline": round(base_score) if base_score is not None else None,
+ "delta": round(delta) if delta is not None else None,
+ })
+ rows.sort(key=lambda x: abs(x.get("delta") or 0), reverse=True)
+ return rows
+
+
+def build_url_set_diff(current: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]:
+ """URLs added or removed between crawl link tables."""
+ def _url_map(payload: dict[str, Any]) -> dict[str, str]:
+ out: dict[str, str] = {}
+ for link in payload.get("links") or []:
+ if not isinstance(link, dict):
+ continue
+ raw = str(link.get("url") or "").strip()
+ k = norm_report_url(raw)
+ if k and k not in out:
+ out[k] = raw
+ return out
+
+ cur_map = _url_map(current)
+ base_map = _url_map(baseline)
+ new_norm = sorted(set(cur_map) - set(base_map))
+ removed_norm = sorted(set(base_map) - set(cur_map))
+ return {
+ "new_urls": [cur_map[k] for k in new_norm],
+ "removed_urls": [base_map[k] for k in removed_norm],
+ "new_count": len(new_norm),
+ "removed_count": len(removed_norm),
+ }
+
+
+def build_full_compare(
+ current: dict[str, Any],
+ baseline: dict[str, Any],
+ *,
+ current_report_id: int | None = None,
+ baseline_report_id: int | None = None,
+) -> dict[str, Any]:
+ cur_health = _score_from_categories(current.get("categories") or [])
+ base_health = _score_from_categories(baseline.get("categories") or [])
+ issue_deltas = build_issue_deltas(current, baseline)
+ truncated_sections: dict[str, bool] = {}
+ if len(issue_deltas) > _ISSUE_DELTA_CAP:
+ truncated_sections["issue_deltas"] = True
+ issue_deltas = issue_deltas[:_ISSUE_DELTA_CAP]
+ link_metrics = build_link_metric_deltas(current, baseline)
+ if len(link_metrics) >= _LINK_METRIC_CAP:
+ truncated_sections["link_metric_deltas"] = True
+ google = build_google_metrics(current, baseline)
+ return {
+ "current_report_id": current_report_id,
+ "baseline_report_id": baseline_report_id,
+ "current_generated_at": current.get("report_generated_at"),
+ "baseline_generated_at": baseline.get("report_generated_at"),
+ "health_score": {
+ "current": cur_health,
+ "baseline": base_health,
+ "delta": (cur_health - base_health) if cur_health is not None and base_health is not None else None,
+ },
+ "category_scores": build_category_scores(current, baseline),
+ "priority_counts": build_priority_counts(current, baseline),
+ "issue_deltas": issue_deltas,
+ "lighthouse_url_deltas": build_lighthouse_url_deltas(current, baseline),
+ "link_metric_deltas": link_metrics,
+ "redirect_deltas": build_redirect_deltas(current, baseline),
+ "security_deltas": build_security_deltas(current, baseline),
+ "duplicate_deltas": build_duplicate_deltas(current, baseline),
+ "tech_deltas": build_tech_deltas(current, baseline),
+ "content_metrics": build_content_metrics(current, baseline),
+ "google_metrics": google.get("metrics") or [],
+ "google_available": google.get("available", False),
+ "seo_health_metrics": build_seo_health_deltas(current, baseline),
+ "truncated_sections": truncated_sections,
+ }
diff --git a/src/website_profiling/text_sanitize.py b/src/website_profiling/text_sanitize.py
new file mode 100644
index 00000000..3cb9bfe5
--- /dev/null
+++ b/src/website_profiling/text_sanitize.py
@@ -0,0 +1,24 @@
+"""Strip lone UTF-16 surrogates so strings are safe for UTF-8 JSON/HTTP."""
+from __future__ import annotations
+
+from typing import Any
+
+
+def strip_surrogates(text: str) -> str:
+ """Replace lone surrogates (invalid in UTF-8) with U+FFFD."""
+ if not text:
+ return text
+ return text.encode("utf-8", errors="replace").decode("utf-8")
+
+
+def sanitize_unicode_deep(obj: Any) -> Any:
+ """Recursively sanitize strings in nested dicts/lists."""
+ if isinstance(obj, str):
+ return strip_surrogates(obj)
+ if isinstance(obj, dict):
+ return {k: sanitize_unicode_deep(v) for k, v in obj.items()}
+ if isinstance(obj, list):
+ return [sanitize_unicode_deep(v) for v in obj]
+ if isinstance(obj, tuple):
+ return tuple(sanitize_unicode_deep(v) for v in obj)
+ return obj
diff --git a/src/website_profiling/tools/audit_tools/__init__.py b/src/website_profiling/tools/audit_tools/__init__.py
new file mode 100644
index 00000000..47d05972
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/__init__.py
@@ -0,0 +1,5 @@
+"""Read-only audit query tools for MCP and in-app chat."""
+from .context import AuditToolContext
+from .registry import TOOL_DEFINITIONS, dispatch_tool
+
+__all__ = ["AuditToolContext", "TOOL_DEFINITIONS", "dispatch_tool"]
diff --git a/src/website_profiling/tools/audit_tools/_slice.py b/src/website_profiling/tools/audit_tools/_slice.py
new file mode 100644
index 00000000..22f71a13
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/_slice.py
@@ -0,0 +1,132 @@
+"""Shared list slicing and payload field helpers for audit tools."""
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import pandas as pd
+
+
+def parse_limit(raw: Any, default: int, max_cap: int) -> int:
+ try:
+ limit = int(raw)
+ except (TypeError, ValueError):
+ limit = default
+ return max(1, min(limit, max_cap))
+
+
+def cap_list(items: list[Any], limit: int, *, max_cap: int | None = None) -> dict[str, Any]:
+ cap = max_cap if max_cap is not None else limit
+ limit = max(1, min(limit, cap))
+ total = len(items)
+ truncated = total > limit
+ return {"items": items[:limit], "total": total, "truncated": truncated}
+
+
+def payload_field(
+ payload: dict[str, Any],
+ key: str,
+ limit: int = 50,
+ *,
+ max_cap: int = 50,
+ filter_fn: Callable[[Any], bool] | None = None,
+ item_key: str = "items",
+) -> dict[str, Any]:
+ raw = payload.get(key)
+ if raw is None:
+ return {item_key: [], "total": 0, "truncated": False, "missing": True}
+ if not isinstance(raw, list):
+ return {item_key: [raw] if raw else [], "total": 1 if raw else 0, "truncated": False}
+ items = raw
+ if filter_fn:
+ items = [x for x in items if filter_fn(x)]
+ sliced = cap_list(items, limit, max_cap=max_cap)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def payload_dict_slice(
+ payload: dict[str, Any],
+ key: str,
+ *,
+ fields: list[str] | None = None,
+) -> dict[str, Any]:
+ raw = payload.get(key)
+ if not isinstance(raw, dict):
+ return {"data": None, "missing": True}
+ if fields:
+ return {"data": {k: raw.get(k) for k in fields if k in raw}, "missing": False}
+ return {"data": raw, "missing": False}
+
+
+def crawl_filter(
+ df: pd.DataFrame | None,
+ *,
+ status: str = "",
+ url_contains: str = "",
+ has_schema: bool | None = None,
+ schema_type: str = "",
+ limit: int = 30,
+ max_cap: int = 30,
+) -> dict[str, Any]:
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ records = df.to_dict(orient="records")
+ if status:
+ records = [r for r in records if str(r.get("status") or "") == status]
+ if url_contains:
+ needle = url_contains.lower()
+ records = [r for r in records if needle in str(r.get("url") or "").lower()]
+ if has_schema is not None:
+ records = [
+ r for r in records
+ if _row_has_schema(r) == has_schema
+ ]
+ if schema_type:
+ needle = schema_type.lower()
+ records = [r for r in records if needle in _row_schema_types(r)]
+ pages = [
+ {
+ "url": str(r.get("url") or ""),
+ "status": str(r.get("status") or ""),
+ "title": str(r.get("title") or ""),
+ "has_schema": _row_has_schema(r),
+ "schema_types": _row_schema_types_list(r),
+ }
+ for r in records
+ ]
+ sliced = cap_list(pages, limit, max_cap=max_cap)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _row_has_schema(row: dict[str, Any]) -> bool:
+ val = str(row.get("has_schema") or "").lower()
+ return val in ("true", "1", "yes")
+
+
+def _parse_page_analysis(row: dict[str, Any]) -> dict[str, Any]:
+ import json
+
+ pa = row.get("page_analysis")
+ if isinstance(pa, dict):
+ return pa
+ if isinstance(pa, str) and pa.strip():
+ try:
+ parsed = json.loads(pa)
+ return parsed if isinstance(parsed, dict) else {}
+ except json.JSONDecodeError:
+ return {}
+ return {}
+
+
+def _row_schema_types_list(row: dict[str, Any]) -> list[str]:
+ pa = _parse_page_analysis(row)
+ types = pa.get("json_ld_types") or pa.get("schema_types") or []
+ if isinstance(types, str):
+ return [types] if types else []
+ if isinstance(types, list):
+ return [str(t) for t in types if t]
+ return []
+
+
+def _row_schema_types(row: dict[str, Any]) -> str:
+ return " ".join(_row_schema_types_list(row)).lower()
diff --git a/src/website_profiling/tools/audit_tools/backlinks.py b/src/website_profiling/tools/audit_tools/backlinks.py
new file mode 100644
index 00000000..b63ac491
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/backlinks.py
@@ -0,0 +1,138 @@
+"""Backlinks and competitor link gap tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.gsc_links_store import read_gsc_links_status
+from ._slice import cap_list, parse_limit, payload_dict_slice
+from .context import AuditToolContext
+
+
+def get_gsc_links_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required for GSC links data"}
+ data = scoped.load_gsc_links(conn)
+ if not data:
+ return {"error": "no GSC links data — import GSC Links CSV in Integrations", "missing": True}
+ limit = parse_limit(args.get("limit"), 20, 50)
+ out = {
+ "imported_at": data.get("imported_at"),
+ "export_types": data.get("export_types") or [],
+ "row_counts": data.get("row_counts") or {},
+ "top_linking_sites": cap_list(list(data.get("top_linking_sites") or []), limit, max_cap=50)["items"],
+ "top_linked_pages": cap_list(list(data.get("top_linked_pages") or []), limit, max_cap=50)["items"],
+ "sample_links_full_count": data.get("sample_links_full_count"),
+ "latest_links_full_count": data.get("latest_links_full_count"),
+ "property_id": scoped.property_id,
+ }
+ return out
+
+
+def get_gsc_links_import_status(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ return read_gsc_links_status(conn, int(scoped.property_id))
+
+
+def get_competitor_link_gap(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ gap = payload.get("competitor_link_gap")
+ if not isinstance(gap, dict):
+ return {"error": "competitor_link_gap not in report — configure competitor_domains and import GSC links", "missing": True}
+ return {"competitor_link_gap": gap}
+
+
+def get_bing_backlinks_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "bing_backlinks")
+
+
+def get_gsc_sample_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ data = scoped.load_gsc_links(conn)
+ if not data:
+ return {"error": "no GSC links data", "missing": True, "links": [], "total": 0, "truncated": False}
+ links = list(data.get("sample_links") or [])
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(links, limit, max_cap=100)
+ return {
+ "links": sliced["items"],
+ "total": data.get("sample_links_full_count") or sliced["total"],
+ "truncated": sliced["truncated"],
+ "full_count": data.get("sample_links_full_count"),
+ }
+
+
+def get_gsc_latest_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ data = scoped.load_gsc_links(conn)
+ if not data:
+ return {"error": "no GSC links data", "missing": True, "links": [], "total": 0, "truncated": False}
+ links = list(data.get("latest_links") or [])
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(links, limit, max_cap=100)
+ return {
+ "links": sliced["items"],
+ "total": data.get("latest_links_full_count") or sliced["total"],
+ "truncated": sliced["truncated"],
+ "full_count": data.get("latest_links_full_count"),
+ }
+
+
+def get_third_party_links_overlay(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ data = scoped.load_gsc_links(conn)
+ if not data:
+ return {"error": "no GSC links data", "missing": True, "overlays": []}
+ overlays = data.get("third_party_overlays") or []
+ if not isinstance(overlays, list):
+ overlays = []
+ provider = str(args.get("provider") or "").strip().lower()
+ if provider:
+ overlays = [
+ o for o in overlays
+ if isinstance(o, dict) and str(o.get("provider") or "").lower() == provider
+ ]
+ return {"overlays": overlays, "count": len(overlays)}
+
+
+def get_backlinks_velocity(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ limit = parse_limit(args.get("limit"), 52, 52)
+ cur = conn.execute(
+ """SELECT captured_at, referring_domains, top_domains
+ FROM gsc_links_snapshots
+ WHERE property_id = %s
+ ORDER BY captured_at ASC
+ LIMIT %s""",
+ (int(scoped.property_id), limit),
+ )
+ snapshots = []
+ for row in cur.fetchall() or []:
+ captured = row["captured_at"] if hasattr(row, "keys") else row[0]
+ domains = row["referring_domains"] if hasattr(row, "keys") else row[1]
+ top = row["top_domains"] if hasattr(row, "keys") else row[2]
+ snapshots.append({
+ "captured_at": captured.isoformat() if hasattr(captured, "isoformat") else str(captured or ""),
+ "referring_domains": domains,
+ "top_domains": top,
+ })
+ return {"snapshots": snapshots, "count": len(snapshots), "property_id": scoped.property_id}
diff --git a/src/website_profiling/tools/audit_tools/charts.py b/src/website_profiling/tools/audit_tools/charts.py
new file mode 100644
index 00000000..edc64197
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/charts.py
@@ -0,0 +1,104 @@
+"""Chart and aggregate distribution tools from report payload."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+
+def get_crawl_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ summary = payload.get("summary") or {}
+ return {
+ "summary": summary,
+ "crawl_run_id": payload.get("crawl_run_id"),
+ "crawl_run_created_at": payload.get("crawl_run_created_at"),
+ "report_generated_at": payload.get("report_generated_at"),
+ }
+
+
+def _label_value_pair(payload: dict[str, Any], labels_key: str, values_key: str) -> list[dict[str, Any]]:
+ labels = payload.get(labels_key) or []
+ values = payload.get(values_key) or []
+ if not isinstance(labels, list):
+ labels = []
+ if not isinstance(values, list):
+ values = []
+ out: list[dict[str, Any]] = []
+ for i, label in enumerate(labels):
+ val = values[i] if i < len(values) else None
+ out.append({"label": label, "value": val})
+ return out
+
+
+def get_mime_type_breakdown(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {"items": _label_value_pair(payload, "mime_labels", "mime_values")}
+
+
+def get_title_length_distribution(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {"items": _label_value_pair(payload, "title_labels", "title_counts")}
+
+
+def get_domain_link_distribution(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {"items": _label_value_pair(payload, "domain_labels", "domain_values")}
+
+
+def get_outlink_distribution(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {"items": _label_value_pair(payload, "outlink_labels", "outlink_counts")}
+
+
+def get_issue_priority_breakdown(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """Chart-friendly issue counts by priority (for chat visualization)."""
+ from .report import get_report_summary
+
+ summary = get_report_summary(conn, ctx, args)
+ if summary.get("error"):
+ return summary
+ counts = summary.get("issue_counts") or {}
+ if not isinstance(counts, dict):
+ counts = {}
+ items = [
+ {"label": label, "value": int(counts[label])}
+ for label in ("Critical", "High", "Medium", "Low")
+ if counts.get(label)
+ ]
+ return {
+ "items": items,
+ "total_issues": summary.get("total_issues"),
+ "health_score": summary.get("health_score"),
+ }
+
+
+def get_top_crawled_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ top = payload.get("top_pages") or []
+ if not isinstance(top, list):
+ top = []
+ limit = parse_limit(args.get("limit"), 20, 50)
+ sliced = cap_list(top, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/compare.py b/src/website_profiling/tools/audit_tools/compare.py
new file mode 100644
index 00000000..7453fb82
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/compare.py
@@ -0,0 +1,48 @@
+"""Report comparison tool."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...db.report_store import read_report_payload
+from ...reporting.compare_payload import build_full_compare
+from .context import AuditToolContext
+
+
+def compare_reports(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ baseline_id = args.get("baseline_report_id")
+ if baseline_id is None:
+ return {"error": "baseline_report_id is required"}
+ try:
+ baseline_rid = int(baseline_id)
+ except (TypeError, ValueError):
+ return {"error": "invalid baseline_report_id"}
+
+ current_rid = scoped.report_id
+ if current_rid is None:
+ cur_row = conn.execute("SELECT id FROM report_payload ORDER BY id DESC LIMIT 1").fetchone()
+ if cur_row is None:
+ return {"error": "no current report found"}
+ current_rid = int(_row_id(cur_row))
+
+ current = read_report_payload(conn, current_rid)
+ baseline = read_report_payload(conn, baseline_rid)
+ if not current:
+ return {"error": f"report {current_rid} not found"}
+ if not baseline:
+ return {"error": f"report {baseline_rid} not found"}
+
+ return build_full_compare(
+ current,
+ baseline,
+ current_report_id=current_rid,
+ baseline_report_id=baseline_rid,
+ )
+
+
+def _row_id(row: Any) -> Any:
+ if hasattr(row, "keys"):
+ return row["id"]
+ return row[0]
diff --git a/src/website_profiling/tools/audit_tools/compare_helpers.py b/src/website_profiling/tools/audit_tools/compare_helpers.py
new file mode 100644
index 00000000..4005fa6e
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/compare_helpers.py
@@ -0,0 +1,46 @@
+"""Shared helpers for compare slice tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...db.report_store import read_report_payload
+from .context import AuditToolContext
+
+
+def _row_id(row: Any) -> Any:
+ if hasattr(row, "keys"):
+ return row["id"]
+ return row[0]
+
+
+def load_compare_pair(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+) -> tuple[dict[str, Any] | None, dict[str, Any] | None, int | None, int | None, dict[str, Any] | None]:
+ """Return (current, baseline, current_rid, baseline_rid, error)."""
+ scoped = ctx.with_args(args)
+ baseline_id = args.get("baseline_report_id")
+ if baseline_id is None:
+ return None, None, None, None, {"error": "baseline_report_id is required"}
+ try:
+ baseline_rid = int(baseline_id)
+ except (TypeError, ValueError):
+ return None, None, None, None, {"error": "invalid baseline_report_id"}
+
+ current_rid = scoped.report_id
+ if current_rid is None:
+ cur_row = conn.execute("SELECT id FROM report_payload ORDER BY id DESC LIMIT 1").fetchone()
+ if cur_row is None:
+ return None, None, None, None, {"error": "no current report found"}
+ current_rid = int(_row_id(cur_row))
+
+ current = read_report_payload(conn, current_rid)
+ baseline = read_report_payload(conn, baseline_rid)
+ if not current:
+ return None, None, None, None, {"error": f"report {current_rid} not found"}
+ if not baseline:
+ return None, None, None, None, {"error": f"report {baseline_rid} not found"}
+ return current, baseline, current_rid, baseline_rid, None
diff --git a/src/website_profiling/tools/audit_tools/compare_slices.py b/src/website_profiling/tools/audit_tools/compare_slices.py
new file mode 100644
index 00000000..60c32c4f
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/compare_slices.py
@@ -0,0 +1,136 @@
+"""Focused compare/drift slice tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...reporting.compare_payload import (
+ build_category_scores,
+ build_issue_deltas,
+ build_lighthouse_url_deltas,
+ build_link_metric_deltas,
+ build_redirect_deltas,
+ build_seo_health_deltas,
+ build_url_set_diff,
+)
+from ._slice import cap_list, parse_limit
+from .compare_helpers import load_compare_pair
+from .context import AuditToolContext
+
+
+def _compare_meta(current_rid: int | None, baseline_rid: int | None, current: dict, baseline: dict) -> dict[str, Any]:
+ return {
+ "current_report_id": current_rid,
+ "baseline_report_id": baseline_rid,
+ "current_generated_at": current.get("report_generated_at"),
+ "baseline_generated_at": baseline.get("report_generated_at"),
+ }
+
+
+def compare_issue_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ deltas = build_issue_deltas(current, baseline)
+ limit = parse_limit(args.get("limit"), 50, 100)
+ sliced = cap_list(deltas, limit, max_cap=100)
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "issue_deltas": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def compare_category_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "category_scores": build_category_scores(current, baseline),
+ }
+
+
+def compare_seo_health_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "seo_health_metrics": build_seo_health_deltas(current, baseline),
+ }
+
+
+def compare_lighthouse_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ deltas = build_lighthouse_url_deltas(current, baseline)
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(deltas, limit, max_cap=50)
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "lighthouse_url_deltas": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def compare_url_set_diff(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ diff = build_url_set_diff(current, baseline)
+ limit = parse_limit(args.get("limit"), 50, 200)
+ new_urls = diff.get("new_urls") or []
+ removed_urls = diff.get("removed_urls") or []
+ new_sliced = cap_list(new_urls, limit, max_cap=200)
+ removed_sliced = cap_list(removed_urls, limit, max_cap=200)
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "new_urls": new_sliced["items"],
+ "new_count": diff.get("new_count", len(new_urls)),
+ "new_truncated": new_sliced["truncated"],
+ "removed_urls": removed_sliced["items"],
+ "removed_count": diff.get("removed_count", len(removed_urls)),
+ "removed_truncated": removed_sliced["truncated"],
+ }
+
+
+def compare_redirect_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ deltas = build_redirect_deltas(current, baseline)
+ limit = parse_limit(args.get("limit"), 50, 100)
+ sliced = cap_list(deltas, limit, max_cap=100)
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "redirect_deltas": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def compare_link_metric_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args)
+ if err:
+ return err
+ assert current is not None and baseline is not None
+ deltas = build_link_metric_deltas(current, baseline)
+ limit = parse_limit(args.get("limit"), 50, 200)
+ sliced = cap_list(deltas, limit, max_cap=200)
+ return {
+ **_compare_meta(cur_rid, base_rid, current, baseline),
+ "link_metric_deltas": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
diff --git a/src/website_profiling/tools/audit_tools/content.py b/src/website_profiling/tools/audit_tools/content.py
new file mode 100644
index 00000000..c4bee0a8
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/content.py
@@ -0,0 +1,70 @@
+"""Content quality and social coverage tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit, payload_dict_slice, payload_field
+from .context import AuditToolContext
+
+
+def get_content_analytics(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "content_analytics")
+
+
+def get_content_duplicates(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "duplicates": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ return payload_field(payload, "content_duplicates", limit, max_cap=50, item_key="duplicates")
+
+
+def get_social_coverage(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "social_coverage")
+
+
+def get_keyword_opportunities(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "keyword_opportunities")
+
+
+def get_ner_site_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "ner_site_summary")
+
+
+def list_thin_content_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ analytics = payload.get("content_analytics") or {}
+ thin = analytics.get("thin_pages") if isinstance(analytics, dict) else None
+ if not isinstance(thin, list) or not thin:
+ thin_count = int((payload.get("seo_health") or {}).get("thin_content") or 0)
+ return {
+ "pages": [],
+ "total": thin_count,
+ "truncated": False,
+ "note": "thin page URLs not listed; only count available in seo_health",
+ }
+ sliced = cap_list(thin, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/context.py b/src/website_profiling/tools/audit_tools/context.py
new file mode 100644
index 00000000..9f09ec87
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/context.py
@@ -0,0 +1,102 @@
+"""Execution context for audit tools (property + report scope)."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from psycopg import Connection
+
+from ...db.crawl_store import get_latest_crawl_run_id, read_crawl
+from ...db.property_store import get_property_by_id
+from ...db.report_store import read_report_payload
+from ...integrations.google.gsc_links_store import read_latest_gsc_links_data
+from ...integrations.google.keyword_store import read_latest_keyword_data
+from ...integrations.google.store import read_latest_google_data
+
+
+@dataclass
+class AuditToolContext:
+ property_id: Optional[int] = None
+ report_id: Optional[int] = None
+
+ def load_payload(self, conn: Connection) -> dict[str, Any]:
+ payload = read_report_payload(conn, self.report_id)
+ return payload if isinstance(payload, dict) else {}
+
+ def load_crawl_df(self, conn: Connection):
+ payload = self.load_payload(conn)
+ run_id = payload.get("crawl_run_id")
+ try:
+ rid = int(run_id) if run_id is not None else None
+ except (TypeError, ValueError):
+ rid = None
+ if rid is None:
+ rid = get_latest_crawl_run_id(conn)
+ return read_crawl(conn, rid)
+
+ def load_google(self, conn: Connection) -> Optional[dict[str, Any]]:
+ google = read_latest_google_data(conn, self.property_id)
+ if google:
+ return google
+ payload = self.load_payload(conn)
+ embedded = payload.get("google")
+ return embedded if isinstance(embedded, dict) else None
+
+ def load_keywords(self, conn: Connection) -> Optional[dict[str, Any]]:
+ kw = read_latest_keyword_data(conn, self.property_id)
+ if kw:
+ return kw
+ payload = self.load_payload(conn)
+ embedded = payload.get("keywords")
+ return embedded if isinstance(embedded, dict) else None
+
+ def load_gsc_links(self, conn: Connection) -> Optional[dict[str, Any]]:
+ links = read_latest_gsc_links_data(conn, self.property_id, for_report=False)
+ if links:
+ return links
+ payload = self.load_payload(conn)
+ embedded = payload.get("gsc_links")
+ return embedded if isinstance(embedded, dict) else None
+
+ def load_report_payload_by_id(self, conn: Connection, report_id: int) -> dict[str, Any]:
+ data = read_report_payload(conn, report_id)
+ return data if isinstance(data, dict) else {}
+
+ def resolve_property_domain(self, conn: Connection) -> str:
+ if self.property_id is not None:
+ prop = get_property_by_id(conn, int(self.property_id))
+ if prop:
+ domain = str(prop.get("canonical_domain") or "").strip().lower()
+ if domain:
+ return domain
+ payload = self.load_payload(conn)
+ for key in ("canonical_domain",):
+ val = str(payload.get(key) or "").strip().lower()
+ if val:
+ return val
+ top = payload.get("top_pages") or []
+ if top and isinstance(top[0], dict):
+ from urllib.parse import urlparse
+
+ host = urlparse(str(top[0].get("url") or "")).hostname
+ if host:
+ return host.lower()
+ return ""
+
+ def with_args(self, args: dict[str, Any]) -> AuditToolContext:
+ """Merge tool args property_id/report_id when provided."""
+ pid = args.get("property_id")
+ rid = args.get("report_id")
+ new_pid = self.property_id
+ new_rid = self.report_id
+ if pid is not None:
+ try:
+ new_pid = int(pid)
+ except (TypeError, ValueError):
+ pass
+ if rid is not None:
+ try:
+ new_rid = int(rid)
+ except (TypeError, ValueError):
+ pass
+ return AuditToolContext(property_id=new_pid, report_id=new_rid)
diff --git a/src/website_profiling/tools/audit_tools/crawl.py b/src/website_profiling/tools/audit_tools/crawl.py
new file mode 100644
index 00000000..43fb33ab
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/crawl.py
@@ -0,0 +1,379 @@
+"""Crawl page query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.page_lookup import slice_from_google_row
+from ._slice import _parse_page_analysis, cap_list, parse_limit, payload_dict_slice
+from .context import AuditToolContext
+
+_PAGE_LIMIT_MAX = 30
+
+
+def _page_row(rec: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "url": str(rec.get("url") or ""),
+ "status": str(rec.get("status") or ""),
+ "title": str(rec.get("title") or ""),
+ "inlinks": rec.get("inlinks"),
+ }
+
+
+def search_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+
+ status_filter = str(args.get("status") or "").strip()
+ url_contains = str(args.get("url_contains") or "").strip().lower()
+ limit = args.get("limit", _PAGE_LIMIT_MAX)
+ try:
+ limit = int(limit)
+ except (TypeError, ValueError):
+ limit = _PAGE_LIMIT_MAX
+ limit = max(1, min(limit, _PAGE_LIMIT_MAX))
+
+ records = df.to_dict(orient="records")
+ if status_filter:
+ records = [r for r in records if str(r.get("status") or "") == status_filter]
+ if url_contains:
+ records = [r for r in records if url_contains in str(r.get("url") or "").lower()]
+
+ total = len(records)
+ truncated = total > limit
+ pages = [_page_row(r) for r in records[:limit]]
+ return {"pages": pages, "total": total, "truncated": truncated}
+
+
+def get_page_details(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip().rstrip("/")
+ if not url:
+ return {"error": "url is required"}
+
+ df = scoped.load_crawl_df(conn)
+ crawl_row: dict[str, Any] | None = None
+ if df is not None and not df.empty and "url" in df.columns:
+ norm = url.rstrip("/")
+ for _, row in df.iterrows():
+ row_url = str(row.get("url") or "").rstrip("/")
+ if row_url == norm or row_url == url:
+ crawl_row = {
+ "url": row_url,
+ "status": str(row.get("status") or ""),
+ "title": str(row.get("title") or ""),
+ "meta_description": str(row.get("meta_description") or ""),
+ "h1": row.get("h1"),
+ "word_count": row.get("word_count"),
+ "inlinks": row.get("inlinks"),
+ "outlinks": row.get("outlinks"),
+ "content_type": str(row.get("content_type") or ""),
+ }
+ break
+
+ payload = scoped.load_payload(conn)
+ lighthouse_by_url = payload.get("lighthouse_by_url") or {}
+ lh = lighthouse_by_url.get(url) or lighthouse_by_url.get(url + "/")
+ if not lh and crawl_row:
+ lh = lighthouse_by_url.get(crawl_row.get("url", ""))
+
+ google_raw = scoped.load_google(conn)
+ gsc_ga4 = None
+ if google_raw:
+ gsc_ga4 = slice_from_google_row(google_raw, url)
+
+ return {
+ "url": url,
+ "crawl": crawl_row,
+ "lighthouse": lh if isinstance(lh, dict) else None,
+ "gsc_ga4": gsc_ga4,
+ "found_in_crawl": crawl_row is not None,
+ }
+
+
+def get_internal_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip().rstrip("/")
+ if not url:
+ return {"error": "url is required"}
+
+ from ...db.crawl_store import read_edges
+
+ payload = scoped.load_payload(conn)
+ run_id = payload.get("crawl_run_id")
+ try:
+ rid = int(run_id) if run_id is not None else None
+ except (TypeError, ValueError):
+ rid = None
+
+ edges = read_edges(conn, rid)
+ outlinks = [b for a, b in edges if a.rstrip("/") == url]
+ inlinks = [a for a, b in edges if b.rstrip("/") == url]
+ limit = 50
+ return {
+ "url": url,
+ "outlinks": outlinks[:limit],
+ "inlinks": inlinks[:limit],
+ "outlink_count": len(outlinks),
+ "inlink_count": len(inlinks),
+ "truncated": len(outlinks) > limit or len(inlinks) > limit,
+ }
+
+
+def list_redirects(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "redirects": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 50, 50)
+ redirects = payload.get("redirects") or []
+ sliced = cap_list(redirects if isinstance(redirects, list) else [], limit, max_cap=50)
+ return {"redirects": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_broken_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "broken": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 50, 50)
+ issues = payload.get("issues") or {}
+ broken = issues.get("broken") if isinstance(issues, dict) else []
+ sliced = cap_list(broken if isinstance(broken, list) else [], limit, max_cap=50)
+ return {"broken": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_status_code_breakdown(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {
+ "status_counts": payload.get("status_counts") or {},
+ "summary": {
+ k: (payload.get("summary") or {}).get(k)
+ for k in ("total_urls", "count_2xx", "count_3xx", "count_4xx", "count_5xx", "success_rate")
+ },
+ }
+
+
+def get_response_time_stats(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "response_time_stats")
+
+
+def get_depth_distribution(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "depth_distribution")
+
+
+def get_crawl_segments(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ segments = payload.get("crawl_segments")
+ if not segments:
+ return {"error": "crawl_segments not in report — set crawl_path_segments in config", "missing": True}
+ return {"crawl_segments": segments}
+
+
+def get_browser_diagnostics_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ meta = payload.get("report_meta") or {}
+ browser = meta.get("browser_diagnostics") if isinstance(meta, dict) else None
+ if browser:
+ return {"browser_diagnostics": browser}
+ return {"browser_diagnostics": None, "note": "no browser diagnostics in report_meta"}
+
+
+def get_seo_health(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "seo_health")
+
+
+def _status_prefix_pages(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ prefix: str,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ records = df.to_dict(orient="records")
+ pages = [
+ {"url": str(r.get("url") or ""), "status": str(r.get("status") or ""), "title": str(r.get("title") or "")}
+ for r in records
+ if str(r.get("status") or "").startswith(prefix)
+ ]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_status_4xx_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _status_prefix_pages(conn, ctx, args, "4")
+
+
+def list_status_5xx_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _status_prefix_pages(conn, ctx, args, "5")
+
+
+def get_page_analysis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip().rstrip("/")
+ if not url:
+ return {"error": "url is required"}
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"error": "no crawl data", "url": url}
+ for _, row in df.iterrows():
+ row_url = str(row.get("url") or "").rstrip("/")
+ if row_url == url or row_url == url.rstrip("/"):
+ rec = row.to_dict()
+ return {"url": row_url, "page_analysis": _parse_page_analysis(rec), "fetch_method": rec.get("fetch_method")}
+ return {"error": "url not found in crawl", "url": url}
+
+
+def search_pages_advanced(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ records = df.to_dict(orient="records")
+ status_filter = str(args.get("status") or "").strip()
+ url_contains = str(args.get("url_contains") or "").strip().lower()
+ fetch_method = str(args.get("fetch_method") or "").strip().lower()
+ noindex_only = args.get("noindex_only")
+ missing_title = args.get("missing_title")
+ min_word = args.get("min_word_count")
+ max_word = args.get("max_word_count")
+
+ def _truthy(val: Any) -> bool:
+ return str(val or "").lower() in ("true", "1", "yes")
+
+ def _flag(val: Any) -> bool:
+ return val is True or _truthy(val)
+
+ filtered = []
+ for r in records:
+ if status_filter and str(r.get("status") or "") != status_filter:
+ continue
+ if url_contains and url_contains not in str(r.get("url") or "").lower():
+ continue
+ if fetch_method and str(r.get("fetch_method") or "").lower() != fetch_method:
+ continue
+ if _flag(noindex_only) and not _truthy(r.get("noindex")):
+ continue
+ if _flag(missing_title) and str(r.get("title") or "").strip():
+ continue
+ wc = r.get("word_count")
+ try:
+ wc_val = int(wc) if wc is not None else None
+ except (TypeError, ValueError):
+ wc_val = None
+ if min_word is not None:
+ try:
+ if wc_val is None or wc_val < int(min_word):
+ continue
+ except (TypeError, ValueError):
+ pass
+ if max_word is not None:
+ try:
+ if wc_val is None or wc_val > int(max_word):
+ continue
+ except (TypeError, ValueError):
+ pass
+ filtered.append({
+ "url": str(r.get("url") or ""),
+ "status": str(r.get("status") or ""),
+ "title": str(r.get("title") or ""),
+ "word_count": wc_val,
+ "noindex": _truthy(r.get("noindex")),
+ "fetch_method": str(r.get("fetch_method") or ""),
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(filtered, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_pages_with_console_errors(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ pages = []
+ for _, row in df.iterrows():
+ pa = _parse_page_analysis(row.to_dict())
+ errors = pa.get("console_errors") or pa.get("js_errors") or []
+ if not errors:
+ continue
+ if isinstance(errors, str):
+ errors = [errors]
+ pages.append({
+ "url": str(row.get("url") or ""),
+ "error_count": len(errors) if isinstance(errors, list) else 1,
+ "errors": (errors[:5] if isinstance(errors, list) else [errors]),
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_pages_by_fetch_method(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ method = str(args.get("fetch_method") or "").strip().lower()
+ if not method:
+ return {"error": "fetch_method is required (e.g. static or rendered)"}
+ return search_pages_advanced(conn, ctx, {**args, "fetch_method": method})
+
+
+def get_crawl_links_table(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "links": [], "total": 0, "truncated": False}
+ links = payload.get("links") or []
+ if not isinstance(links, list):
+ links = []
+ url_contains = str(args.get("url_contains") or "").strip().lower()
+ if url_contains:
+ links = [l for l in links if isinstance(l, dict) and url_contains in str(l.get("url") or "").lower()]
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(links, limit, max_cap=100)
+ return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_graph_edges_sample(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "edges": [], "total": 0, "truncated": False}
+ edges = payload.get("graph_edges") or []
+ if not isinstance(edges, list):
+ edges = []
+ nodes = payload.get("graph_nodes")
+ limit = parse_limit(args.get("limit"), 50, 200)
+ sliced = cap_list(edges, limit, max_cap=200)
+ return {
+ "edges": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "graph_node_count": len(nodes) if isinstance(nodes, list) else None,
+ }
diff --git a/src/website_profiling/tools/audit_tools/google.py b/src/website_profiling/tools/audit_tools/google.py
new file mode 100644
index 00000000..f97abbe0
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/google.py
@@ -0,0 +1,137 @@
+"""Google Search Console / GA4 summary tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.page_lookup import slice_from_google_row
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+
+def get_google_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "property_id": scoped.property_id}
+
+ gsc = data.get("gsc") if isinstance(data.get("gsc"), dict) else {}
+ ga4 = data.get("ga4") if isinstance(data.get("ga4"), dict) else {}
+ gsc_summary = gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {}
+ ga4_summary = ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {}
+
+ top_queries = gsc.get("top_queries") or []
+ top_pages = gsc.get("top_pages") or []
+ if isinstance(top_queries, list):
+ top_queries = top_queries[:10]
+ if isinstance(top_pages, list):
+ top_pages = top_pages[:10]
+
+ return {
+ "fetched_at": data.get("fetched_at"),
+ "date_range": data.get("date_range"),
+ "gsc": {
+ "site_url": gsc.get("site_url"),
+ "summary": gsc_summary,
+ "top_queries": top_queries,
+ "top_pages": top_pages,
+ },
+ "ga4": {
+ "property_id": ga4.get("property_id"),
+ "summary": ga4_summary,
+ "top_pages": (ga4.get("top_pages") or [])[:10] if isinstance(ga4.get("top_pages"), list) else [],
+ },
+ "errors": data.get("errors") or [],
+ "property_id": scoped.property_id,
+ }
+
+
+def get_gsc_top_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "queries": [], "total": 0}
+ gsc = data.get("gsc") if isinstance(data.get("gsc"), dict) else {}
+ queries = gsc.get("top_queries") or gsc.get("queries") or []
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(queries if isinstance(queries, list) else [], limit, max_cap=50)
+ return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_gsc_top_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "pages": [], "total": 0}
+ gsc = data.get("gsc") if isinstance(data.get("gsc"), dict) else {}
+ pages = gsc.get("top_pages") or gsc.get("pages") or []
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages if isinstance(pages, list) else [], limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_ga4_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found"}
+ ga4 = data.get("ga4") if isinstance(data.get("ga4"), dict) else {}
+ if not ga4:
+ return {"error": "no GA4 data — connect GA4 property in Integrations", "missing": True}
+ top_pages = ga4.get("top_pages") or []
+ limit = parse_limit(args.get("limit"), 20, 50)
+ sliced = cap_list(top_pages if isinstance(top_pages, list) else [], limit, max_cap=50)
+ return {
+ "property_id": ga4.get("property_id"),
+ "summary": ga4.get("summary") or {},
+ "top_pages": sliced["items"],
+ "fetched_at": data.get("fetched_at"),
+ }
+
+
+def get_gsc_page_query_slice(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip()
+ if not url:
+ return {"error": "url is required"}
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found"}
+ slice_data = slice_from_google_row(data, url)
+ return {"url": url, "gsc_ga4": slice_data}
+
+
+def get_ga4_page_metrics(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ path = str(args.get("path") or args.get("url") or "").strip()
+ if not path:
+ return {"error": "path or url is required"}
+ data = scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found"}
+ ga4 = data.get("ga4") if isinstance(data.get("ga4"), dict) else {}
+ if not ga4:
+ return {"error": "no GA4 data", "missing": True}
+ from ...integrations.google.normalize import url_to_path
+
+ def _ga4_path_key(raw: str) -> str:
+ text = str(raw or "").strip()
+ if text.startswith(("http://", "https://")):
+ text = url_to_path(text)
+ if not text.startswith("/"):
+ text = f"/{text}"
+ return text.lower().rstrip("/") or "/"
+
+ needle = _ga4_path_key(path)
+ for row in ga4.get("top_pages") or []:
+ if not isinstance(row, dict):
+ continue
+ row_path = _ga4_path_key(str(row.get("path") or row.get("page") or ""))
+ if row_path == needle:
+ return {"path": path, "metrics": row, "fetched_at": data.get("fetched_at")}
+ slice_data = slice_from_google_row(data, path)
+ ga4_slice = slice_data.get("ga4") if isinstance(slice_data, dict) else None
+ if ga4_slice:
+ return {"path": path, "metrics": ga4_slice, "fetched_at": data.get("fetched_at")}
+ return {"error": "path not found in GA4 top pages", "path": path, "missing": True}
diff --git a/src/website_profiling/tools/audit_tools/health.py b/src/website_profiling/tools/audit_tools/health.py
new file mode 100644
index 00000000..80b98fa8
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/health.py
@@ -0,0 +1,147 @@
+"""Health history query tools."""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from psycopg import Connection
+
+from ...db._common import _row_field
+from .context import AuditToolContext
+
+
+def get_health_history(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ property_id = scoped.property_id
+ if property_id is None:
+ return {"error": "property_id is required"}
+
+ limit = args.get("limit", 10)
+ try:
+ limit = int(limit)
+ except (TypeError, ValueError):
+ limit = 10
+ limit = max(1, min(limit, 30))
+
+ cur = conn.execute(
+ """SELECT health_score, category_scores, issue_counts, generated_at, report_id
+ FROM audit_health_snapshots
+ WHERE property_id = %s
+ ORDER BY generated_at DESC, id DESC
+ LIMIT %s""",
+ (property_id, limit),
+ )
+ snapshots = []
+ for row in cur.fetchall() or []:
+ cat_scores = _row_field(row, "category_scores", index=1)
+ issue_counts = _row_field(row, "issue_counts", index=2)
+ if isinstance(cat_scores, str):
+ try:
+ cat_scores = json.loads(cat_scores)
+ except json.JSONDecodeError:
+ cat_scores = {}
+ if isinstance(issue_counts, str):
+ try:
+ issue_counts = json.loads(issue_counts)
+ except json.JSONDecodeError:
+ issue_counts = {}
+ gen = _row_field(row, "generated_at", index=3)
+ snapshots.append({
+ "health_score": _row_field(row, "health_score", index=0),
+ "category_scores": cat_scores,
+ "issue_counts": issue_counts,
+ "generated_at": gen.isoformat() if hasattr(gen, "isoformat") else str(gen or ""),
+ "report_id": _row_field(row, "report_id", index=4),
+ })
+
+ return {
+ "property_id": property_id,
+ "snapshots": snapshots,
+ "count": len(snapshots),
+ }
+
+
+def list_report_history(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ from ._slice import parse_limit as _parse_limit
+
+ limit = _parse_limit(args.get("limit"), 20, 50)
+ clauses: list[str] = []
+ params: list[Any] = []
+ domain = scoped.resolve_property_domain(conn)
+ if scoped.property_id is not None:
+ clauses.append("canonical_domain = %s")
+ params.append(domain)
+ if not clauses and domain:
+ clauses.append("canonical_domain = %s")
+ params.append(domain)
+ where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+ params.append(limit)
+ cur = conn.execute(
+ f"""SELECT id, site_name, canonical_domain, generated_at
+ FROM report_payload {where}
+ ORDER BY generated_at DESC
+ LIMIT %s""",
+ tuple(params),
+ )
+ reports = []
+ for row in cur.fetchall() or []:
+ reports.append({
+ "report_id": _row_field(row, "id", index=0),
+ "site_name": _row_field(row, "site_name", index=1),
+ "canonical_domain": _row_field(row, "canonical_domain", index=2),
+ "generated_at": _iso(_row_field(row, "generated_at", index=3)),
+ })
+ return {"reports": reports, "count": len(reports)}
+
+
+def _iso(val: Any) -> str:
+ return val.isoformat() if hasattr(val, "isoformat") else str(val or "")
+
+
+def get_category_health_history(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ property_id = scoped.property_id
+ if property_id is None:
+ return {"error": "property_id is required"}
+ category_id = str(args.get("category_id") or "").strip()
+ limit = args.get("limit", 10)
+ try:
+ limit = int(limit)
+ except (TypeError, ValueError):
+ limit = 10
+ limit = max(1, min(limit, 30))
+
+ cur = conn.execute(
+ """SELECT category_scores, generated_at, report_id, health_score
+ FROM audit_health_snapshots
+ WHERE property_id = %s
+ ORDER BY generated_at DESC, id DESC
+ LIMIT %s""",
+ (property_id, limit),
+ )
+ points = []
+ for row in cur.fetchall() or []:
+ cat_scores = _row_field(row, "category_scores", index=0)
+ if isinstance(cat_scores, str):
+ try:
+ cat_scores = json.loads(cat_scores)
+ except json.JSONDecodeError:
+ cat_scores = {}
+ if not isinstance(cat_scores, dict):
+ cat_scores = {}
+ score = cat_scores.get(category_id) if category_id else None
+ gen = _row_field(row, "generated_at", index=1)
+ points.append({
+ "generated_at": gen.isoformat() if hasattr(gen, "isoformat") else str(gen or ""),
+ "report_id": _row_field(row, "report_id", index=2),
+ "health_score": _row_field(row, "health_score", index=3),
+ "category_score": score,
+ "category_scores": cat_scores if not category_id else None,
+ })
+ return {
+ "property_id": property_id,
+ "category_id": category_id or None,
+ "points": points,
+ "count": len(points),
+ }
diff --git a/src/website_profiling/tools/audit_tools/indexation_tools.py b/src/website_profiling/tools/audit_tools/indexation_tools.py
new file mode 100644
index 00000000..8661187d
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/indexation_tools.py
@@ -0,0 +1,72 @@
+"""Indexation coverage tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+_GAP_TYPES = frozenset({"sitemap_only", "crawled_not_in_sitemap", "gsc_not_crawled"})
+
+
+def get_indexation_coverage(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ cov = payload.get("indexation_coverage")
+ if not isinstance(cov, dict):
+ return {
+ "error": "indexation_coverage not in report — run audit with GSC connected",
+ "missing": True,
+ }
+ return {"indexation_coverage": cov}
+
+
+def list_indexation_gaps(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "urls": [], "total": 0, "truncated": False}
+ cov = payload.get("indexation_coverage")
+ if not isinstance(cov, dict):
+ return {"error": "indexation_coverage not in report", "missing": True, "urls": [], "total": 0, "truncated": False}
+ gap_type = str(args.get("gap_type") or "").strip()
+ if gap_type not in _GAP_TYPES:
+ return {
+ "error": f"gap_type must be one of: {', '.join(sorted(_GAP_TYPES))}",
+ "urls": [],
+ "total": 0,
+ "truncated": False,
+ }
+ lists = cov.get("lists") or {}
+ urls = lists.get(gap_type) if isinstance(lists, dict) else []
+ if not isinstance(urls, list):
+ urls = []
+ totals = cov.get("lists_total") or {}
+ total_all = totals.get(gap_type) if isinstance(totals, dict) else len(urls)
+ limit = parse_limit(args.get("limit"), 50, 200)
+ sliced = cap_list(urls, limit, max_cap=200)
+ return {
+ "gap_type": gap_type,
+ "urls": sliced["items"],
+ "total": int(total_all or sliced["total"]),
+ "truncated": sliced["truncated"] or (int(total_all or 0) > limit),
+ "counts": cov.get("counts"),
+ }
+
+
+def get_indexation_url_join(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ cov = payload.get("indexation_coverage")
+ if not isinstance(cov, dict):
+ return {"error": "indexation_coverage not in report", "missing": True}
+ url_join = cov.get("url_join")
+ if url_join is None:
+ return {"error": "url_join not in indexation_coverage", "missing": True}
+ return {"url_join": url_join, "counts": cov.get("counts")}
diff --git a/src/website_profiling/tools/audit_tools/international.py b/src/website_profiling/tools/audit_tools/international.py
new file mode 100644
index 00000000..86e7c330
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/international.py
@@ -0,0 +1,25 @@
+"""Hreflang and language tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from .context import AuditToolContext
+from ._slice import payload_dict_slice
+
+
+def get_hreflang_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "hreflang_summary")
+
+
+def get_language_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "language_summary")
diff --git a/src/website_profiling/tools/audit_tools/issues.py b/src/website_profiling/tools/audit_tools/issues.py
new file mode 100644
index 00000000..3c13f9a3
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/issues.py
@@ -0,0 +1,41 @@
+"""Category-scoped issue query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...reporting.terminology import category_display_name
+from .context import AuditToolContext
+from .report import _health_score, _iter_category_issues, list_issues
+
+
+def list_issues_by_category(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ category_id = str(args.get("category_id") or "").strip()
+ if not category_id:
+ return {"error": "category_id is required"}
+ return list_issues(conn, ctx, {**args, "category_id": category_id})
+
+
+def get_category_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ category_id = str(args.get("category_id") or "").strip()
+ if not category_id:
+ return {"error": "category_id is required"}
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ if str(cat.get("id") or "") != category_id:
+ continue
+ issues = _iter_category_issues({"categories": [cat]})
+ return {
+ "category_id": category_id,
+ "name": category_display_name(str(cat.get("name") or category_id)),
+ "score": cat.get("score"),
+ "issues": issues,
+ "issue_count": len(issues),
+ }
+ return {"error": f"category {category_id} not found", "category_id": category_id}
diff --git a/src/website_profiling/tools/audit_tools/keywords.py b/src/website_profiling/tools/audit_tools/keywords.py
new file mode 100644
index 00000000..b59f8fe1
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/keywords.py
@@ -0,0 +1,244 @@
+"""Keyword query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.keyword_store import read_keyword_history
+from ._slice import cap_list, parse_limit, payload_field
+from .context import AuditToolContext
+
+_KEYWORD_LIMIT_DEFAULT = 20
+_KEYWORD_LIMIT_MAX = 50
+
+
+def get_keyword_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required for keyword data"}
+
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "property_id": scoped.property_id}
+
+ rows = data.get("rows") or []
+ if not isinstance(rows, list):
+ rows = []
+
+ striking = data.get("striking_distance") or []
+ striking_count = len(striking) if isinstance(striking, list) else 0
+
+ top_n = args.get("limit", _KEYWORD_LIMIT_DEFAULT)
+ try:
+ top_n = int(top_n)
+ except (TypeError, ValueError):
+ top_n = _KEYWORD_LIMIT_DEFAULT
+ top_n = max(1, min(top_n, _KEYWORD_LIMIT_MAX))
+
+ top_rows = []
+ for row in rows[:top_n]:
+ if not isinstance(row, dict):
+ continue
+ top_rows.append({
+ "keyword": row.get("keyword"),
+ "score": row.get("score"),
+ "gsc_position": row.get("gsc_position"),
+ "gsc_clicks": row.get("gsc_clicks"),
+ "gsc_impressions": row.get("gsc_impressions"),
+ "recommended_action": row.get("recommended_action"),
+ })
+
+ return {
+ "fetched_at": data.get("fetched_at"),
+ "total_keywords": data.get("total_keywords") or len(rows),
+ "striking_distance_count": striking_count,
+ "top_keywords": top_rows,
+ "property_id": scoped.property_id,
+ }
+
+
+def search_keywords(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required for keyword data"}
+
+ query = str(args.get("query") or "").strip().lower()
+ if not query:
+ return {"error": "query is required"}
+
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "keywords": [], "total": 0}
+
+ rows = data.get("rows") or []
+ matches = []
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ kw = str(row.get("keyword") or "").lower()
+ if query in kw:
+ matches.append({
+ "keyword": row.get("keyword"),
+ "gsc_position": row.get("gsc_position"),
+ "gsc_clicks": row.get("gsc_clicks"),
+ "gsc_impressions": row.get("gsc_impressions"),
+ "recommended_action": row.get("recommended_action"),
+ })
+
+ limit = 30
+ total = len(matches)
+ return {
+ "keywords": matches[:limit],
+ "total": total,
+ "truncated": total > limit,
+ }
+
+
+def _keyword_list_tool(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ key: str,
+ item_key: str,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required for keyword data"}
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", item_key: [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ items = data.get(key) or []
+ if key == "semantic_keyword_clusters":
+ payload = scoped.load_payload(conn)
+ items = payload.get("semantic_keyword_clusters") or items
+ sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_striking_distance_keywords(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _keyword_list_tool(conn, ctx, args, "striking_distance", "keywords")
+
+
+def get_keyword_cannibalisation(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _keyword_list_tool(conn, ctx, args, "cannibalisation", "issues")
+
+
+def get_query_page_misalignment(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _keyword_list_tool(conn, ctx, args, "query_page_misalignment", "misalignments")
+
+
+def get_semantic_keyword_clusters(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "clusters": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 20, 50)
+ return payload_field(payload, "semantic_keyword_clusters", limit, max_cap=50, item_key="clusters")
+
+
+def get_keyword_history(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ keyword = str(args.get("keyword") or "").strip()
+ if not keyword:
+ return {"error": "keyword is required"}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ rows = read_keyword_history(conn, keyword, limit=limit, property_id=scoped.property_id)
+ return {"keyword": keyword, "history": rows, "count": len(rows)}
+
+
+def _filter_keyword_rows(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ predicate: Any,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required", "keywords": [], "total": 0, "truncated": False}
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "keywords": [], "total": 0, "truncated": False}
+ rows = data.get("rows") or []
+ matches = [r for r in rows if isinstance(r, dict) and predicate(r)]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(matches, limit, max_cap=50)
+ return {"keywords": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_keyword_serp_overlay(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "keywords": [], "total": 0, "truncated": False}
+ rows = data.get("rows") or []
+ with_serp = [
+ r for r in rows
+ if isinstance(r, dict) and r.get("serp_estimated_competition") is not None
+ ]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(with_serp, limit, max_cap=50)
+ return {
+ "serp_overlay_count": data.get("serp_overlay_count"),
+ "keywords": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def list_keywords_by_action(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ action = str(args.get("recommended_action") or "").strip().lower()
+ if not action:
+ return {"error": "recommended_action is required"}
+ return _filter_keyword_rows(
+ conn, ctx, args,
+ lambda r: str(r.get("recommended_action") or "").lower() == action,
+ )
+
+
+def list_keywords_by_position(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ min_pos = args.get("min_position")
+ max_pos = args.get("max_position")
+ try:
+ min_v = float(min_pos) if min_pos is not None else None
+ max_v = float(max_pos) if max_pos is not None else None
+ except (TypeError, ValueError):
+ return {"error": "min_position and max_position must be numbers"}
+
+ def _in_range(row: dict[str, Any]) -> bool:
+ pos = row.get("gsc_position")
+ try:
+ p = float(pos)
+ except (TypeError, ValueError):
+ return False
+ if min_v is not None and p < min_v:
+ return False
+ if max_v is not None and p > max_v:
+ return False
+ return True
+
+ return _filter_keyword_rows(conn, ctx, args, _in_range)
+
+
+def list_keywords_by_impressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ min_impr = args.get("min_impressions")
+ try:
+ min_v = int(min_impr) if min_impr is not None else 0
+ except (TypeError, ValueError):
+ return {"error": "min_impressions must be an integer"}
+ def _impressions(row: dict[str, Any]) -> int:
+ raw = row.get("gsc_impressions")
+ try:
+ return int(float(raw)) if raw is not None else 0
+ except (TypeError, ValueError):
+ return 0
+
+ return _filter_keyword_rows(
+ conn, ctx, args,
+ lambda r: _impressions(r) >= min_v,
+ )
diff --git a/src/website_profiling/tools/audit_tools/lighthouse.py b/src/website_profiling/tools/audit_tools/lighthouse.py
new file mode 100644
index 00000000..02d87e00
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/lighthouse.py
@@ -0,0 +1,139 @@
+"""Lighthouse query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...db.lighthouse_store import read_lighthouse_page_summaries, read_lighthouse_summary
+from ._slice import cap_list, parse_limit, payload_dict_slice
+from .context import AuditToolContext
+
+
+def get_lighthouse_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+
+ summary = payload.get("lighthouse_summary")
+ if not isinstance(summary, dict):
+ db_summary = read_lighthouse_summary(conn)
+ summary = db_summary if isinstance(db_summary, dict) else {}
+
+ human = payload.get("lighthouse_human_summary")
+ diagnostics = payload.get("lighthouse_diagnostics")
+ page_summaries = payload.get("lighthouse_by_url")
+ if not isinstance(page_summaries, dict):
+ page_summaries = read_lighthouse_page_summaries(conn) or {}
+
+ poor_pages = []
+ for url, data in list(page_summaries.items())[:20]:
+ if not isinstance(data, dict):
+ continue
+ perf = data.get("performance") or data.get("scores", {}).get("performance")
+ if perf is not None and float(perf) < 50:
+ poor_pages.append({"url": url, "performance": perf})
+
+ return {
+ "summary": summary,
+ "human_summary": human if isinstance(human, str) else None,
+ "diagnostics_count": len(diagnostics) if isinstance(diagnostics, list) else 0,
+ "pages_audited": len(page_summaries) if isinstance(page_summaries, dict) else 0,
+ "poor_performance_pages": poor_pages[:10],
+ }
+
+
+def get_lighthouse_for_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip().rstrip("/")
+ if not url:
+ return {"error": "url is required"}
+
+ payload = scoped.load_payload(conn)
+ by_url = payload.get("lighthouse_by_url") or {}
+ if not isinstance(by_url, dict):
+ by_url = read_lighthouse_page_summaries(conn) or {}
+
+ data = by_url.get(url) or by_url.get(url + "/")
+ if not data:
+ return {"error": "no lighthouse data for url", "url": url}
+ return {"url": url, "lighthouse": data}
+
+
+def get_lighthouse_diagnostics(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "diagnostics": [], "total": 0}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ diag = payload.get("lighthouse_diagnostics") or []
+ sliced = cap_list(diag if isinstance(diag, list) else [], limit, max_cap=50)
+ return {"diagnostics": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_crux_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ crux = payload.get("crux_summary")
+ if not crux:
+ return {"error": "crux_summary not in report — CrUX fetch may have failed or been skipped", "missing": True}
+ return payload_dict_slice(payload, "crux_summary")
+
+
+def list_slow_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ threshold = parse_limit(args.get("performance_threshold"), 50, 100)
+ by_url = payload.get("lighthouse_by_url") or {}
+ slow = []
+ if isinstance(by_url, dict):
+ for url, data in by_url.items():
+ if not isinstance(data, dict):
+ continue
+ perf = data.get("performance") or (data.get("scores") or {}).get("performance")
+ if perf is not None and float(perf) < threshold:
+ slow.append({"url": url, "performance": perf})
+ slow.sort(key=lambda x: float(x.get("performance") or 0))
+ sliced = cap_list(slow, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "threshold": threshold}
+
+
+def get_lighthouse_human_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ text = payload.get("lighthouse_human_summary")
+ if not text:
+ summary = payload.get("lighthouse_summary") or {}
+ if isinstance(summary, dict):
+ text = summary.get("human_summary_full") or summary.get("human_summary")
+ return {
+ "human_summary": str(text or ""),
+ "has_summary": bool(str(text or "").strip()),
+ }
+
+
+def list_lighthouse_poor_seo_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ threshold = parse_limit(args.get("seo_threshold"), 80, 100)
+ by_url = payload.get("lighthouse_by_url") or {}
+ poor = []
+ if isinstance(by_url, dict):
+ for url, data in by_url.items():
+ if not isinstance(data, dict):
+ continue
+ seo = data.get("seo") or (data.get("scores") or {}).get("seo")
+ if seo is not None and float(seo) < threshold:
+ poor.append({"url": url, "seo": seo})
+ poor.sort(key=lambda x: float(x.get("seo") or 0))
+ sliced = cap_list(poor, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "threshold": threshold}
diff --git a/src/website_profiling/tools/audit_tools/links.py b/src/website_profiling/tools/audit_tools/links.py
new file mode 100644
index 00000000..b6d305d0
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/links.py
@@ -0,0 +1,78 @@
+"""Internal linking and URL architecture tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit, payload_field
+from .context import AuditToolContext
+
+
+def list_orphan_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "orphans": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 50, 50)
+ orphans = payload.get("orphan_urls") or []
+ if not isinstance(orphans, list):
+ orphans = []
+ items = [{"url": str(u)} for u in orphans if u]
+ sliced = cap_list(items, limit)
+ return {"orphans": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_top_linked_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ top = payload.get("top_pages") or []
+ if not isinstance(top, list):
+ top = []
+ sliced = cap_list(top, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_outbound_link_domains(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "domains": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ domains = payload.get("outbound_link_domains") or []
+ sliced = cap_list(domains if isinstance(domains, list) else [], limit, max_cap=50)
+ return {"domains": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_link_graph_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ nodes = payload.get("graph_nodes") or []
+ edges = payload.get("graph_edges") or []
+ top_pages = payload.get("top_pages") or []
+ hubs = []
+ if isinstance(top_pages, list):
+ for p in top_pages[:10]:
+ if isinstance(p, dict):
+ hubs.append({"url": p.get("url"), "inlinks": p.get("inlinks")})
+ return {
+ "node_count": len(nodes) if isinstance(nodes, list) else 0,
+ "edge_count": len(edges) if isinstance(edges, list) else 0,
+ "top_hubs": hubs,
+ }
+
+
+def get_url_fingerprints(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "fingerprints": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ fps = payload.get("url_fingerprints") or []
+ sliced = cap_list(fps if isinstance(fps, list) else [], limit, max_cap=50)
+ return {"fingerprints": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/onpage.py b/src/website_profiling/tools/audit_tools/onpage.py
new file mode 100644
index 00000000..7b6279cf
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/onpage.py
@@ -0,0 +1,121 @@
+"""On-page SEO issue tools from content_urls and crawl data."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+_CONTENT_BUCKETS = frozenset({
+ "missing_h1",
+ "missing_title",
+ "multiple_h1",
+ "missing_meta_desc",
+ "meta_desc_short",
+ "meta_desc_long",
+ "thin_content",
+})
+
+
+def _content_urls_bucket(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ bucket: str,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "items": [], "total": 0, "truncated": False}
+ content_urls = payload.get("content_urls") or {}
+ if not isinstance(content_urls, dict):
+ return {"error": "content_urls not in report", "missing": True, "items": [], "total": 0, "truncated": False}
+ items = content_urls.get(bucket) or []
+ if not isinstance(items, list):
+ items = []
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(items, limit, max_cap=50)
+ return {
+ "bucket": bucket,
+ "items": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def list_content_url_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ bucket = str(args.get("bucket") or "").strip()
+ if bucket not in _CONTENT_BUCKETS:
+ return {
+ "error": f"bucket must be one of: {', '.join(sorted(_CONTENT_BUCKETS))}",
+ "items": [],
+ "total": 0,
+ "truncated": False,
+ }
+ return _content_urls_bucket(conn, ctx, args, bucket)
+
+
+def list_pages_missing_title(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "missing_title")
+
+
+def list_pages_missing_h1(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "missing_h1")
+
+
+def list_pages_multiple_h1(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "multiple_h1")
+
+
+def list_pages_missing_meta_description(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "missing_meta_desc")
+
+
+def list_pages_meta_desc_too_short(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "meta_desc_short")
+
+
+def list_pages_meta_desc_too_long(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _content_urls_bucket(conn, ctx, args, "meta_desc_long")
+
+
+def list_seo_onpage_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "issues": [], "total": 0, "truncated": False}
+ issues = payload.get("issues") or {}
+ seo = issues.get("seo") if isinstance(issues, dict) else []
+ if not isinstance(seo, list):
+ seo = []
+ issue_type = str(args.get("issue_type") or "").strip().lower()
+ if issue_type:
+ seo = [x for x in seo if isinstance(x, dict) and str(x.get("type") or "").lower() == issue_type]
+ limit = parse_limit(args.get("limit"), 50, 50)
+ sliced = cap_list(seo, limit, max_cap=50)
+ return {"issues": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_pages_noindex(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ if "noindex" not in df.columns:
+ return {"pages": [], "total": 0, "truncated": False, "note": "noindex column not in crawl"}
+ records = df.to_dict(orient="records")
+ pages = []
+ for r in records:
+ val = str(r.get("noindex") or "").lower()
+ if val not in ("true", "1", "yes"):
+ continue
+ pages.append({
+ "url": str(r.get("url") or ""),
+ "status": str(r.get("status") or ""),
+ "title": str(r.get("title") or ""),
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/ops.py b/src/website_profiling/tools/audit_tools/ops.py
new file mode 100644
index 00000000..e3c4afe8
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/ops.py
@@ -0,0 +1,155 @@
+"""Integration alerts and ops tools."""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from psycopg import Connection
+
+from ...db._common import _row_field
+from ...db.property_store import get_property_by_id
+from ...tools.alert_checker import check_all_alerts
+from ._slice import parse_limit
+from .context import AuditToolContext
+
+
+def get_integration_alerts(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ alerts = check_all_alerts(int(scoped.property_id))
+ return {"alerts": alerts, "count": len(alerts), "property_id": scoped.property_id}
+
+
+def get_property_ops(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ cur = conn.execute(
+ """SELECT schedule_cron, alert_webhook_url, alert_email
+ FROM properties WHERE id = %s""",
+ (int(scoped.property_id),),
+ )
+ row = cur.fetchone()
+ if not row:
+ return {"error": "property not found"}
+ return {
+ "property_id": scoped.property_id,
+ "schedule_cron": _row_field(row, "schedule_cron", index=0),
+ "alert_webhook_url": _row_field(row, "alert_webhook_url", index=1),
+ "alert_email": _row_field(row, "alert_email", index=2),
+ "has_schedule": bool(str(_row_field(row, "schedule_cron", index=0) or "").strip()),
+ "has_alert_webhook": bool(str(_row_field(row, "alert_webhook_url", index=1) or "").strip()),
+ }
+
+
+def get_google_integration_status(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ prop = get_property_by_id(conn, int(scoped.property_id))
+ if not prop:
+ return {"error": "property not found"}
+ google = scoped.load_google(conn)
+ gsc_links_status = None
+ try:
+ from ...integrations.google.gsc_links_store import read_gsc_links_status
+ gsc_links_status = read_gsc_links_status(conn, int(scoped.property_id))
+ except Exception:
+ gsc_links_status = None
+ return {
+ "property_id": scoped.property_id,
+ "google_connected": bool(prop.get("google_refresh_token")),
+ "google_connected_at": prop.get("google_connected_at"),
+ "google_connected_email": prop.get("google_connected_email"),
+ "gsc_site_url": prop.get("gsc_site_url"),
+ "ga4_property_id": prop.get("ga4_property_id"),
+ "google_date_range_days": prop.get("google_date_range_days"),
+ "google_data_fetched_at": (google or {}).get("fetched_at"),
+ "gsc_links": gsc_links_status,
+ }
+
+
+def list_crawl_runs(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ limit = parse_limit(args.get("limit"), 20, 50)
+ params: list[Any] = []
+ where = ""
+ if scoped.property_id is not None:
+ where = "WHERE property_id = %s"
+ params.append(int(scoped.property_id))
+ params.append(limit)
+ cur = conn.execute(
+ f"""SELECT id, created_at, start_url, render_mode, property_id
+ FROM crawl_runs {where}
+ ORDER BY id DESC
+ LIMIT %s""",
+ tuple(params),
+ )
+ runs = []
+ for row in cur.fetchall() or []:
+ created = _row_field(row, "created_at", index=1)
+ runs.append({
+ "id": _row_field(row, "id", index=0),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "start_url": _row_field(row, "start_url", index=2),
+ "render_mode": _row_field(row, "render_mode", index=3),
+ "property_id": _row_field(row, "property_id", index=4),
+ })
+ return {"runs": runs, "count": len(runs)}
+
+
+def list_log_uploads(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ limit = parse_limit(args.get("limit"), 10, 30)
+ cur = conn.execute(
+ """SELECT id, filename, line_count, uploaded_at
+ FROM log_file_uploads
+ WHERE property_id = %s
+ ORDER BY uploaded_at DESC
+ LIMIT %s""",
+ (int(scoped.property_id), limit),
+ )
+ uploads = []
+ for row in cur.fetchall() or []:
+ uploaded = _row_field(row, "uploaded_at", index=3)
+ uploads.append({
+ "id": _row_field(row, "id", index=0),
+ "filename": _row_field(row, "filename", index=1),
+ "line_count": _row_field(row, "line_count", index=2),
+ "uploaded_at": uploaded.isoformat() if hasattr(uploaded, "isoformat") else str(uploaded or ""),
+ })
+ return {"uploads": uploads, "count": len(uploads), "property_id": scoped.property_id}
+
+
+def get_latest_log_analysis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ cur = conn.execute(
+ """SELECT filename, line_count, analysis, uploaded_at
+ FROM log_file_uploads
+ WHERE property_id = %s
+ ORDER BY uploaded_at DESC
+ LIMIT 1""",
+ (int(scoped.property_id),),
+ )
+ row = cur.fetchone()
+ if not row:
+ return {"error": "no log uploads found", "missing": True}
+ analysis = _row_field(row, "analysis", index=2)
+ if isinstance(analysis, str):
+ try:
+ analysis = json.loads(analysis)
+ except json.JSONDecodeError:
+ analysis = {}
+ uploaded = _row_field(row, "uploaded_at", index=3)
+ return {
+ "filename": _row_field(row, "filename", index=0),
+ "line_count": _row_field(row, "line_count", index=1),
+ "analysis": analysis if isinstance(analysis, dict) else {},
+ "uploaded_at": uploaded.isoformat() if hasattr(uploaded, "isoformat") else str(uploaded or ""),
+ "property_id": scoped.property_id,
+ }
diff --git a/src/website_profiling/tools/audit_tools/properties.py b/src/website_profiling/tools/audit_tools/properties.py
new file mode 100644
index 00000000..9a3a126e
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/properties.py
@@ -0,0 +1,45 @@
+"""Property query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...db.property_store import get_property_by_id, list_properties_public
+from .context import AuditToolContext
+
+
+def _public_property_row(prop: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "id": prop.get("id"),
+ "name": prop.get("name"),
+ "canonical_domain": prop.get("canonical_domain"),
+ "site_url": prop.get("site_url"),
+ "gsc_site_url": prop.get("gsc_site_url"),
+ "ga4_property_id": prop.get("ga4_property_id"),
+ "google_auth_mode": prop.get("google_auth_mode"),
+ "google_connected": prop.get("google_connected") or bool(prop.get("google_connected_at")),
+ "google_connected_at": prop.get("google_connected_at"),
+ "google_connected_email": prop.get("google_connected_email"),
+ "google_date_range_days": prop.get("google_date_range_days"),
+ "crawl_authorized_at": prop.get("crawl_authorized_at"),
+ }
+
+
+def list_properties(conn: Connection, _ctx: AuditToolContext, _args: dict[str, Any]) -> dict[str, Any]:
+ rows = list_properties_public(conn)
+ return {"properties": rows, "count": len(rows)}
+
+
+def get_property(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ property_id = args.get("property_id") or ctx.property_id
+ if property_id is None:
+ return {"error": "property_id is required"}
+ try:
+ pid = int(property_id)
+ except (TypeError, ValueError):
+ return {"error": "invalid property_id"}
+ prop = get_property_by_id(conn, pid)
+ if not prop:
+ return {"error": f"property {pid} not found"}
+ return {"property": _public_property_row(prop)}
diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py
new file mode 100644
index 00000000..b482964f
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/registry.py
@@ -0,0 +1,320 @@
+"""Tool registry and dispatch for MCP and chat agent."""
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from psycopg import Connection
+
+from ...db.storage import db_session
+from .backlinks import (
+ get_backlinks_velocity,
+ get_bing_backlinks_summary,
+ get_competitor_link_gap,
+ get_gsc_latest_links,
+ get_gsc_links_import_status,
+ get_gsc_links_summary,
+ get_gsc_sample_links,
+ get_third_party_links_overlay,
+)
+from .charts import (
+ get_crawl_summary,
+ get_domain_link_distribution,
+ get_issue_priority_breakdown,
+ get_mime_type_breakdown,
+ get_outlink_distribution,
+ get_title_length_distribution,
+ get_top_crawled_pages,
+)
+from .compare import compare_reports
+from .compare_slices import (
+ compare_category_deltas,
+ compare_issue_deltas,
+ compare_lighthouse_deltas,
+ compare_link_metric_deltas,
+ compare_redirect_deltas,
+ compare_seo_health_deltas,
+ compare_url_set_diff,
+)
+from .content import (
+ get_content_analytics,
+ get_content_duplicates,
+ get_keyword_opportunities,
+ get_ner_site_summary,
+ get_social_coverage,
+ list_thin_content_pages,
+)
+from .context import AuditToolContext
+from .crawl import (
+ get_browser_diagnostics_summary,
+ get_crawl_links_table,
+ get_crawl_segments,
+ get_depth_distribution,
+ get_graph_edges_sample,
+ get_internal_links,
+ get_page_analysis,
+ get_page_details,
+ get_response_time_stats,
+ get_seo_health,
+ get_status_code_breakdown,
+ list_broken_links,
+ list_pages_by_fetch_method,
+ list_pages_with_console_errors,
+ list_redirects,
+ list_status_4xx_pages,
+ list_status_5xx_pages,
+ search_pages,
+ search_pages_advanced,
+)
+from .google import (
+ get_ga4_page_metrics,
+ get_ga4_summary,
+ get_google_summary,
+ get_gsc_page_query_slice,
+ get_gsc_top_pages,
+ get_gsc_top_queries,
+)
+from .health import get_category_health_history, get_health_history, list_report_history
+from .indexation_tools import get_indexation_coverage, get_indexation_url_join, list_indexation_gaps
+from .international import get_hreflang_summary, get_language_summary
+from .issues import get_category_issues, list_issues_by_category
+from .keywords import (
+ get_keyword_cannibalisation,
+ get_keyword_history,
+ get_keyword_serp_overlay,
+ get_keyword_summary,
+ get_query_page_misalignment,
+ get_semantic_keyword_clusters,
+ get_striking_distance_keywords,
+ list_keywords_by_action,
+ list_keywords_by_impressions,
+ list_keywords_by_position,
+ search_keywords,
+)
+from .lighthouse import (
+ get_crux_summary,
+ get_lighthouse_diagnostics,
+ get_lighthouse_for_url,
+ get_lighthouse_human_summary,
+ get_lighthouse_summary,
+ list_lighthouse_poor_seo_pages,
+ list_slow_pages,
+)
+from .onpage import (
+ list_content_url_issues,
+ list_pages_meta_desc_too_long,
+ list_pages_meta_desc_too_short,
+ list_pages_missing_h1,
+ list_pages_missing_meta_description,
+ list_pages_missing_title,
+ list_pages_multiple_h1,
+ list_pages_noindex,
+ list_seo_onpage_issues,
+)
+from .links import (
+ get_link_graph_summary,
+ get_outbound_link_domains,
+ get_top_linked_pages,
+ get_url_fingerprints,
+ list_orphan_pages,
+)
+from .ops import (
+ get_google_integration_status,
+ get_integration_alerts,
+ get_latest_log_analysis,
+ get_property_ops,
+ list_crawl_runs,
+ list_log_uploads,
+)
+from .properties import get_property, list_properties
+from .report import (
+ get_category_scores,
+ get_critical_issues,
+ get_executive_summary,
+ get_report_meta,
+ get_report_summary,
+ get_site_level,
+ list_issues,
+)
+from .report_extras import (
+ get_audit_recommendations,
+ get_category_recommendations,
+ get_ml_errors,
+ get_ssl_expiry_info,
+ list_audit_categories,
+ list_issues_with_ai_fixes,
+)
+from .schema import get_schema_coverage, list_pages_without_schema, search_pages_by_schema_type
+from .security import get_security_findings
+from .tech import get_tech_stack_summary
+from .tool_catalog import TOOL_DEFINITIONS
+from .workflow import list_issue_workflow
+
+ToolHandler = Callable[[Connection, AuditToolContext, dict[str, Any]], dict[str, Any]]
+
+_TOOL_HANDLERS: dict[str, ToolHandler] = {
+ "list_properties": list_properties,
+ "get_property": get_property,
+ "get_report_summary": get_report_summary,
+ "get_category_scores": get_category_scores,
+ "get_executive_summary": get_executive_summary,
+ "get_report_meta": get_report_meta,
+ "get_site_level": get_site_level,
+ "list_report_history": list_report_history,
+ "list_issues": list_issues,
+ "get_critical_issues": get_critical_issues,
+ "list_issues_by_category": list_issues_by_category,
+ "get_category_issues": get_category_issues,
+ "list_issue_workflow": list_issue_workflow,
+ "search_pages": search_pages,
+ "get_page_details": get_page_details,
+ "get_internal_links": get_internal_links,
+ "list_redirects": list_redirects,
+ "list_broken_links": list_broken_links,
+ "get_status_code_breakdown": get_status_code_breakdown,
+ "get_response_time_stats": get_response_time_stats,
+ "get_depth_distribution": get_depth_distribution,
+ "get_crawl_segments": get_crawl_segments,
+ "get_browser_diagnostics_summary": get_browser_diagnostics_summary,
+ "get_schema_coverage": get_schema_coverage,
+ "list_pages_without_schema": list_pages_without_schema,
+ "search_pages_by_schema_type": search_pages_by_schema_type,
+ "get_seo_health": get_seo_health,
+ "list_orphan_pages": list_orphan_pages,
+ "get_top_linked_pages": get_top_linked_pages,
+ "get_outbound_link_domains": get_outbound_link_domains,
+ "get_link_graph_summary": get_link_graph_summary,
+ "get_url_fingerprints": get_url_fingerprints,
+ "get_indexation_coverage": get_indexation_coverage,
+ "get_hreflang_summary": get_hreflang_summary,
+ "get_language_summary": get_language_summary,
+ "get_content_analytics": get_content_analytics,
+ "get_content_duplicates": get_content_duplicates,
+ "get_social_coverage": get_social_coverage,
+ "get_keyword_opportunities": get_keyword_opportunities,
+ "get_ner_site_summary": get_ner_site_summary,
+ "list_thin_content_pages": list_thin_content_pages,
+ "get_keyword_summary": get_keyword_summary,
+ "search_keywords": search_keywords,
+ "get_striking_distance_keywords": get_striking_distance_keywords,
+ "get_keyword_cannibalisation": get_keyword_cannibalisation,
+ "get_query_page_misalignment": get_query_page_misalignment,
+ "get_semantic_keyword_clusters": get_semantic_keyword_clusters,
+ "get_keyword_history": get_keyword_history,
+ "get_google_summary": get_google_summary,
+ "get_gsc_top_queries": get_gsc_top_queries,
+ "get_gsc_top_pages": get_gsc_top_pages,
+ "get_ga4_summary": get_ga4_summary,
+ "get_gsc_page_query_slice": get_gsc_page_query_slice,
+ "get_gsc_links_summary": get_gsc_links_summary,
+ "get_gsc_links_import_status": get_gsc_links_import_status,
+ "get_competitor_link_gap": get_competitor_link_gap,
+ "get_bing_backlinks_summary": get_bing_backlinks_summary,
+ "get_lighthouse_summary": get_lighthouse_summary,
+ "get_lighthouse_for_url": get_lighthouse_for_url,
+ "get_lighthouse_diagnostics": get_lighthouse_diagnostics,
+ "get_crux_summary": get_crux_summary,
+ "list_slow_pages": list_slow_pages,
+ "get_health_history": get_health_history,
+ "compare_reports": compare_reports,
+ "get_integration_alerts": get_integration_alerts,
+ "get_tech_stack_summary": get_tech_stack_summary,
+ "get_security_findings": get_security_findings,
+ "get_audit_recommendations": get_audit_recommendations,
+ "get_ml_errors": get_ml_errors,
+ "get_ssl_expiry_info": get_ssl_expiry_info,
+ "list_audit_categories": list_audit_categories,
+ "get_category_recommendations": get_category_recommendations,
+ "list_issues_with_ai_fixes": list_issues_with_ai_fixes,
+ "list_seo_onpage_issues": list_seo_onpage_issues,
+ "list_content_url_issues": list_content_url_issues,
+ "list_pages_missing_title": list_pages_missing_title,
+ "list_pages_missing_h1": list_pages_missing_h1,
+ "list_pages_multiple_h1": list_pages_multiple_h1,
+ "list_pages_missing_meta_description": list_pages_missing_meta_description,
+ "list_pages_meta_desc_too_short": list_pages_meta_desc_too_short,
+ "list_pages_meta_desc_too_long": list_pages_meta_desc_too_long,
+ "list_pages_noindex": list_pages_noindex,
+ "get_crawl_summary": get_crawl_summary,
+ "get_issue_priority_breakdown": get_issue_priority_breakdown,
+ "get_mime_type_breakdown": get_mime_type_breakdown,
+ "get_title_length_distribution": get_title_length_distribution,
+ "get_domain_link_distribution": get_domain_link_distribution,
+ "get_outlink_distribution": get_outlink_distribution,
+ "get_top_crawled_pages": get_top_crawled_pages,
+ "list_indexation_gaps": list_indexation_gaps,
+ "get_indexation_url_join": get_indexation_url_join,
+ "get_gsc_sample_links": get_gsc_sample_links,
+ "get_gsc_latest_links": get_gsc_latest_links,
+ "get_third_party_links_overlay": get_third_party_links_overlay,
+ "get_backlinks_velocity": get_backlinks_velocity,
+ "get_property_ops": get_property_ops,
+ "get_google_integration_status": get_google_integration_status,
+ "list_crawl_runs": list_crawl_runs,
+ "list_log_uploads": list_log_uploads,
+ "get_latest_log_analysis": get_latest_log_analysis,
+ "get_keyword_serp_overlay": get_keyword_serp_overlay,
+ "list_keywords_by_action": list_keywords_by_action,
+ "list_keywords_by_position": list_keywords_by_position,
+ "list_keywords_by_impressions": list_keywords_by_impressions,
+ "get_lighthouse_human_summary": get_lighthouse_human_summary,
+ "list_lighthouse_poor_seo_pages": list_lighthouse_poor_seo_pages,
+ "get_page_analysis": get_page_analysis,
+ "search_pages_advanced": search_pages_advanced,
+ "list_pages_with_console_errors": list_pages_with_console_errors,
+ "list_pages_by_fetch_method": list_pages_by_fetch_method,
+ "get_crawl_links_table": get_crawl_links_table,
+ "get_graph_edges_sample": get_graph_edges_sample,
+ "list_status_4xx_pages": list_status_4xx_pages,
+ "list_status_5xx_pages": list_status_5xx_pages,
+ "get_ga4_page_metrics": get_ga4_page_metrics,
+ "get_category_health_history": get_category_health_history,
+ "compare_issue_deltas": compare_issue_deltas,
+ "compare_category_deltas": compare_category_deltas,
+ "compare_seo_health_deltas": compare_seo_health_deltas,
+ "compare_lighthouse_deltas": compare_lighthouse_deltas,
+ "compare_url_set_diff": compare_url_set_diff,
+ "compare_redirect_deltas": compare_redirect_deltas,
+ "compare_link_metric_deltas": compare_link_metric_deltas,
+}
+
+
+def dispatch_tool(
+ name: str,
+ args: dict[str, Any] | None,
+ *,
+ context: AuditToolContext | None = None,
+ conn: Connection | None = None,
+) -> dict[str, Any]:
+ """Run a tool by name. Uses db_session when conn is not provided."""
+ handler = _TOOL_HANDLERS.get(name)
+ if handler is None:
+ return {"error": f"unknown tool: {name}"}
+
+ ctx = context or AuditToolContext()
+ payload_args = dict(args or {})
+ merged_ctx = ctx.with_args(payload_args)
+
+ if conn is not None:
+ return handler(conn, merged_ctx, payload_args)
+
+ with db_session() as session:
+ return handler(session, merged_ctx, payload_args)
+
+
+def openai_tools_schema() -> list[dict[str, Any]]:
+ """Convert TOOL_DEFINITIONS to OpenAI function-calling format."""
+ out: list[dict[str, Any]] = []
+ for tool in TOOL_DEFINITIONS:
+ out.append({
+ "type": "function",
+ "function": {
+ "name": tool["name"],
+ "description": tool.get("description", ""),
+ "parameters": tool.get("inputSchema", {"type": "object", "properties": {}}),
+ },
+ })
+ return out
+
+
+def tool_handler_names() -> set[str]:
+ return set(_TOOL_HANDLERS.keys())
diff --git a/src/website_profiling/tools/audit_tools/report.py b/src/website_profiling/tools/audit_tools/report.py
new file mode 100644
index 00000000..15d1f000
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/report.py
@@ -0,0 +1,190 @@
+"""Report summary and issue query tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...reporting.terminology import category_display_name
+from .context import AuditToolContext
+
+_PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}
+_ISSUE_LIMIT_DEFAULT = 20
+_ISSUE_LIMIT_MAX = 50
+
+
+def _normalize_priority(p: str) -> str:
+ raw = (p or "").strip()
+ if not raw:
+ return ""
+ cap = raw[0].upper() + raw[1:].lower()
+ if cap in _PRIORITY_ORDER:
+ return cap
+ return raw
+
+
+def _iter_category_issues(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ cat_id = str(cat.get("id") or "")
+ cat_name = category_display_name(str(cat.get("name") or cat_id))
+ for issue in cat.get("issues") or []:
+ if not isinstance(issue, dict):
+ continue
+ rec = str(issue.get("llm_recommendation") or issue.get("recommendation") or "")
+ rows.append({
+ "category_id": cat_id,
+ "category": cat_name,
+ "priority": str(issue.get("priority") or "Medium"),
+ "message": str(issue.get("message") or ""),
+ "url": str(issue.get("url") or ""),
+ "recommendation": rec,
+ })
+ rows.sort(key=lambda x: _PRIORITY_ORDER.get(x.get("priority", "Low"), 99))
+ return rows
+
+
+def _health_score(payload: dict[str, Any]) -> int | None:
+ scores = [
+ float(c.get("score"))
+ for c in (payload.get("categories") or [])
+ if isinstance(c, dict) and isinstance(c.get("score"), (int, float))
+ ]
+ return round(sum(scores) / len(scores)) if scores else None
+
+
+def _issue_counts(issues: list[dict[str, Any]]) -> dict[str, int]:
+ counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}
+ for issue in issues:
+ p = str(issue.get("priority") or "Medium")
+ counts[p] = counts.get(p, 0) + 1
+ return counts
+
+
+def get_report_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ all_issues = _iter_category_issues(payload)
+ summary = payload.get("summary") or {}
+ categories = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ categories.append({
+ "id": cat.get("id"),
+ "name": category_display_name(str(cat.get("name") or "")),
+ "score": cat.get("score"),
+ "issue_count": len(cat.get("issues") or []),
+ })
+ return {
+ "site_name": payload.get("site_name"),
+ "report_generated_at": payload.get("report_generated_at"),
+ "health_score": _health_score(payload),
+ "issue_counts": _issue_counts(all_issues),
+ "total_issues": len(all_issues),
+ "crawl_summary": {
+ "total_urls": summary.get("total_urls"),
+ "count_2xx": summary.get("count_2xx"),
+ "count_3xx": summary.get("count_3xx"),
+ "count_4xx": summary.get("count_4xx"),
+ "count_5xx": summary.get("count_5xx"),
+ "success_rate": summary.get("success_rate"),
+ },
+ "categories": categories,
+ "property_id": scoped.property_id,
+ "report_id": scoped.report_id,
+ }
+
+
+def list_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "issues": [], "total": 0, "truncated": False}
+
+ limit = args.get("limit", _ISSUE_LIMIT_DEFAULT)
+ try:
+ limit = int(limit)
+ except (TypeError, ValueError):
+ limit = _ISSUE_LIMIT_DEFAULT
+ limit = max(1, min(limit, _ISSUE_LIMIT_MAX))
+
+ priority_filter = _normalize_priority(str(args.get("priority") or ""))
+ category_id = str(args.get("category_id") or "").strip()
+ url_contains = str(args.get("url_contains") or "").strip().lower()
+
+ issues = _iter_category_issues(payload)
+ if priority_filter:
+ issues = [i for i in issues if i.get("priority") == priority_filter]
+ if category_id:
+ issues = [i for i in issues if i.get("category_id") == category_id]
+ if url_contains:
+ issues = [i for i in issues if url_contains in str(i.get("url") or "").lower()]
+
+ total = len(issues)
+ truncated = total > limit
+ return {
+ "issues": issues[:limit],
+ "total": total,
+ "truncated": truncated,
+ }
+
+
+def get_critical_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """All Critical-priority audit issues (chat table visualization)."""
+ return list_issues(conn, ctx, {**args, "priority": "Critical"})
+
+
+def get_category_scores(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "categories": []}
+ categories = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ categories.append({
+ "id": cat.get("id"),
+ "name": category_display_name(str(cat.get("name") or "")),
+ "score": cat.get("score"),
+ "issue_count": len(cat.get("issues") or []),
+ })
+ return {"categories": categories, "health_score": _health_score(payload)}
+
+
+def get_executive_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ summary = payload.get("executive_summary")
+ if not summary:
+ return {"error": "executive_summary not generated — enable AI in audit settings", "missing": True}
+ return {"executive_summary": summary}
+
+
+def get_report_meta(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ meta = payload.get("report_meta")
+ if not isinstance(meta, dict):
+ return {"error": "report_meta not in payload", "missing": True}
+ return {"report_meta": meta}
+
+
+def get_site_level(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ site_level = payload.get("site_level")
+ if not isinstance(site_level, dict):
+ return {"error": "site_level not in payload", "missing": True}
+ return {"site_level": site_level}
diff --git a/src/website_profiling/tools/audit_tools/report_extras.py b/src/website_profiling/tools/audit_tools/report_extras.py
new file mode 100644
index 00000000..2906d150
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/report_extras.py
@@ -0,0 +1,115 @@
+"""Additional report payload slices not covered by core report tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+
+def get_audit_recommendations(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "recommendations": []}
+ recs = payload.get("recommendations") or []
+ if not isinstance(recs, list):
+ recs = []
+ return {"recommendations": recs, "count": len(recs)}
+
+
+def get_ml_errors(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "errors": []}
+ errors = payload.get("ml_errors") or []
+ if not isinstance(errors, list):
+ errors = []
+ return {"errors": errors, "count": len(errors)}
+
+
+def get_ssl_expiry_info(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return {
+ "site_ssl_expires_at": payload.get("site_ssl_expires_at"),
+ "site_name": payload.get("site_name"),
+ }
+
+
+def list_audit_categories(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "categories": []}
+ cats = payload.get("categories") or []
+ out = []
+ for cat in cats:
+ if not isinstance(cat, dict):
+ continue
+ issues = cat.get("issues") or []
+ recs = cat.get("recommendations") or []
+ out.append({
+ "id": cat.get("id"),
+ "name": cat.get("name"),
+ "score": cat.get("score"),
+ "issue_count": len(issues) if isinstance(issues, list) else 0,
+ "recommendation_count": len(recs) if isinstance(recs, list) else 0,
+ })
+ return {"categories": out, "count": len(out)}
+
+
+def get_category_recommendations(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ category_id = str(args.get("category_id") or "").strip()
+ if not category_id:
+ return {"error": "category_id is required"}
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ cid = str(cat.get("id") or "").strip()
+ if cid == category_id:
+ recs = cat.get("recommendations") or []
+ return {
+ "category_id": category_id,
+ "category_name": cat.get("name"),
+ "recommendations": recs if isinstance(recs, list) else [],
+ }
+ return {"error": f"category {category_id} not found", "recommendations": []}
+
+
+def list_issues_with_ai_fixes(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "issues": [], "total": 0, "truncated": False}
+ matches: list[dict[str, Any]] = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ cat_name = str(cat.get("name") or cat.get("id") or "")
+ for iss in cat.get("issues") or []:
+ if not isinstance(iss, dict):
+ continue
+ llm = str(iss.get("llm_recommendation") or "").strip()
+ if not llm:
+ continue
+ matches.append({
+ "category": cat_name,
+ "priority": iss.get("priority"),
+ "url": iss.get("url"),
+ "message": iss.get("message"),
+ "recommendation": iss.get("recommendation"),
+ "llm_recommendation": llm,
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(matches, limit, max_cap=50)
+ return {"issues": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/schema.py b/src/website_profiling/tools/audit_tools/schema.py
new file mode 100644
index 00000000..1582e171
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/schema.py
@@ -0,0 +1,53 @@
+"""Schema markup audit tools (read-only, from crawl data)."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import crawl_filter, parse_limit
+from .context import AuditToolContext
+
+
+def get_schema_coverage(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"error": "no crawl data", "with_schema": 0, "without_schema": 0, "total": 0}
+ total = len(df)
+ with_schema = 0
+ type_counts: dict[str, int] = {}
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ has = str(rec.get("has_schema") or "").lower() in ("true", "1", "yes")
+ if has:
+ with_schema += 1
+ from ._slice import _row_schema_types_list # noqa: PLC0415
+
+ for t in _row_schema_types_list(rec):
+ type_counts[t] = type_counts.get(t, 0) + 1
+ top_types = sorted(type_counts.items(), key=lambda x: -x[1])[:20]
+ return {
+ "total_pages": total,
+ "with_schema": with_schema,
+ "without_schema": total - with_schema,
+ "coverage_pct": round(100 * with_schema / total, 1) if total else 0,
+ "top_schema_types": [{"type": k, "count": v} for k, v in top_types],
+ }
+
+
+def list_pages_without_schema(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ limit = parse_limit(args.get("limit"), 30, 30)
+ return crawl_filter(df, has_schema=False, limit=limit)
+
+
+def search_pages_by_schema_type(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ schema_type = str(args.get("schema_type") or "").strip()
+ if not schema_type:
+ return {"error": "schema_type is required"}
+ df = scoped.load_crawl_df(conn)
+ limit = parse_limit(args.get("limit"), 30, 30)
+ return crawl_filter(df, schema_type=schema_type, limit=limit)
diff --git a/src/website_profiling/tools/audit_tools/security.py b/src/website_profiling/tools/audit_tools/security.py
new file mode 100644
index 00000000..0730390f
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/security.py
@@ -0,0 +1,28 @@
+"""Security findings tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+
+def get_security_findings(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "findings": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 50, 50)
+ severity = str(args.get("severity") or "").strip().lower()
+ findings = payload.get("security_findings") or []
+ if not isinstance(findings, list):
+ findings = []
+ if severity:
+ findings = [
+ f for f in findings
+ if isinstance(f, dict) and str(f.get("severity") or "").lower() == severity
+ ]
+ sliced = cap_list(findings, limit, max_cap=50)
+ return {"findings": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/tech.py b/src/website_profiling/tools/audit_tools/tech.py
new file mode 100644
index 00000000..9dcc8b7c
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/tech.py
@@ -0,0 +1,17 @@
+"""Technology stack tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from .context import AuditToolContext
+from ._slice import payload_dict_slice
+
+
+def get_tech_stack_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found"}
+ return payload_dict_slice(payload, "tech_stack_summary")
diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py
new file mode 100644
index 00000000..452964ec
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/tool_catalog.py
@@ -0,0 +1,183 @@
+"""TOOL_DEFINITIONS catalog for MCP and chat agent."""
+from __future__ import annotations
+
+from typing import Any
+
+_PID = {"type": "integer", "description": "Property ID"}
+_RID = {"type": "integer", "description": "Report ID (defaults to latest)"}
+_LIMIT = {"type": "integer", "minimum": 1, "maximum": 50}
+_URL = {"type": "string", "description": "Page URL"}
+
+
+def _tool(name: str, description: str, properties: dict[str, Any], required: list[str] | None = None) -> dict[str, Any]:
+ return {
+ "name": name,
+ "description": description,
+ "inputSchema": {
+ "type": "object",
+ "properties": properties,
+ "required": required or [],
+ },
+ }
+
+
+TOOL_DEFINITIONS: list[dict[str, Any]] = [
+ _tool("list_properties", "List all configured site properties (domains) in Site Audit.", {}),
+ _tool("get_property", "Get details for one property by property_id.", {"property_id": _PID}, ["property_id"]),
+ _tool(
+ "get_report_summary",
+ "Health score, issue counts by priority, crawl stats, and category scores for the latest or specified report.",
+ {"property_id": _PID, "report_id": _RID},
+ ),
+ _tool("get_category_scores", "Category scores and overall health score for a report.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_executive_summary", "AI-generated executive summary narrative for the audit report.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_report_meta", "Report metadata: crawl scope, fetch methods, integration freshness.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_site_level", "Site-level robots.txt and sitemap.xml checks.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_report_history", "Past audit reports for a property (report IDs and dates).", {"property_id": _PID, "limit": _LIMIT}),
+ _tool(
+ "list_issues",
+ "List audit issues with optional filters. Returns paginated results (max 50).",
+ {
+ "property_id": _PID,
+ "report_id": _RID,
+ "priority": {"type": "string", "enum": ["Critical", "High", "Medium", "Low"]},
+ "category_id": {"type": "string"},
+ "url_contains": {"type": "string"},
+ "limit": _LIMIT,
+ },
+ ),
+ _tool("list_issues_by_category", "List issues for one category_id only.", {"category_id": {"type": "string"}, "property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ["category_id"]),
+ _tool("get_category_issues", "Full issue list and score for a single audit category.", {"category_id": {"type": "string"}, "property_id": _PID, "report_id": _RID}, ["category_id"]),
+ _tool("list_issue_workflow", "Issue triage workflow status from issue_status table.", {"property_id": _PID, "status": {"type": "string"}, "limit": _LIMIT}),
+ _tool("search_pages", "Search crawled pages by status code or URL substring. Max 30 results.", {"property_id": _PID, "report_id": _RID, "status": {"type": "string"}, "url_contains": {"type": "string"}, "limit": {"type": "integer", "maximum": 30}}),
+ _tool("get_page_details", "Crawl row, Lighthouse snippet, and GSC/GA4 slice for one URL.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]),
+ _tool("get_internal_links", "Inlinks and outlinks for one URL from crawl graph.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]),
+ _tool("list_redirects", "Redirect chains detected in crawl (3xx with final URL).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_broken_links", "Broken internal links (4xx/5xx) from crawl issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_status_code_breakdown", "HTTP status code counts from crawl summary.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_response_time_stats", "Response time percentiles (p50, p95) from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_depth_distribution", "Crawl depth histogram (clicks from start URL).", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_crawl_segments", "Per-path-prefix crawl segment rollups (requires crawl_path_segments config).", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_browser_diagnostics_summary", "Aggregated JS console errors and browser diagnostics from rendered crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_schema_coverage", "Site-wide schema.org coverage from crawl (has_schema + JSON-LD types).", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_pages_without_schema", "URLs missing structured data markup. Empty list means full coverage.", {"property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 30}}),
+ _tool("search_pages_by_schema_type", "Find pages with a specific JSON-LD type (e.g. Organization, Article).", {"schema_type": {"type": "string"}, "property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 30}}, ["schema_type"]),
+ _tool("get_seo_health", "On-page SEO KPI counts: titles, meta descriptions, H1s, thin content.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_orphan_pages", "Crawled URLs with zero internal inlinks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_top_linked_pages", "Most-linked internal pages by inlink count.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_outbound_link_domains", "External domains linked from the site.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_link_graph_summary", "Internal link graph node/edge counts and top hub pages.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_url_fingerprints", "URL pattern fingerprints for duplicate URL detection.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_indexation_coverage", "Sitemap vs crawl vs GSC URL set comparison and gap lists.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_hreflang_summary", "Hreflang alternate tag coverage and issues.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_language_summary", "Detected page language distribution.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_content_analytics", "Word count stats, thin pages, top site keywords from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_content_duplicates", "Near-duplicate content clusters from ML analysis.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_social_coverage", "Open Graph and Twitter card coverage percentages.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_keyword_opportunities", "On-page keyword opportunity hints from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_ner_site_summary", "Named-entity summary across site content.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_thin_content_pages", "Pages flagged as thin content (low word count).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_keyword_summary", "Top keywords, striking-distance count, and GSC metrics.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("search_keywords", "Search keyword list by substring match.", {"property_id": _PID, "query": {"type": "string"}, "limit": _LIMIT}, ["property_id", "query"]),
+ _tool("get_striking_distance_keywords", "Keywords ranking positions 4–20 (striking distance opportunities).", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_keyword_cannibalisation", "Queries where multiple pages rank in GSC.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_query_page_misalignment", "GSC queries whose landing page may not match intent.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_semantic_keyword_clusters", "LLM-generated semantic keyword clusters.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_keyword_history", "Time-series GSC metrics for one keyword.", {"property_id": _PID, "keyword": {"type": "string"}, "limit": _LIMIT}, ["property_id", "keyword"]),
+ _tool("get_google_summary", "GSC and GA4 headline metrics, top queries and pages.", {"property_id": _PID}),
+ _tool("get_gsc_top_queries", "Top Search Console queries by clicks.", {"property_id": _PID, "limit": _LIMIT}),
+ _tool("get_gsc_top_pages", "Top Search Console pages by clicks.", {"property_id": _PID, "limit": _LIMIT}),
+ _tool("get_ga4_summary", "GA4 organic summary and top landing pages.", {"property_id": _PID, "limit": _LIMIT}),
+ _tool("get_gsc_page_query_slice", "GSC queries and metrics for a single page URL.", {"url": _URL, "property_id": _PID}, ["url"]),
+ _tool("get_gsc_links_summary", "GSC Links CSV import summary: top linking sites and linked pages.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_gsc_links_import_status", "Whether GSC Links data is imported and when.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_competitor_link_gap", "Domains linking to competitors but not you (requires competitor_domains config).", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_bing_backlinks_summary", "Bing Webmaster backlinks summary (if API key configured).", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_lighthouse_summary", "Site-wide Lighthouse summary and poor-performance pages.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_lighthouse_for_url", "Lighthouse scores and audits for one URL.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]),
+ _tool("get_lighthouse_diagnostics", "Lighthouse audit diagnostics across sampled pages.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_crux_summary", "Chrome UX Report field data (CrUX) for origin.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_slow_pages", "Pages with Lighthouse performance below threshold (default 50).", {"property_id": _PID, "report_id": _RID, "performance_threshold": {"type": "integer"}, "limit": _LIMIT}),
+ _tool("get_health_history", "Historical health score snapshots for trend analysis.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("compare_reports", "Full audit drift comparison between current and baseline report.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]),
+ _tool("get_integration_alerts", "Stale GSC Links imports and health score drop alerts.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_tech_stack_summary", "Detected technologies (CMS, analytics, CDN) from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_security_findings", "Security header and TLS findings from security scan.", {"property_id": _PID, "report_id": _RID, "severity": {"type": "string"}, "limit": _LIMIT}),
+ # Report extras
+ _tool("get_audit_recommendations", "Actionable SEO recommendation bullets from the audit.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_ml_errors", "ML analysis errors (duplicates, NER, clusters) if any failed.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_ssl_expiry_info", "Site TLS certificate expiry from the audit.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_audit_categories", "All audit categories with scores and issue counts.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_category_recommendations", "Category-level recommendations for one category_id.", {"category_id": {"type": "string"}, "property_id": _PID, "report_id": _RID}, ["category_id"]),
+ _tool("list_issues_with_ai_fixes", "Issues that include LLM-generated fix suggestions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ # On-page
+ _tool("list_seo_onpage_issues", "Flat SEO issue list (titles, meta, H1, thin content) with optional issue_type filter.", {"property_id": _PID, "report_id": _RID, "issue_type": {"type": "string"}, "limit": _LIMIT}),
+ _tool("list_content_url_issues", "Pages in a content_urls bucket (missing_title, missing_h1, thin_content, etc.).", {"bucket": {"type": "string"}, "property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ["bucket"]),
+ _tool("list_pages_missing_title", "Pages with no title tag.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_h1", "Pages with no H1.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_multiple_h1", "Pages with multiple H1 elements.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_meta_description", "Pages with no meta description.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_meta_desc_too_short", "Pages with meta description under 70 characters.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_meta_desc_too_long", "Pages with meta description over 160 characters.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_noindex", "Crawled pages with noindex directive.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ # Charts / aggregates
+ _tool("get_crawl_summary", "Full crawl summary block (URL counts, success rate, timing).", {"property_id": _PID, "report_id": _RID}),
+ _tool(
+ "get_issue_priority_breakdown",
+ "Issue counts by priority (Critical/High/Medium/Low) as chart data for chat UI.",
+ {"property_id": _PID, "report_id": _RID},
+ ),
+ _tool(
+ "get_critical_issues",
+ "All Critical-priority audit issues with URL, category, and message (for chat issue table).",
+ {"property_id": _PID, "report_id": _RID, "limit": _LIMIT},
+ ),
+ _tool("get_mime_type_breakdown", "Content-Type distribution from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_title_length_distribution", "Title length histogram from crawl.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_domain_link_distribution", "Internal link domain breakdown chart data.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_outlink_distribution", "Outlink count distribution chart data.", {"property_id": _PID, "report_id": _RID}),
+ _tool("get_top_crawled_pages", "Top pages by inlinks from crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ # Indexation depth
+ _tool("list_indexation_gaps", "URLs in an indexation gap list (sitemap_only, crawled_not_in_sitemap, gsc_not_crawled).", {"gap_type": {"type": "string"}, "property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}, ["gap_type"]),
+ _tool("get_indexation_url_join", "GSC vs crawl URL join table from indexation coverage.", {"property_id": _PID, "report_id": _RID}),
+ # Backlinks depth
+ _tool("get_gsc_sample_links", "Sample backlinks from GSC Links CSV import.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 100}}, ["property_id"]),
+ _tool("get_gsc_latest_links", "Latest discovered backlinks from GSC Links import.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 100}}, ["property_id"]),
+ _tool("get_third_party_links_overlay", "Moz/Majestic third-party backlink overlays.", {"property_id": _PID, "provider": {"type": "string"}}, ["property_id"]),
+ _tool("get_backlinks_velocity", "Referring-domain trend from gsc_links_snapshots.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ # Ops / integrations
+ _tool("get_property_ops", "Schedule cron and alert webhook/email settings (read-only).", {"property_id": _PID}, ["property_id"]),
+ _tool("get_google_integration_status", "Google OAuth, GSC/GA4 mapping, and data freshness.", {"property_id": _PID}, ["property_id"]),
+ _tool("list_crawl_runs", "Recent crawl run history for a property.", {"property_id": _PID, "limit": _LIMIT}),
+ _tool("list_log_uploads", "Access log file uploads for a property.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_latest_log_analysis", "Most recent parsed access log analysis.", {"property_id": _PID}, ["property_id"]),
+ # Keywords depth
+ _tool("get_keyword_serp_overlay", "Keywords with SERP competition overlay data.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("list_keywords_by_action", "Keywords filtered by recommended_action.", {"property_id": _PID, "recommended_action": {"type": "string"}, "limit": _LIMIT}, ["property_id", "recommended_action"]),
+ _tool("list_keywords_by_position", "Keywords filtered by GSC position range.", {"property_id": _PID, "min_position": {"type": "number"}, "max_position": {"type": "number"}, "limit": _LIMIT}, ["property_id"]),
+ _tool("list_keywords_by_impressions", "Keywords with at least min_impressions.", {"property_id": _PID, "min_impressions": {"type": "integer"}, "limit": _LIMIT}, ["property_id"]),
+ # Lighthouse depth
+ _tool("get_lighthouse_human_summary", "Natural-language Lighthouse summary narrative.", {"property_id": _PID, "report_id": _RID}),
+ _tool("list_lighthouse_poor_seo_pages", "Pages with Lighthouse SEO score below threshold.", {"property_id": _PID, "report_id": _RID, "seo_threshold": {"type": "integer"}, "limit": _LIMIT}),
+ # Crawl depth
+ _tool("get_page_analysis", "Full page_analysis JSON (schema, console errors, accessibility) for one URL.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]),
+ _tool("search_pages_advanced", "Search crawl with filters: status, noindex, word count, fetch method, missing title.", {"property_id": _PID, "report_id": _RID, "status": {"type": "string"}, "url_contains": {"type": "string"}, "noindex_only": {"type": "boolean"}, "missing_title": {"type": "boolean"}, "min_word_count": {"type": "integer"}, "max_word_count": {"type": "integer"}, "fetch_method": {"type": "string"}, "limit": {"type": "integer", "maximum": 50}}),
+ _tool("list_pages_with_console_errors", "Rendered pages with JS console errors.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_by_fetch_method", "Pages crawled via static or rendered fetch.", {"property_id": _PID, "report_id": _RID, "fetch_method": {"type": "string"}, "limit": _LIMIT}, ["fetch_method"]),
+ _tool("get_crawl_links_table", "Paginated links table from report payload.", {"property_id": _PID, "report_id": _RID, "url_contains": {"type": "string"}, "limit": {"type": "integer", "maximum": 100}}),
+ _tool("get_graph_edges_sample", "Sample of internal link graph edges.", {"property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}),
+ _tool("list_status_4xx_pages", "All crawled 4xx pages.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_status_5xx_pages", "All crawled 5xx pages.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ # Google depth
+ _tool("get_ga4_page_metrics", "GA4 metrics for a landing page path or URL.", {"property_id": _PID, "path": {"type": "string"}, "url": _URL}),
+ # Health depth
+ _tool("get_category_health_history", "Category score trend from audit_health_snapshots.", {"property_id": _PID, "category_id": {"type": "string"}, "limit": _LIMIT}, ["property_id"]),
+ # Compare slices
+ _tool("compare_issue_deltas", "New and resolved issues vs baseline report.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 100}}, ["baseline_report_id"]),
+ _tool("compare_category_deltas", "Category score changes vs baseline report.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]),
+ _tool("compare_seo_health_deltas", "On-page SEO KPI changes vs baseline report.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]),
+ _tool("compare_lighthouse_deltas", "Lighthouse score changes per URL vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": _LIMIT}, ["baseline_report_id"]),
+ _tool("compare_url_set_diff", "URLs added or removed from crawl vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}, ["baseline_report_id"]),
+ _tool("compare_redirect_deltas", "Redirect chain changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 100}}, ["baseline_report_id"]),
+ _tool("compare_link_metric_deltas", "Per-URL inlink/outlink/word-count changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}, ["baseline_report_id"]),
+]
diff --git a/src/website_profiling/tools/audit_tools/workflow.py b/src/website_profiling/tools/audit_tools/workflow.py
new file mode 100644
index 00000000..23ec6327
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/workflow.py
@@ -0,0 +1,44 @@
+"""Issue workflow status tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...db._common import _row_field
+from ._slice import parse_limit
+from .context import AuditToolContext
+
+
+def list_issue_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ limit = parse_limit(args.get("limit"), 50, 50)
+ status_filter = str(args.get("status") or "").strip()
+ cur = conn.execute(
+ """SELECT issue_key, url, category, priority, message, status, assignee, note, updated_at
+ FROM issue_status
+ WHERE property_id = %s
+ ORDER BY updated_at DESC
+ LIMIT %s""",
+ (scoped.property_id, limit),
+ )
+ rows = []
+ for row in cur.fetchall() or []:
+ st = str(_row_field(row, "status", index=5) or "")
+ if status_filter and st != status_filter:
+ continue
+ updated = _row_field(row, "updated_at", index=8)
+ rows.append({
+ "issue_key": _row_field(row, "issue_key", index=0),
+ "url": _row_field(row, "url", index=1),
+ "category": _row_field(row, "category", index=2),
+ "priority": _row_field(row, "priority", index=3),
+ "message": _row_field(row, "message", index=4),
+ "status": st,
+ "assignee": _row_field(row, "assignee", index=6),
+ "note": _row_field(row, "note", index=7),
+ "updated_at": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""),
+ })
+ return {"issues": rows, "count": len(rows), "property_id": scoped.property_id}
diff --git a/tests/test_audit_tools.py b/tests/test_audit_tools.py
new file mode 100644
index 00000000..b203e677
--- /dev/null
+++ b/tests/test_audit_tools.py
@@ -0,0 +1,356 @@
+"""Tests for audit_tools registry and issue/page queries."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from website_profiling.tools.audit_tools import AuditToolContext, dispatch_tool
+from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx
+from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, openai_tools_schema
+from website_profiling.tools.audit_tools.report import (
+ get_category_scores,
+ get_report_summary,
+ list_issues,
+)
+
+
+def _sample_payload() -> dict:
+ return {
+ "site_name": "Example",
+ "report_generated_at": "2026-06-07T12:00:00Z",
+ "crawl_run_id": 9,
+ "summary": {"total_urls": 10, "count_2xx": 8, "count_4xx": 2},
+ "lighthouse_summary": {"performance": 72},
+ "lighthouse_human_summary": "OK",
+ "lighthouse_diagnostics": [{"id": "x"}],
+ "lighthouse_by_url": {
+ "https://ex.com/slow": {"performance": 40},
+ "https://ex.com/ok": {"performance": 90},
+ },
+ "google": {"fetched_at": "2026-06-07", "gsc": {"summary": {"clicks": 1}}},
+ "keywords": {
+ "fetched_at": "2026-06-07",
+ "total_keywords": 2,
+ "rows": [{"keyword": "widgets", "score": 0.5, "gsc_clicks": 3}],
+ "striking_distance": [{"keyword": "repair"}],
+ },
+ "categories": [
+ {
+ "id": "technical_seo",
+ "name": "Technical SEO",
+ "score": 80,
+ "issues": [
+ {"priority": "Critical", "message": "Missing title", "url": "https://ex.com/a", "recommendation": "Add title"},
+ {"priority": "High", "message": "Slow page", "url": "https://ex.com/blog/slow", "recommendation": "Optimize"},
+ {"priority": "Low", "message": "Minor", "url": "https://ex.com/z", "recommendation": "Fix"},
+ ],
+ },
+ {
+ "id": "link_health",
+ "name": "Links",
+ "score": 70,
+ "issues": [
+ {"priority": "High", "message": "Broken link", "url": "https://ex.com/404", "recommendation": "Fix link"},
+ ],
+ },
+ "not-a-dict",
+ ],
+ }
+
+
+def test_tool_definitions_have_required_fields() -> None:
+ for tool in TOOL_DEFINITIONS:
+ assert tool.get("name")
+ assert tool.get("description")
+ assert "inputSchema" in tool
+
+
+def test_openai_tools_schema() -> None:
+ schema = openai_tools_schema()
+ assert len(schema) == len(TOOL_DEFINITIONS)
+ assert schema[0]["type"] == "function"
+
+
+def test_dispatch_unknown_tool() -> None:
+ conn = MagicMock()
+ result = dispatch_tool("nonexistent", {}, conn=conn)
+ assert result["error"] == "unknown tool: nonexistent"
+
+
+def test_dispatch_via_db_session() -> None:
+ conn = MagicMock()
+ with patch("website_profiling.tools.audit_tools.registry.db_session") as mock_sess:
+ mock_sess.return_value.__enter__.return_value = conn
+ with patch(
+ "website_profiling.tools.audit_tools.properties.list_properties_public",
+ return_value=[],
+ ):
+ result = dispatch_tool("list_properties", {})
+ assert result["count"] == 0
+
+
+def test_context_with_args_and_loaders() -> None:
+ conn = MagicMock()
+ ctx = Ctx(property_id=1, report_id=2)
+ merged = ctx.with_args({"property_id": 3, "report_id": 4})
+ assert merged.property_id == 3
+ assert merged.report_id == 4
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_report_payload",
+ return_value=_sample_payload(),
+ ):
+ payload = ctx.load_payload(conn)
+ assert payload["site_name"] == "Example"
+
+ df = pd.DataFrame([{"url": "https://ex.com", "status": "200"}])
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_report_payload",
+ return_value=_sample_payload(),
+ ), patch(
+ "website_profiling.tools.audit_tools.context.read_crawl",
+ return_value=df,
+ ):
+ assert not ctx.load_crawl_df(conn).empty
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_latest_google_data",
+ return_value={"gsc": {}},
+ ):
+ assert ctx.load_google(conn) is not None
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_latest_google_data",
+ return_value=None,
+ ), patch(
+ "website_profiling.tools.audit_tools.context.read_report_payload",
+ return_value=_sample_payload(),
+ ):
+ assert ctx.load_google(conn)["gsc"]["summary"]["clicks"] == 1
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_latest_keyword_data",
+ return_value={"rows": []},
+ ):
+ assert ctx.load_keywords(conn) is not None
+
+
+def test_list_issues_filtering_and_limit() -> None:
+ conn = MagicMock()
+ ctx = AuditToolContext()
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ all_issues = list_issues(conn, ctx, {})
+ assert all_issues["total"] == 4
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ critical = list_issues(conn, ctx, {"priority": "Critical"})
+ assert critical["total"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ blog = list_issues(conn, ctx, {"url_contains": "/blog"})
+ assert blog["total"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ capped = list_issues(conn, ctx, {"limit": 2})
+ assert capped["truncated"] is True
+
+
+def test_list_issues_empty_report() -> None:
+ conn = MagicMock()
+ with patch.object(Ctx, "load_payload", return_value={}):
+ result = list_issues(conn, AuditToolContext(), {})
+ assert result["error"] == "no report found"
+
+
+def test_get_report_summary_and_categories() -> None:
+ conn = MagicMock()
+ ctx = AuditToolContext()
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ summary = get_report_summary(conn, ctx, {})
+ assert summary["health_score"] == 75
+ assert summary["total_issues"] == 4
+
+ with patch.object(Ctx, "load_payload", return_value={}):
+ empty = get_category_scores(conn, ctx, {})
+ assert empty["error"] == "no report found"
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ cats = get_category_scores(conn, ctx, {})
+ assert len(cats["categories"]) == 2
+
+
+def test_properties_tools() -> None:
+ conn = MagicMock()
+ with patch(
+ "website_profiling.tools.audit_tools.properties.list_properties_public",
+ return_value=[{"id": 1}],
+ ):
+ assert dispatch_tool("list_properties", {}, conn=conn)["count"] == 1
+
+ with patch(
+ "website_profiling.tools.audit_tools.properties.get_property_by_id",
+ return_value=None,
+ ):
+ missing = dispatch_tool("get_property", {"property_id": 9}, conn=conn)
+ assert "not found" in missing["error"]
+
+ with patch(
+ "website_profiling.tools.audit_tools.properties.get_property_by_id",
+ return_value={"id": 1, "name": "ex.com", "canonical_domain": "ex.com"},
+ ):
+ ok = dispatch_tool("get_property", {"property_id": 1}, conn=conn)
+ assert ok["property"]["id"] == 1
+
+ assert dispatch_tool("get_property", {}, context=AuditToolContext(), conn=conn)["error"]
+
+
+def test_crawl_tools() -> None:
+ conn = MagicMock()
+ df = pd.DataFrame([
+ {"url": "https://ex.com/ok", "status": "200", "title": "OK", "inlinks": 3},
+ {"url": "https://ex.com/missing", "status": "404", "title": "", "inlinks": 0},
+ ])
+ ctx = AuditToolContext(property_id=1)
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ pages = dispatch_tool("search_pages", {"status": "404"}, context=ctx, conn=conn)
+ assert pages["total"] == 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()), patch.object(
+ Ctx, "load_payload", return_value=_sample_payload(),
+ ), patch(
+ "website_profiling.tools.audit_tools.crawl.slice_from_google_row",
+ return_value={"gsc": {"clicks": 1}},
+ ):
+ detail = dispatch_tool(
+ "get_page_details",
+ {"url": "https://ex.com/ok"},
+ context=ctx,
+ conn=conn,
+ )
+ assert detail["found_in_crawl"] is False
+
+ with patch(
+ "website_profiling.db.crawl_store.read_edges",
+ return_value=[("https://ex.com", "https://ex.com/child")],
+ ), patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ links = dispatch_tool(
+ "get_internal_links",
+ {"url": "https://ex.com"},
+ context=ctx,
+ conn=conn,
+ )
+ assert links["outlink_count"] == 1
+
+ assert dispatch_tool("get_page_details", {}, context=ctx, conn=conn)["error"]
+
+
+def test_lighthouse_keywords_google_health() -> None:
+ conn = MagicMock()
+ ctx = AuditToolContext(property_id=1, report_id=1)
+ df = pd.DataFrame([
+ {"url": "https://ex.com/ok", "status": "200", "title": "OK", "inlinks": 3},
+ {"url": "https://ex.com/missing", "status": "404", "title": "", "inlinks": 0},
+ ])
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ lh = dispatch_tool("get_lighthouse_summary", {}, context=ctx, conn=conn)
+ assert lh["pages_audited"] == 2
+ assert lh["poor_performance_pages"]
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ url_lh = dispatch_tool(
+ "get_lighthouse_for_url",
+ {"url": "https://ex.com/slow"},
+ context=ctx,
+ conn=conn,
+ )
+ assert url_lh["lighthouse"]["performance"] == 40
+
+ with patch.object(Ctx, "load_keywords", return_value=_sample_payload()["keywords"]):
+ kw = dispatch_tool("get_keyword_summary", {"property_id": 1}, context=ctx, conn=conn)
+ assert kw["total_keywords"] == 2
+
+ with patch.object(Ctx, "load_keywords", return_value=_sample_payload()["keywords"]):
+ search = dispatch_tool(
+ "search_keywords",
+ {"property_id": 1, "query": "widget"},
+ context=ctx,
+ conn=conn,
+ )
+ assert search["total"] == 1
+
+ with patch.object(Ctx, "load_google", return_value={
+ "fetched_at": "x",
+ "gsc": {"summary": {"clicks": 5}, "top_queries": [], "top_pages": []},
+ "ga4": {"summary": {"sessions": 10}, "top_pages": []},
+ }):
+ g = dispatch_tool("get_google_summary", {}, context=ctx, conn=conn)
+ assert g["gsc"]["summary"]["clicks"] == 5
+
+ from datetime import datetime, timezone
+
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor = lambda cur: None # type: ignore
+ from tests.db_test_fakes import FakeConn, FakeCursor
+
+ fake = FakeConn()
+ fake.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[{
+ "health_score": 80,
+ "category_scores": "{}",
+ "issue_counts": "{}",
+ "generated_at": now,
+ "report_id": 1,
+ }],
+ ),
+ )
+ hist = dispatch_tool("get_health_history", {"property_id": 1}, conn=fake)
+ assert hist["count"] == 1
+
+ assert dispatch_tool("get_keyword_summary", {}, context=AuditToolContext(), conn=conn)["error"]
+ assert dispatch_tool("search_keywords", {"property_id": 1}, context=ctx, conn=conn)["error"]
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ miss_lh = dispatch_tool("get_lighthouse_for_url", {"url": "https://ex.com/none"}, context=ctx, conn=conn)
+ assert "error" in miss_lh
+
+ with patch.object(Ctx, "load_google", return_value=None):
+ no_g = dispatch_tool("get_google_summary", {}, context=ctx, conn=conn)
+ assert no_g["error"]
+
+ assert dispatch_tool("get_health_history", {}, context=AuditToolContext(), conn=conn)["error"]
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ url_search = dispatch_tool("search_pages", {"url_contains": "missing"}, context=ctx, conn=conn)
+ assert url_search["total"] == 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df), patch.object(
+ Ctx, "load_payload", return_value=_sample_payload(),
+ ), patch(
+ "website_profiling.tools.audit_tools.crawl.slice_from_google_row",
+ return_value={},
+ ):
+ found = dispatch_tool(
+ "get_page_details",
+ {"url": "https://ex.com/ok"},
+ context=ctx,
+ conn=conn,
+ )
+ assert found["found_in_crawl"] is True
+
+ assert dispatch_tool("get_internal_links", {}, context=ctx, conn=conn)["error"]
+
+ bad_prop = dispatch_tool("get_property", {"property_id": "bad"}, conn=conn)
+ assert bad_prop["error"] == "invalid property_id"
+
+ with patch.object(Ctx, "load_payload", return_value=_sample_payload()):
+ cat_filter = dispatch_tool(
+ "list_issues",
+ {"category_id": "technical_seo"},
+ context=ctx,
+ conn=conn,
+ )
+ assert cat_filter["total"] == 3
diff --git a/tests/test_audit_tools_coverage.py b/tests/test_audit_tools_coverage.py
new file mode 100644
index 00000000..84922341
--- /dev/null
+++ b/tests/test_audit_tools_coverage.py
@@ -0,0 +1,677 @@
+"""Line-coverage tests for audit_tools edge paths."""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+
+from website_profiling.tools.audit_tools import _slice, dispatch_tool
+from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx
+from website_profiling.tools.audit_tools import compare as compare_mod
+from website_profiling.tools.audit_tools import crawl as crawl_mod
+from website_profiling.tools.audit_tools import keywords as kw_mod
+from website_profiling.tools.audit_tools import report as report_mod
+from website_profiling.tools.audit_tools import health as health_mod
+from website_profiling.tools.audit_tools import google as google_mod
+from website_profiling.tools.audit_tools import lighthouse as lh_mod
+from website_profiling.tools.audit_tools import links as links_mod
+from website_profiling.tools.audit_tools import backlinks as bl_mod
+from website_profiling.tools.audit_tools import content as content_mod
+from website_profiling.tools.audit_tools import issues as issues_mod
+from website_profiling.tools.audit_tools import schema as schema_mod
+
+
+def test_slice_edge_cases() -> None:
+ assert _slice.payload_field({"n": 1}, "n", 5)["total"] == 1
+ assert _slice.payload_field({"x": [1]}, "x", 5, filter_fn=lambda i: False)["total"] == 0
+ assert _slice.crawl_filter(None)["total"] == 0
+ df = pd.DataFrame([
+ {"url": "https://ex.com", "status": "200", "page_analysis": "not-json"},
+ {"url": "https://ex.com/2", "status": "200", "has_schema": "true", "page_analysis": "{}"},
+ ])
+ assert _slice.crawl_filter(df, url_contains="ex")["total"] == 2
+ assert _slice.crawl_filter(df, status="404")["total"] == 0
+ row = {"page_analysis": json.dumps({"schema_types": "Article"})}
+ assert _slice._row_schema_types_list(row) == ["Article"]
+ assert _slice._row_schema_types(row) == "article"
+
+
+def test_report_helpers() -> None:
+ assert report_mod._normalize_priority("") == ""
+ assert report_mod._normalize_priority("critical") == "Critical"
+ issues = report_mod._iter_category_issues({"categories": ["bad", {"id": "x", "issues": ["bad", {"priority": "Low", "message": "m"}]}]})
+ assert issues
+ assert report_mod._health_score({"categories": [{"score": "bad"}]}) is None
+ conn = MagicMock()
+ ctx = Ctx()
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert report_mod.get_executive_summary(conn, ctx, {})["error"]
+ assert report_mod.get_report_meta(conn, ctx, {})["error"]
+ assert report_mod.get_site_level(conn, ctx, {})["error"]
+ with patch.object(Ctx, "load_payload", return_value={"executive_summary": {"headline": "OK"}}):
+ assert report_mod.get_executive_summary(conn, ctx, {})["executive_summary"]["headline"] == "OK"
+
+
+def test_compare_paths() -> None:
+ conn = MagicMock()
+ ctx = Ctx(property_id=1)
+ with patch.object(compare_mod, "read_report_payload", return_value={"categories": []}):
+ assert compare_mod.compare_reports(conn, ctx, {"baseline_report_id": "x"})["error"] == "invalid baseline_report_id"
+
+ conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value=None)))
+ with patch.object(compare_mod, "read_report_payload", return_value=None):
+ assert "no current" in compare_mod.compare_reports(conn, ctx, {"baseline_report_id": 1})["error"]
+
+ conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value={"id": 5})))
+ with patch.object(compare_mod, "read_report_payload", side_effect=[{"a": 1}, None]):
+ assert "not found" in compare_mod.compare_reports(conn, Ctx(report_id=None), {"baseline_report_id": 1})["error"]
+
+ assert compare_mod._row_id({"id": 3}) == 3
+ assert compare_mod._row_id((7,)) == 7
+
+
+def test_context_loaders(conn: MagicMock | None = None) -> None:
+ conn = conn or MagicMock()
+ ctx = Ctx()
+ with patch("website_profiling.tools.audit_tools.context.read_report_payload", return_value=None):
+ assert ctx.load_report_payload_by_id(conn, 1) == {}
+ with patch.object(Ctx, "load_payload", return_value={"top_pages": [{"url": "https://www.ex.com/page"}]}):
+ assert ctx.resolve_property_domain(conn) == "www.ex.com"
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert ctx.resolve_property_domain(conn) == ""
+ with patch("website_profiling.tools.audit_tools.context.read_crawl", return_value=pd.DataFrame()), patch.object(
+ Ctx, "load_payload", return_value={"crawl_run_id": "bad"},
+ ):
+ assert ctx.load_crawl_df(conn).empty
+
+
+def test_keywords_and_google(conn: MagicMock | None = None) -> None:
+ conn = conn or MagicMock()
+ ctx = Ctx(property_id=1)
+ assert kw_mod.search_keywords(conn, ctx, {"query": ""})["error"]
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.get_striking_distance_keywords(conn, ctx, {})["error"]
+ with patch.object(Ctx, "load_keywords", return_value={"rows": "bad"}):
+ assert kw_mod.get_keyword_summary(conn, ctx, {"limit": "x"})["total_keywords"] == 0
+ with patch.object(Ctx, "load_google", return_value=None):
+ assert google_mod.get_gsc_top_queries(conn, ctx, {})["error"]
+ assert google_mod.get_gsc_page_query_slice(conn, ctx, {})["error"]
+
+
+def test_crawl_lighthouse_links(conn: MagicMock | None = None) -> None:
+ conn = conn or MagicMock()
+ ctx = Ctx()
+ with patch.object(Ctx, "load_payload", return_value={"report_meta": {}}):
+ assert crawl_mod.get_browser_diagnostics_summary(conn, ctx, {})["browser_diagnostics"] is None
+ df = pd.DataFrame([{"url": "https://ex.com/a", "status": "200", "title": "T", "meta_description": "d", "h1": "h", "word_count": 1, "inlinks": 0, "outlinks": 0, "content_type": "text/html"}])
+ with patch.object(Ctx, "load_crawl_df", return_value=df), patch.object(Ctx, "load_payload", return_value={"lighthouse_by_url": {}}), patch.object(Ctx, "load_google", return_value=None):
+ assert crawl_mod.get_page_details(conn, ctx, {"url": "https://ex.com/a"})["found_in_crawl"]
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert links_mod.list_orphan_pages(conn, ctx, {})["error"]
+ with patch.object(Ctx, "load_payload", return_value={"lighthouse_by_url": {}}), patch(
+ "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_page_summaries", return_value={},
+ ):
+ assert lh_mod.get_lighthouse_for_url(conn, ctx, {"url": "https://ex.com"})["error"]
+
+
+def test_security_workflow_backlinks() -> None:
+ from website_profiling.tools.audit_tools import security as sec_mod
+ from website_profiling.tools.audit_tools import workflow as wf_mod
+
+ conn = MagicMock()
+ payload = {"security_findings": [{"url": "u", "severity": "High", "finding_type": "x", "message": "m"}]}
+ ctx = Ctx()
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert sec_mod.get_security_findings(conn, ctx, {"severity": "low"})["total"] == 0
+ assert sec_mod.get_security_findings(conn, ctx, {"severity": "high"})["total"] == 1
+ assert sec_mod.get_security_findings(conn, ctx, {"severity": "High", "limit": "bad"})["total"] == 1
+
+ now = datetime.now(timezone.utc)
+ row = ("k", "u", "c", "Low", "m", "open", None, None, now)
+ conn.execute = MagicMock(return_value=MagicMock(fetchall=MagicMock(return_value=[row])))
+ ctx2 = Ctx(property_id=1)
+ assert wf_mod.list_issue_workflow(conn, ctx2, {"status": "closed"})["count"] == 0
+ assert wf_mod.list_issue_workflow(conn, ctx2, {"limit": "bad"})["count"] == 1
+
+ with patch.object(Ctx, "load_gsc_links", return_value=None):
+ assert bl_mod.get_gsc_links_summary(conn, Ctx(property_id=1), {})["missing"]
+
+
+def test_health_list_history() -> None:
+ conn = MagicMock()
+ ctx = Ctx(property_id=1)
+ with patch.object(Ctx, "resolve_property_domain", return_value=""):
+ conn.execute = MagicMock(return_value=MagicMock(fetchall=MagicMock(return_value=[])))
+ assert health_mod.list_report_history(conn, ctx, {})["count"] == 0
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert health_mod.get_health_history(conn, ctx, {"limit": "bad"})["count"] == 0
+
+
+def test_issues_content_schema() -> None:
+ conn = MagicMock()
+ ctx = Ctx()
+ with patch.object(Ctx, "load_payload", return_value={"categories": []}):
+ assert issues_mod.get_category_issues(conn, ctx, {"category_id": "x"})["error"]
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert schema_mod.get_schema_coverage(conn, ctx, {})["error"]
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert content_mod.get_content_duplicates(conn, ctx, {"limit": "x"})["error"]
+
+
+def test_misc_dispatch() -> None:
+ conn = MagicMock()
+ ctx = Ctx(property_id=None)
+ assert dispatch_tool("get_integration_alerts", {}, context=ctx, conn=conn)["error"]
+ with patch("website_profiling.tools.audit_tools.ops.check_all_alerts", return_value=[]):
+ assert dispatch_tool("get_integration_alerts", {"property_id": 1}, context=Ctx(property_id=1), conn=conn)["count"] == 0
+
+
+def test_remaining_module_paths() -> None:
+ from website_profiling.tools.audit_tools import backlinks as bl
+ from website_profiling.tools.audit_tools import links as links_mod
+ from website_profiling.tools.audit_tools import content as ct
+ from website_profiling.tools.audit_tools import indexation_tools as idx
+ from website_profiling.tools.audit_tools import international as intl
+ from website_profiling.tools.audit_tools import tech as tech_mod
+
+ conn = MagicMock()
+ ctx = Ctx(property_id=1)
+ payload = {
+ "orphan_urls": "bad",
+ "top_pages": "bad",
+ "outbound_link_domains": "bad",
+ "url_fingerprints": "bad",
+ "content_duplicates": "bad",
+ "report_meta": {"a": 1},
+ "site_level": {"robots": True},
+ "indexation_coverage": None,
+ "competitor_link_gap": None,
+ "bing_backlinks": None,
+ "crux_summary": None,
+ "crawl_segments": None,
+ "tech_stack_summary": None,
+ "hreflang_summary": None,
+ "language_summary": None,
+ "content_analytics": None,
+ "social_coverage": None,
+ "keyword_opportunities": None,
+ "ner_site_summary": None,
+ "response_time_stats": None,
+ "depth_distribution": None,
+ "seo_health": None,
+ "issues": {},
+ "redirects": "bad",
+ "security_findings": "bad",
+ "keywords": {"rows": [1, 2]},
+ }
+
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert links_mod.list_orphan_pages(conn, ctx, {})["total"] == 0
+ assert links_mod.get_top_linked_pages(conn, ctx, {})["total"] == 0
+ assert links_mod.get_outbound_link_domains(conn, ctx, {})["total"] == 0
+ assert links_mod.get_url_fingerprints(conn, ctx, {})["total"] == 0
+ assert ct.get_content_duplicates(conn, ctx, {})["total"] in (0, 1)
+ assert report_mod.get_report_meta(conn, ctx, {})["report_meta"]["a"] == 1
+ assert report_mod.get_site_level(conn, ctx, {})["site_level"]["robots"] is True
+ assert idx.get_indexation_coverage(conn, ctx, {})["missing"]
+ assert bl.get_competitor_link_gap(conn, ctx, {})["missing"]
+ assert bl.get_bing_backlinks_summary(conn, ctx, {})["missing"]
+ assert crawl_mod.get_crawl_segments(conn, ctx, {})["missing"]
+ assert tech_mod.get_tech_stack_summary(conn, ctx, {})["missing"]
+ assert intl.get_hreflang_summary(conn, ctx, {})["missing"]
+ assert intl.get_language_summary(conn, ctx, {})["missing"]
+ assert ct.get_content_analytics(conn, ctx, {})["missing"]
+ assert ct.get_social_coverage(conn, ctx, {})["missing"]
+ assert ct.get_keyword_opportunities(conn, ctx, {})["missing"]
+ assert ct.get_ner_site_summary(conn, ctx, {})["missing"]
+ assert crawl_mod.get_response_time_stats(conn, ctx, {})["missing"]
+ assert crawl_mod.get_depth_distribution(conn, ctx, {})["missing"]
+ assert crawl_mod.get_seo_health(conn, ctx, {})["missing"]
+ assert crawl_mod.list_redirects(conn, ctx, {})["total"] == 0
+ assert crawl_mod.list_broken_links(conn, ctx, {})["total"] == 0
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": "x"}):
+ assert kw_mod.get_keyword_summary(conn, ctx, {})["total_keywords"] == 0
+
+ with patch.object(Ctx, "load_google", return_value={"gsc": {"top_queries": "x"}}):
+ assert google_mod.get_gsc_top_queries(conn, ctx, {})["total"] == 0
+
+ with patch.object(Ctx, "load_google", return_value={"ga4": {}}):
+ assert google_mod.get_ga4_summary(conn, ctx, {})["missing"]
+
+ with patch.object(Ctx, "load_google", return_value={"gsc": {}, "ga4": {"summary": {}, "top_pages": "x"}}):
+ assert google_mod.get_ga4_summary(conn, ctx, {})["top_pages"] == []
+
+ lh_payload = {"lighthouse_summary": None, "lighthouse_diagnostics": "x", "lighthouse_by_url": "x", "crux_summary": {"ok": True}}
+ with patch.object(Ctx, "load_payload", return_value=lh_payload), patch(
+ "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_summary", return_value={"p": 1},
+ ), patch(
+ "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_page_summaries", return_value={"u": {"performance": 30}},
+ ):
+ assert lh_mod.get_lighthouse_summary(conn, ctx, {})["pages_audited"] == 1
+ assert lh_mod.get_lighthouse_diagnostics(conn, ctx, {})["total"] == 0
+ assert lh_mod.list_slow_pages(conn, ctx, {})["total"] == 0
+
+ df = pd.DataFrame([{"url": "https://ex.com", "status": "200"}])
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert crawl_mod.search_pages(conn, ctx, {"status": "404", "limit": "bad"})["pages"] == []
+
+ with patch.object(Ctx, "load_payload", return_value={"categories": [{"id": "x", "issues": []}]}):
+ assert issues_mod.list_issues_by_category(conn, ctx, {"category_id": "x"})["total"] == 0
+
+ with patch("website_profiling.tools.audit_tools.context.read_latest_keyword_data", return_value=None), patch.object(
+ Ctx, "load_payload", return_value={"keywords": {"rows": []}},
+ ):
+ assert Ctx(property_id=1).load_keywords(conn) == {"rows": []}
+
+ with patch("website_profiling.tools.audit_tools.context.read_latest_google_data", return_value=None), patch.object(
+ Ctx, "load_payload", return_value={"google": {"gsc": {}}},
+ ):
+ assert Ctx(property_id=1).load_google(conn) == {"gsc": {}}
+
+ # _slice branches
+ assert _slice._parse_page_analysis({"page_analysis": 123}) == {}
+ assert _slice._row_schema_types_list({"page_analysis": json.dumps({"json_ld_types": []})}) == []
+ df2 = pd.DataFrame([{"url": "u", "status": "200", "has_schema": "yes", "page_analysis": "{}"}])
+ assert _slice.crawl_filter(df2, has_schema=True)["total"] == 1
+
+ from tests.db_test_fakes import FakeConn, FakeCursor
+ from datetime import datetime, timezone
+
+ now = datetime.now(timezone.utc)
+ fake = FakeConn()
+ fake.set_next_cursor(FakeCursor(fetchall_value=[(80, "{}", "{}", now, 1)]))
+ hist = health_mod.get_health_history(fake, Ctx(property_id=1), {"limit": 99})
+ assert hist["count"] == 1
+
+ fake2 = FakeConn()
+ fake2.set_next_cursor(FakeCursor(fetchall_value=[]))
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"):
+ assert health_mod.list_report_history(fake2, Ctx(property_id=1), {"limit": "bad"})["count"] == 0
+
+ assert bl.get_gsc_links_import_status(conn, Ctx(property_id=1), {}) # patched in expanded tests
+ with patch("website_profiling.tools.audit_tools.backlinks.read_gsc_links_status", return_value={"hasData": False}):
+ assert bl.get_gsc_links_import_status(conn, Ctx(property_id=1), {})["hasData"] is False
+
+
+def test_new_tools_coverage() -> None:
+ from website_profiling.tools.audit_tools import backlinks as bl_mod
+ from website_profiling.tools.audit_tools import charts as charts_mod
+ from website_profiling.tools.audit_tools import indexation_tools as idx_mod
+ from website_profiling.tools.audit_tools import compare_helpers as ch_mod
+ from website_profiling.tools.audit_tools import compare_slices as cs_mod
+ from website_profiling.tools.audit_tools import onpage as onpage_mod
+ from website_profiling.tools.audit_tools import ops as ops_mod
+ from website_profiling.tools.audit_tools import report_extras as rex_mod
+ from website_profiling.reporting.compare_payload import build_url_set_diff
+
+ conn = MagicMock()
+ ctx = Ctx(property_id=1, report_id=1)
+ payload = {
+ "summary": {"total_urls": 2},
+ "recommendations": ["r1"],
+ "ml_errors": ["e1"],
+ "site_ssl_expires_at": "2027-01-01",
+ "content_urls": {
+ "missing_title": [{"url": "https://ex.com/a"}],
+ "missing_h1": [],
+ "multiple_h1": [],
+ "missing_meta_desc": [],
+ "meta_desc_short": [],
+ "meta_desc_long": [],
+ "thin_content": [],
+ },
+ "issues": {"seo": [{"type": "missing_title", "url": "https://ex.com/a", "message": "m"}]},
+ "mime_labels": ["html"], "mime_values": [1],
+ "title_labels": ["0"], "title_counts": [1],
+ "domain_labels": ["ex.com"], "domain_values": [1],
+ "outlink_labels": ["0"], "outlink_counts": [1],
+ "top_pages": [{"url": "https://ex.com/"}],
+ "links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/new"}],
+ "graph_edges": [[1, 2]],
+ "graph_nodes": [1, 2],
+ "indexation_coverage": {
+ "lists": {"sitemap_only": ["https://ex.com/s"], "crawled_not_in_sitemap": [], "gsc_not_crawled": []},
+ "lists_total": {"sitemap_only": 1},
+ "url_join": [],
+ "counts": {},
+ },
+ "categories": [
+ {"id": "technical_seo", "name": "Tech", "score": 80, "issues": [{"llm_recommendation": "fix"}], "recommendations": ["rec"]},
+ ],
+ "lighthouse_human_summary": "summary text",
+ "lighthouse_by_url": {"https://ex.com/bad": {"seo": 50, "performance": 40}},
+ }
+ df = pd.DataFrame([
+ {"url": "https://ex.com/a", "status": "404", "noindex": "true", "title": "", "word_count": 10, "fetch_method": "static", "page_analysis": json.dumps({"console_errors": ["e"]})},
+ {"url": "https://ex.com/b", "status": "500", "noindex": "false", "title": "T", "word_count": 500, "fetch_method": "rendered", "page_analysis": "{}"},
+ ])
+
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert rex_mod.get_audit_recommendations(conn, ctx, {})["count"] == 1
+ assert rex_mod.get_ml_errors(conn, ctx, {})["count"] == 1
+ assert rex_mod.get_ssl_expiry_info(conn, ctx, {})["site_ssl_expires_at"]
+ assert rex_mod.list_audit_categories(conn, ctx, {})["count"] == 1
+ assert rex_mod.get_category_recommendations(conn, ctx, {"category_id": "technical_seo"})["recommendations"]
+ assert rex_mod.list_issues_with_ai_fixes(conn, ctx, {})["total"] == 1
+ assert onpage_mod.list_pages_missing_title(conn, ctx, {})["total"] == 1
+ assert onpage_mod.list_pages_missing_h1(conn, ctx, {})["total"] == 0
+ assert onpage_mod.list_pages_multiple_h1(conn, ctx, {})["total"] == 0
+ assert onpage_mod.list_pages_missing_meta_description(conn, ctx, {})["total"] == 0
+ assert onpage_mod.list_pages_meta_desc_too_short(conn, ctx, {})["total"] == 0
+ assert onpage_mod.list_pages_meta_desc_too_long(conn, ctx, {})["total"] == 0
+ assert onpage_mod.list_seo_onpage_issues(conn, ctx, {"issue_type": "missing_title"})["total"] == 1
+ assert onpage_mod.list_pages_noindex(conn, ctx, {})["total"] == 1
+ assert charts_mod.get_crawl_summary(conn, ctx, {})["summary"]["total_urls"] == 2
+ assert charts_mod.get_mime_type_breakdown(conn, ctx, {})["items"]
+ assert crawl_mod.list_status_4xx_pages(conn, ctx, {})["total"] == 1
+ assert crawl_mod.list_status_5xx_pages(conn, ctx, {})["total"] == 1
+ assert crawl_mod.get_page_analysis(conn, ctx, {"url": "https://ex.com/a"})["page_analysis"]
+ assert crawl_mod.search_pages_advanced(conn, ctx, {"noindex_only": True})["total"] == 1
+ assert crawl_mod.list_pages_with_console_errors(conn, ctx, {})["total"] == 1
+ assert crawl_mod.list_pages_by_fetch_method(conn, ctx, {"fetch_method": "rendered"})["total"] == 1
+ assert crawl_mod.get_crawl_links_table(conn, ctx, {})["total"] == 2
+ assert crawl_mod.get_graph_edges_sample(conn, ctx, {})["total"] == 1
+ assert lh_mod.get_lighthouse_human_summary(conn, ctx, {})["has_summary"]
+ assert lh_mod.list_lighthouse_poor_seo_pages(conn, ctx, {})["total"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert idx_mod.list_indexation_gaps(conn, ctx, {"gap_type": "sitemap_only"})["total"] == 1
+ assert idx_mod.get_indexation_url_join(conn, ctx, {})["url_join"] == []
+
+ gsc = {"sample_links": [{"a": 1}], "latest_links": [{"b": 2}], "third_party_overlays": [{"provider": "moz"}], "sample_links_full_count": 1, "latest_links_full_count": 1}
+ with patch.object(Ctx, "load_gsc_links", return_value=gsc):
+ assert bl_mod.get_gsc_sample_links(conn, ctx, {})["links"]
+ assert bl_mod.get_gsc_latest_links(conn, ctx, {})["links"]
+ assert bl_mod.get_third_party_links_overlay(conn, ctx, {"provider": "moz"})["count"] == 1
+
+ conn.execute = MagicMock(return_value=MagicMock(
+ fetchall=MagicMock(return_value=[{"captured_at": datetime.now(timezone.utc), "referring_domains": 5, "top_domains": []}]),
+ fetchone=MagicMock(return_value={"schedule_cron": "0 9 * * 1", "alert_webhook_url": "u", "alert_email": "a@b.com"}),
+ ))
+ with patch("website_profiling.tools.audit_tools.ops.get_property_by_id", return_value={"google_refresh_token": "t"}), patch.object(
+ Ctx, "load_google", return_value={"fetched_at": "2026-01-01"},
+ ), patch("website_profiling.integrations.google.gsc_links_store.read_gsc_links_status", return_value={"hasData": True}):
+ assert ops_mod.get_property_ops(conn, ctx, {})["has_schedule"]
+ assert ops_mod.get_google_integration_status(conn, ctx, {})["google_connected"]
+ assert ops_mod.list_crawl_runs(conn, ctx, {})["count"] == 1
+ conn.execute = MagicMock(return_value=MagicMock(
+ fetchall=MagicMock(return_value=[{"id": 1, "filename": "log.txt", "line_count": 10, "uploaded_at": datetime.now(timezone.utc)}]),
+ fetchone=MagicMock(return_value={"filename": "log.txt", "line_count": 10, "analysis": json.dumps({"top_paths": []}), "uploaded_at": datetime.now(timezone.utc)}),
+ ))
+ assert ops_mod.list_log_uploads(conn, ctx, {})["count"] == 1
+ assert ops_mod.get_latest_log_analysis(conn, ctx, {})["analysis"] == {"top_paths": []}
+
+ kw_data = {"rows": [{"keyword": "a", "gsc_position": 5, "gsc_impressions": 100, "recommended_action": "optimize page", "serp_estimated_competition": 0.5}], "serp_overlay_count": 1}
+ with patch.object(Ctx, "load_keywords", return_value=kw_data):
+ assert kw_mod.get_keyword_serp_overlay(conn, ctx, {})["keywords"]
+ assert kw_mod.list_keywords_by_action(conn, ctx, {"recommended_action": "optimize page"})["total"] == 1
+ assert kw_mod.list_keywords_by_position(conn, ctx, {"min_position": 1, "max_position": 10})["total"] == 1
+ assert kw_mod.list_keywords_by_impressions(conn, ctx, {"min_impressions": 50})["total"] == 1
+
+ with patch.object(Ctx, "load_google", return_value={"ga4": {"top_pages": [{"path": "/"}], "summary": {}}, "fetched_at": "x"}):
+ assert google_mod.get_ga4_page_metrics(conn, ctx, {"path": "/"})["metrics"]
+
+ from tests.db_test_fakes import FakeConn, FakeCursor
+ fake = FakeConn()
+ fake.set_next_cursor(FakeCursor(fetchall_value=[({"technical_seo": 80}, datetime.now(timezone.utc), 1, 75)]))
+ assert health_mod.get_category_health_history(fake, Ctx(property_id=1), {"category_id": "technical_seo"})["count"] == 1
+
+ cur_p = {"categories": [], "links": [{"url": "https://ex.com/new"}]}
+ base_p = {"categories": [], "links": [{"url": "https://ex.com/old"}]}
+ diff = build_url_set_diff(cur_p, base_p)
+ assert diff["new_count"] >= 1
+
+ def _read_pair(_conn: MagicMock, rid: int) -> dict:
+ return cur_p if int(rid) == 1 else base_p
+
+ with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", side_effect=_read_pair):
+ assert cs_mod.compare_url_set_diff(conn, ctx, {"baseline_report_id": 2})["new_count"] >= 1
+ assert cs_mod.compare_issue_deltas(conn, ctx, {"baseline_report_id": 2})["issue_deltas"] == []
+ assert cs_mod.compare_category_deltas(conn, ctx, {"baseline_report_id": 2})["category_scores"] == []
+ assert cs_mod.compare_seo_health_deltas(conn, ctx, {"baseline_report_id": 2})["seo_health_metrics"] == []
+ assert cs_mod.compare_lighthouse_deltas(conn, ctx, {"baseline_report_id": 2})["lighthouse_url_deltas"] == []
+ assert cs_mod.compare_redirect_deltas(conn, ctx, {"baseline_report_id": 2})["redirect_deltas"] == []
+ assert cs_mod.compare_link_metric_deltas(conn, ctx, {"baseline_report_id": 2})["link_metric_deltas"] == []
+
+ err = ch_mod.load_compare_pair(conn, ctx, {})
+ assert err[4]["error"] == "baseline_report_id is required"
+ with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", side_effect=[{"a": 1}, None]):
+ err2 = ch_mod.load_compare_pair(conn, Ctx(report_id=1), {"baseline_report_id": 2})
+ assert "not found" in err2[4]["error"]
+
+ for fn in (cs_mod.compare_issue_deltas, cs_mod.compare_category_deltas, cs_mod.compare_seo_health_deltas,
+ cs_mod.compare_lighthouse_deltas, cs_mod.compare_redirect_deltas, cs_mod.compare_link_metric_deltas):
+ assert fn(conn, ctx, {})["error"]
+
+ bl_conn = MagicMock()
+ bl_conn.execute = MagicMock(return_value=MagicMock(
+ fetchall=MagicMock(return_value=[{"captured_at": datetime.now(timezone.utc), "referring_domains": 3, "top_domains": []}]),
+ ))
+ assert bl_mod.get_backlinks_velocity(bl_conn, ctx, {})["count"] == 1
+
+ err_base = ch_mod.load_compare_pair(conn, ctx, {"baseline_report_id": "x"})
+ assert err_base[4]["error"] == "invalid baseline_report_id"
+ conn_no_cur = MagicMock()
+ conn_no_cur.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value=None)))
+ err_cur = ch_mod.load_compare_pair(conn_no_cur, Ctx(report_id=None), {"baseline_report_id": 2})
+ assert "no current" in err_cur[4]["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert charts_mod.get_crawl_summary(conn, ctx, {})["error"]
+ assert onpage_mod.list_content_url_issues(conn, ctx, {"bucket": "missing_title"})["error"]
+
+ # error branches
+ assert onpage_mod.list_content_url_issues(conn, ctx, {"bucket": "bad"})["error"]
+ assert rex_mod.get_category_recommendations(conn, ctx, {})["error"]
+ assert ops_mod.get_property_ops(conn, Ctx(property_id=None), {})["error"]
+ assert ops_mod.list_log_uploads(conn, Ctx(property_id=None), {})["error"]
+ assert bl_mod.get_gsc_sample_links(conn, Ctx(property_id=None), {})["error"]
+ assert kw_mod.list_keywords_by_action(conn, ctx, {})["error"]
+ assert kw_mod.list_keywords_by_impressions(conn, ctx, {"min_impressions": "x"})["error"]
+ assert crawl_mod.list_pages_by_fetch_method(conn, ctx, {})["error"]
+ assert crawl_mod.get_page_analysis(conn, ctx, {"url": ""})["error"]
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert onpage_mod.list_pages_noindex(conn, ctx, {})["pages"] == []
+
+ with patch.object(Ctx, "load_payload", return_value={}):
+ for fn, mod in (
+ (charts_mod.get_mime_type_breakdown, charts_mod),
+ (charts_mod.get_title_length_distribution, charts_mod),
+ (charts_mod.get_domain_link_distribution, charts_mod),
+ (charts_mod.get_outlink_distribution, charts_mod),
+ (charts_mod.get_top_crawled_pages, charts_mod),
+ ):
+ assert fn(conn, ctx, {})["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={"mime_labels": "x", "mime_values": 1}):
+ assert charts_mod.get_mime_type_breakdown(conn, ctx, {})["items"] == []
+ with patch.object(Ctx, "load_payload", return_value={"title_labels": None, "title_counts": []}):
+ assert charts_mod.get_title_length_distribution(conn, ctx, {})["items"] == []
+ with patch.object(Ctx, "load_payload", return_value={"domain_labels": None, "domain_values": []}):
+ assert charts_mod.get_domain_link_distribution(conn, ctx, {})["items"] == []
+ with patch.object(Ctx, "load_payload", return_value={"top_pages": "x"}):
+ assert charts_mod.get_top_crawled_pages(conn, ctx, {})["total"] == 0
+
+ conn_rid = MagicMock()
+ conn_rid.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value={"id": 7})))
+ with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", side_effect=[{"links": []}, {"links": []}]):
+ ok = ch_mod.load_compare_pair(conn_rid, Ctx(report_id=None), {"baseline_report_id": 2})
+ assert ok[4] is None
+ assert ok[2] == 7
+
+ assert cs_mod.compare_url_set_diff(conn, ctx, {})["error"]
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.get_keyword_serp_overlay(conn, ctx, {})["error"]
+ assert kw_mod._filter_keyword_rows(conn, ctx, {}, lambda r: True)["error"]
+
+ with patch.object(Ctx, "load_google", return_value={"ga4": {"top_pages": []}}):
+ assert google_mod.get_ga4_page_metrics(conn, ctx, {"path": "/missing"})["missing"]
+
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert crawl_mod.search_pages_advanced(conn, ctx, {"missing_title": True})["total"] == 1
+ assert crawl_mod.search_pages_advanced(conn, ctx, {"max_word_count": 50})["total"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert idx_mod.list_indexation_gaps(conn, ctx, {"gap_type": "bad"})["error"]
+ assert rex_mod.list_issues_with_ai_fixes(conn, ctx, {})["error"]
+
+ ops_conn = MagicMock()
+ ops_conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value=None)))
+ assert ops_mod.get_property_ops(ops_conn, ctx, {})["error"] == "property not found"
+ assert ops_mod.get_latest_log_analysis(ops_conn, ctx, {})["missing"]
+
+ with patch("website_profiling.tools.audit_tools.ops.get_property_by_id", return_value=None):
+ assert ops_mod.get_google_integration_status(conn, ctx, {})["error"] == "property not found"
+
+ with patch.object(Ctx, "load_gsc_links", return_value=None):
+ assert bl_mod.get_gsc_latest_links(conn, ctx, {})["missing"]
+ assert bl_mod.get_third_party_links_overlay(conn, ctx, {})["missing"]
+
+ with patch.object(Ctx, "load_payload", return_value={"content_urls": "bad"}):
+ assert onpage_mod.list_pages_missing_title(conn, ctx, {})["missing"]
+
+ fake_h = FakeConn()
+ fake_h.set_next_cursor(FakeCursor(fetchall_value=[('{"technical_seo": 70}', datetime.now(timezone.utc), 2, 72)]))
+ assert health_mod.get_category_health_history(fake_h, Ctx(property_id=1), {})["count"] == 1
+ fake_h2 = FakeConn()
+ fake_h2.set_next_cursor(FakeCursor(fetchall_value=[("not-json", datetime.now(timezone.utc), 2, 72)]))
+ assert health_mod.get_category_health_history(fake_h2, Ctx(property_id=1), {"category_id": "technical_seo"})["count"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value={"lighthouse_human_summary": "", "lighthouse_summary": {"human_summary": "from summary"}}):
+ assert lh_mod.get_lighthouse_human_summary(conn, ctx, {})["has_summary"]
+
+ assert ch_mod._row_id({"id": 9}) == 9
+ assert ch_mod._row_id((8,)) == 8
+ with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", return_value=None):
+ err_cur_only = ch_mod.load_compare_pair(conn, Ctx(report_id=3), {"baseline_report_id": 2})
+ assert "not found" in err_cur_only[4]["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={"crawl_run_id": 1}), patch(
+ "website_profiling.db.crawl_store.read_edges",
+ return_value=[("https://ex.com/b", "https://ex.com/a")],
+ ):
+ links = crawl_mod.get_internal_links(conn, ctx, {"url": "https://ex.com/a"})
+ assert links["inlink_count"] == 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=None):
+ assert crawl_mod.search_pages(conn, ctx, {})["total"] == 0
+
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert rex_mod.get_audit_recommendations(conn, ctx, {})["count"] == 1
+ assert rex_mod.get_category_recommendations(conn, ctx, {"category_id": "nope"})["error"]
+ assert rex_mod.list_audit_categories(conn, ctx, {})["count"] == 1
+
+ kw_bad = {"rows": [{"keyword": "x", "gsc_position": "bad"}]}
+ with patch.object(Ctx, "load_keywords", return_value=kw_bad):
+ assert kw_mod.list_keywords_by_position(conn, ctx, {"min_position": 1})["total"] == 0
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x", "recommended_action": "x"}]}):
+ assert kw_mod.list_keywords_by_action(conn, ctx, {"recommended_action": "missing"})["total"] == 0
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x", "gsc_impressions": "12.5"}]}):
+ assert kw_mod.list_keywords_by_impressions(conn, ctx, {"min_impressions": 12})["total"] == 1
+
+ with patch.object(Ctx, "load_payload", return_value={}):
+ assert idx_mod.list_indexation_gaps(conn, ctx, {"gap_type": "sitemap_only"})["error"]
+ assert idx_mod.get_indexation_url_join(conn, ctx, {})["error"]
+
+ ops_row = MagicMock()
+ ops_row.keys = MagicMock(return_value=["schedule_cron", "alert_webhook_url", "alert_email"])
+ ops_conn3 = MagicMock()
+ ops_conn3.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value=ops_row)))
+ with patch("website_profiling.tools.audit_tools.ops._row_field", side_effect=lambda row, key, index=0: (
+ None, "https://hooks.example/alerts", None
+ )[index]):
+ ops = ops_mod.get_property_ops(ops_conn3, ctx, {})
+ assert ops["has_alert_webhook"] is True
+ assert ops["has_schedule"] is False
+
+ with patch("website_profiling.tools.audit_tools.ops.get_property_by_id", return_value={"google_refresh_token": "t"}), patch(
+ "website_profiling.integrations.google.gsc_links_store.read_gsc_links_status", side_effect=RuntimeError("db"),
+ ), patch.object(Ctx, "load_google", return_value={}):
+ status = ops_mod.get_google_integration_status(conn, ctx, {})
+ assert status["gsc_links"] is None
+
+ with patch.object(Ctx, "load_payload", return_value={"categories": ["bad"]}):
+ assert rex_mod.list_issues_with_ai_fixes(conn, ctx, {})["total"] == 0
+
+ df_no_col = pd.DataFrame([{"url": "https://ex.com/z", "status": "200"}])
+ with patch.object(Ctx, "load_crawl_df", return_value=df_no_col):
+ assert onpage_mod.list_pages_noindex(conn, ctx, {})["note"]
+
+ with patch.object(Ctx, "load_payload", return_value={"issues": {"seo": "bad"}}):
+ assert onpage_mod.list_seo_onpage_issues(conn, ctx, {})["total"] == 0
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df), patch.object(Ctx, "load_payload", return_value=payload):
+ assert crawl_mod.get_page_analysis(conn, ctx, {"url": "https://ex.com/missing"})["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={"recommendations": "x"}):
+ assert rex_mod.get_audit_recommendations(conn, ctx, {})["count"] == 0
+ with patch.object(Ctx, "load_payload", return_value={"ml_errors": None}):
+ assert rex_mod.get_ml_errors(conn, ctx, {})["count"] == 0
+ with patch.object(Ctx, "load_payload", return_value={"categories": [{"id": "x", "recommendations": "x"}]}):
+ assert rex_mod.get_category_recommendations(conn, ctx, {"category_id": "x"})["recommendations"] == []
+
+ log_conn = MagicMock()
+ log_conn.execute = MagicMock(return_value=MagicMock(
+ fetchone=MagicMock(return_value={"filename": "a.log", "line_count": 1, "analysis": "not-json", "uploaded_at": datetime.now(timezone.utc)}),
+ ))
+ assert ops_mod.get_latest_log_analysis(log_conn, ctx, {})["analysis"] == {}
+
+ with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": {"lists": {}, "counts": {}}}):
+ assert idx_mod.get_indexation_url_join(conn, ctx, {})["missing"]
+
+ with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": {"lists": {"sitemap_only": "bad"}, "lists_total": {}}}):
+ assert idx_mod.list_indexation_gaps(conn, ctx, {"gap_type": "sitemap_only"})["total"] == 0
+
+ with patch.object(Ctx, "load_gsc_links", return_value={"third_party_overlays": "bad"}):
+ assert bl_mod.get_third_party_links_overlay(conn, ctx, {})["count"] == 0
+
+ bl_conn2 = MagicMock()
+ bl_conn2.execute = MagicMock(return_value=MagicMock(fetchall=MagicMock(return_value=[])))
+ assert bl_mod.get_backlinks_velocity(bl_conn2, ctx, {})["count"] == 0
+
+ with patch.object(Ctx, "load_google", return_value={"ga4": {"top_pages": []}, "fetched_at": "t"}), patch(
+ "website_profiling.tools.audit_tools.google.slice_from_google_row", return_value={"ga4": {"sessions": 3}},
+ ):
+ assert google_mod.get_ga4_page_metrics(conn, ctx, {"path": "/x"})["metrics"]["sessions"] == 3
+
+ df_err = pd.DataFrame([{"url": "https://ex.com/e", "status": "200", "page_analysis": json.dumps({"console_errors": "single"})}])
+ with patch.object(Ctx, "load_crawl_df", return_value=df_err):
+ assert crawl_mod.list_pages_with_console_errors(conn, ctx, {})["total"] == 1
+
+ fake_h3 = FakeConn()
+ fake_h3.set_next_cursor(FakeCursor(fetchall_value=[(80, "bad-json", "also-bad", datetime.now(timezone.utc), 1)]))
+ assert health_mod.get_health_history(fake_h3, Ctx(property_id=1), {})["count"] == 1
+
+
+def test_all_tools_empty_payload() -> None:
+ """Hit 'no report found' branches across payload-backed tools."""
+ conn = MagicMock()
+ ctx = Ctx(property_id=1)
+ empty_tools = [
+ "get_executive_summary", "get_report_meta", "get_site_level", "list_redirects",
+ "list_broken_links", "get_status_code_breakdown", "get_response_time_stats",
+ "get_depth_distribution", "get_crawl_segments", "get_browser_diagnostics_summary",
+ "get_seo_health", "list_orphan_pages", "get_top_linked_pages", "get_outbound_link_domains",
+ "get_link_graph_summary", "get_url_fingerprints", "get_indexation_coverage",
+ "get_hreflang_summary", "get_language_summary", "get_content_analytics",
+ "get_content_duplicates", "get_social_coverage", "get_keyword_opportunities",
+ "get_ner_site_summary", "list_thin_content_pages", "get_semantic_keyword_clusters",
+ "get_competitor_link_gap", "get_bing_backlinks_summary", "get_lighthouse_diagnostics",
+ "get_crux_summary", "get_tech_stack_summary", "get_security_findings",
+ "get_category_issues",
+ "get_audit_recommendations", "get_ml_errors", "get_ssl_expiry_info",
+ "list_audit_categories", "get_category_recommendations", "list_issues_with_ai_fixes",
+ "list_seo_onpage_issues", "list_content_url_issues", "list_pages_missing_title",
+ "get_crawl_summary", "get_mime_type_breakdown", "list_indexation_gaps",
+ "get_indexation_url_join", "get_lighthouse_human_summary", "get_crawl_links_table",
+ "get_graph_edges_sample", "compare_issue_deltas",
+ ]
+ with patch.object(Ctx, "load_payload", return_value={}):
+ for name in empty_tools:
+ result = dispatch_tool(name, {"property_id": 1, "category_id": "x"}, context=ctx, conn=conn)
+ assert "error" in result, name
diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py
new file mode 100644
index 00000000..00fbd9e4
--- /dev/null
+++ b/tests/test_audit_tools_expanded.py
@@ -0,0 +1,410 @@
+"""Expanded coverage tests for all audit_tools handlers."""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from website_profiling.tools.audit_tools import AuditToolContext, dispatch_tool
+from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx
+from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, tool_handler_names
+from website_profiling.tools.audit_tools import _slice
+
+
+def _full_payload() -> dict:
+ return {
+ "site_name": "Example",
+ "report_generated_at": "2026-06-07T12:00:00Z",
+ "crawl_run_id": 9,
+ "summary": {
+ "total_urls": 10,
+ "count_2xx": 8,
+ "count_3xx": 1,
+ "count_4xx": 1,
+ "count_5xx": 0,
+ "success_rate": 80,
+ "crawl_time_s": 12.5,
+ "avg_outlinks": 3.2,
+ },
+ "seo_health": {
+ "missing_title": 1,
+ "title_ok": 9,
+ "thin_content": 2,
+ },
+ "status_counts": {"200": 8, "404": 1},
+ "redirects": [{"url": "https://ex.com/old", "status": "301", "final_url": "https://ex.com/new"}],
+ "orphan_urls": ["https://ex.com/orphan"],
+ "executive_summary": {"headline": "OK", "bullets": ["Fix titles"]},
+ "report_meta": {
+ "crawl_scope": {"render_mode": "static"},
+ "browser_diagnostics": {"summary": {"error_count": 1}},
+ },
+ "site_level": {"robots_present": True, "sitemap_present": True},
+ "indexation_coverage": {
+ "counts": {"crawled": 10, "gsc_pages": 8},
+ "lists": {"gsc_not_crawled": [], "sitemap_only": ["https://ex.com/sitemap-only"], "crawled_not_in_sitemap": []},
+ "lists_total": {"gsc_not_crawled": 0, "sitemap_only": 1, "crawled_not_in_sitemap": 0},
+ "url_join": [{"url": "https://ex.com/", "in_crawl": True, "in_gsc": True}],
+ },
+ "hreflang_summary": {"pages_with_hreflang": 2},
+ "language_summary": {"en": 8},
+ "content_analytics": {
+ "word_count_stats": {"mean": 400, "median": 350},
+ "thin_pages": [{"url": "https://ex.com/thin", "word_count": 50}],
+ "top_keywords_site": [{"word": "widgets"}],
+ },
+ "content_duplicates": [{"id": "d1", "representative_url": "https://ex.com/a", "member_count": 2}],
+ "social_coverage": {"og_coverage_pct": 90, "twitter_coverage_pct": 70},
+ "keyword_opportunities": {"hints": []},
+ "ner_site_summary": {"entities": ["Acme"]},
+ "semantic_keyword_clusters": [{"name": "widgets", "keywords": ["widget"]}],
+ "outbound_link_domains": [{"domain": "external.com", "count": 3}],
+ "top_pages": [{"url": "https://ex.com/", "inlinks": 5}],
+ "graph_nodes": [1, 2],
+ "graph_edges": [[1, 2]],
+ "url_fingerprints": [{"pattern": "/blog/*", "count": 3}],
+ "response_time_stats": {"p50": 120, "p95": 400},
+ "depth_distribution": {"0": 1, "1": 9},
+ "crawl_segments": [{"prefix": "/blog", "urls": 4}],
+ "security_findings": [{"url": "https://ex.com", "severity": "high", "finding_type": "hsts", "message": "Missing HSTS"}],
+ "tech_stack_summary": {"technologies": [{"name": "WordPress", "count": 10}]},
+ "competitor_link_gap": {"gaps": [{"domain": "rival.com"}]},
+ "bing_backlinks": {"ok": True, "total": 100},
+ "crux_summary": {"ok": True, "lcp_p75": 2.1},
+ "gsc_links": {"imported_at": "2026-06-01", "top_linking_sites": []},
+ "lighthouse_summary": {"performance": 72},
+ "lighthouse_human_summary": "OK",
+ "lighthouse_diagnostics": [{"id": "render-blocking"}],
+ "lighthouse_by_url": {
+ "https://ex.com/slow": {"performance": 40},
+ "https://ex.com/ok": {"performance": 90},
+ },
+ "google": {
+ "fetched_at": "2026-06-07",
+ "gsc": {
+ "summary": {"clicks": 1, "impressions": 10},
+ "top_queries": [{"query": "widgets", "clicks": 1}],
+ "top_pages": [{"page": "https://ex.com/", "clicks": 1}],
+ "pages": [{"page": "https://ex.com/", "clicks": 1}],
+ },
+ "ga4": {"summary": {"sessions": 5}, "top_pages": [{"path": "/"}]},
+ },
+ "keywords": {
+ "fetched_at": "2026-06-07",
+ "total_keywords": 2,
+ "rows": [{"keyword": "widgets", "score": 0.5, "gsc_clicks": 3}],
+ "striking_distance": [{"keyword": "repair"}],
+ "cannibalisation": [{"query": "widgets", "urls": ["https://ex.com/a", "https://ex.com/b"]}],
+ "query_page_misalignment": [{"keyword": "buy widgets", "url": "https://ex.com/wrong"}],
+ },
+ "recommendations": ["Fix broken links", "Add titles"],
+ "ml_errors": [],
+ "site_ssl_expires_at": "2027-01-01T00:00:00Z",
+ "content_urls": {
+ "missing_title": [{"url": "https://ex.com/notitle", "title": ""}],
+ "missing_h1": [],
+ "multiple_h1": [],
+ "missing_meta_desc": [],
+ "meta_desc_short": [],
+ "meta_desc_long": [],
+ "thin_content": [{"url": "https://ex.com/thin", "content_length": 100}],
+ },
+ "issues": {
+ "broken": [{"url": "https://ex.com/broken", "status": "404"}],
+ "seo": [{"type": "missing_title", "url": "https://ex.com/notitle", "message": "Missing title"}],
+ },
+ "mime_labels": ["text/html"],
+ "mime_values": [9],
+ "title_labels": ["0-30"],
+ "title_counts": [1],
+ "domain_labels": ["ex.com"],
+ "domain_values": [10],
+ "outlink_labels": ["0-5"],
+ "outlink_counts": [8],
+ "links": [
+ {"url": "https://ex.com/", "inlinks": 5, "outlinks": 3, "word_count": 400},
+ {"url": "https://ex.com/new", "inlinks": 1, "outlinks": 2, "word_count": 200},
+ ],
+ "categories": [
+ {
+ "id": "technical_seo",
+ "name": "Technical SEO",
+ "score": 80,
+ "issues": [
+ {"priority": "Critical", "message": "Missing title", "url": "https://ex.com/a", "recommendation": "Add title"},
+ {"priority": "High", "message": "Slow page", "url": "https://ex.com/blog/slow", "recommendation": "Optimize"},
+ ],
+ },
+ {
+ "id": "link_health",
+ "name": "Links",
+ "score": 70,
+ "issues": [{"priority": "High", "message": "Broken link", "url": "https://ex.com/404", "recommendation": "Fix"}],
+ },
+ ],
+ }
+
+
+@pytest.fixture
+def ctx() -> AuditToolContext:
+ return AuditToolContext(property_id=1, report_id=1)
+
+
+@pytest.fixture
+def conn() -> MagicMock:
+ return MagicMock()
+
+
+def test_handler_schema_parity() -> None:
+ names = {t["name"] for t in TOOL_DEFINITIONS}
+ assert names == tool_handler_names()
+ assert len(TOOL_DEFINITIONS) == 123
+
+
+def test_slice_helpers() -> None:
+ assert _slice.parse_limit("bad", 10, 20) == 10
+ assert _slice.parse_limit(100, 10, 20) == 20
+ capped = _slice.cap_list([1, 2, 3], 2)
+ assert capped["truncated"] is True
+ field = _slice.payload_field({"items": [1, 2]}, "items", 1)
+ assert field["total"] == 2
+ assert _slice.payload_field({"x": "y"}, "missing")["missing"] is True
+ assert _slice.payload_dict_slice({"meta": {"a": 1}}, "meta")["missing"] is False
+
+
+def test_crawl_filter_schema(conn: MagicMock, ctx: AuditToolContext) -> None:
+ df = pd.DataFrame([
+ {
+ "url": "https://ex.com/a",
+ "status": "200",
+ "has_schema": "true",
+ "page_analysis": json.dumps({"json_ld_types": ["Organization"]}),
+ },
+ {"url": "https://ex.com/b", "status": "200", "has_schema": "false", "page_analysis": "{}"},
+ ])
+ with patch.object(Ctx, "load_payload", return_value=_full_payload()), patch.object(Ctx, "load_crawl_df", return_value=df):
+ cov = dispatch_tool("get_schema_coverage", {}, context=ctx, conn=conn)
+ assert cov["with_schema"] == 1
+ no_schema = dispatch_tool("list_pages_without_schema", {}, context=ctx, conn=conn)
+ assert no_schema["total"] == 1
+ by_type = dispatch_tool("search_pages_by_schema_type", {"schema_type": "Organization"}, context=ctx, conn=conn)
+ assert by_type["total"] == 1
+
+
+def test_all_payload_tools(conn: MagicMock, ctx: AuditToolContext) -> None:
+ payload = _full_payload()
+ gsc_links = {
+ "imported_at": "2026-06-01",
+ "top_linking_sites": [{"domain": "ref.com"}],
+ "sample_links": [{"source_url": "https://ref.com", "target_url": "https://ex.com/"}],
+ "latest_links": [],
+ "third_party_overlays": [{"provider": "moz", "referring_domains": 10}],
+ }
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(
+ Ctx, "load_keywords", return_value=payload["keywords"],
+ ), patch.object(Ctx, "load_google", return_value=payload["google"]), patch.object(
+ Ctx, "load_gsc_links", return_value=gsc_links,
+ ), patch(
+ "website_profiling.tools.audit_tools.backlinks.read_gsc_links_status",
+ return_value={"hasData": True},
+ ), patch(
+ "website_profiling.tools.audit_tools.keywords.read_keyword_history",
+ return_value=[{"fetched_at": "2026-06-07", "position": 5}],
+ ), patch(
+ "website_profiling.tools.audit_tools.ops.check_all_alerts",
+ return_value=[{"type": "health_drop"}],
+ ), patch(
+ "website_profiling.tools.audit_tools.crawl.slice_from_google_row",
+ return_value={"queries": []},
+ ), patch(
+ "website_profiling.tools.audit_tools.ops.get_property_by_id",
+ return_value={"id": 1, "google_refresh_token": "tok", "gsc_site_url": "sc-domain:ex.com"},
+ ), patch.object(
+ conn, "execute",
+ return_value=MagicMock(fetchone=MagicMock(return_value={"schedule_cron": "0 9 * * 1", "alert_webhook_url": None, "alert_email": None}), fetchall=MagicMock(return_value=[])),
+ ):
+ tools = [
+ ("get_executive_summary", {}),
+ ("get_report_meta", {}),
+ ("get_site_level", {}),
+ ("list_redirects", {}),
+ ("list_broken_links", {}),
+ ("get_status_code_breakdown", {}),
+ ("get_response_time_stats", {}),
+ ("get_depth_distribution", {}),
+ ("get_crawl_segments", {}),
+ ("get_browser_diagnostics_summary", {}),
+ ("get_seo_health", {}),
+ ("list_orphan_pages", {}),
+ ("get_top_linked_pages", {}),
+ ("get_outbound_link_domains", {}),
+ ("get_link_graph_summary", {}),
+ ("get_url_fingerprints", {}),
+ ("get_indexation_coverage", {}),
+ ("get_hreflang_summary", {}),
+ ("get_language_summary", {}),
+ ("get_content_analytics", {}),
+ ("get_content_duplicates", {}),
+ ("get_social_coverage", {}),
+ ("get_keyword_opportunities", {}),
+ ("get_ner_site_summary", {}),
+ ("list_thin_content_pages", {}),
+ ("get_striking_distance_keywords", {"property_id": 1}),
+ ("get_keyword_cannibalisation", {"property_id": 1}),
+ ("get_query_page_misalignment", {"property_id": 1}),
+ ("get_semantic_keyword_clusters", {}),
+ ("get_keyword_history", {"property_id": 1, "keyword": "widgets"}),
+ ("get_gsc_top_queries", {}),
+ ("get_gsc_top_pages", {}),
+ ("get_ga4_summary", {}),
+ ("get_gsc_page_query_slice", {"url": "https://ex.com/"}),
+ ("get_gsc_links_summary", {"property_id": 1}),
+ ("get_gsc_links_import_status", {"property_id": 1}),
+ ("get_competitor_link_gap", {}),
+ ("get_bing_backlinks_summary", {}),
+ ("get_lighthouse_diagnostics", {}),
+ ("get_crux_summary", {}),
+ ("list_slow_pages", {}),
+ ("get_integration_alerts", {"property_id": 1}),
+ ("get_tech_stack_summary", {}),
+ ("get_security_findings", {}),
+ ("list_issues_by_category", {"category_id": "technical_seo"}),
+ ("get_category_issues", {"category_id": "technical_seo"}),
+ ("get_audit_recommendations", {}),
+ ("get_ml_errors", {}),
+ ("get_ssl_expiry_info", {}),
+ ("list_audit_categories", {}),
+ ("get_category_recommendations", {"category_id": "technical_seo"}),
+ ("list_issues_with_ai_fixes", {}),
+ ("list_seo_onpage_issues", {}),
+ ("list_content_url_issues", {"bucket": "missing_title"}),
+ ("list_pages_missing_title", {}),
+ ("list_pages_noindex", {}),
+ ("get_crawl_summary", {}),
+ ("get_mime_type_breakdown", {}),
+ ("get_title_length_distribution", {}),
+ ("get_top_crawled_pages", {}),
+ ("list_indexation_gaps", {"gap_type": "sitemap_only"}),
+ ("get_indexation_url_join", {}),
+ ("get_gsc_sample_links", {"property_id": 1}),
+ ("get_gsc_latest_links", {"property_id": 1}),
+ ("get_third_party_links_overlay", {"property_id": 1}),
+ ("get_property_ops", {"property_id": 1}),
+ ("get_google_integration_status", {"property_id": 1}),
+ ("get_keyword_serp_overlay", {"property_id": 1}),
+ ("get_lighthouse_human_summary", {}),
+ ("list_lighthouse_poor_seo_pages", {}),
+ ("get_crawl_links_table", {}),
+ ("get_graph_edges_sample", {}),
+ ]
+ for name, args in tools:
+ result = dispatch_tool(name, args, context=ctx, conn=conn)
+ assert "error" not in result or result.get("missing"), f"{name} failed: {result}"
+
+
+def test_empty_and_error_paths(conn: MagicMock, ctx: AuditToolContext) -> None:
+ with patch.object(Ctx, "load_payload", return_value={}):
+ for name in (
+ "get_executive_summary",
+ "get_indexation_coverage",
+ "get_crawl_segments",
+ "get_competitor_link_gap",
+ "get_crux_summary",
+ "list_orphan_pages",
+ ):
+ r = dispatch_tool(name, {}, context=ctx, conn=conn)
+ assert "error" in r
+
+ with patch.object(Ctx, "load_payload", return_value=_full_payload()):
+ assert dispatch_tool("list_issues_by_category", {}, context=ctx, conn=conn)["error"]
+ assert dispatch_tool("get_category_issues", {}, context=ctx, conn=conn)["error"]
+ assert dispatch_tool("get_category_issues", {"category_id": "nope"}, context=ctx, conn=conn)["error"]
+ assert dispatch_tool("search_pages_by_schema_type", {}, context=ctx, conn=conn)["error"]
+ assert dispatch_tool("get_keyword_history", {"property_id": 1}, context=ctx, conn=conn)["error"]
+ no_prop_ctx = AuditToolContext(property_id=None)
+ assert dispatch_tool("get_gsc_links_summary", {}, context=no_prop_ctx, conn=conn)["error"]
+ assert dispatch_tool("get_integration_alerts", {}, context=AuditToolContext(property_id=None), conn=conn)["error"]
+
+ with patch.object(Ctx, "load_google", return_value={"gsc": {}, "ga4": {}}):
+ assert dispatch_tool("get_ga4_summary", {}, context=ctx, conn=conn).get("missing") or dispatch_tool(
+ "get_ga4_summary", {}, context=ctx, conn=conn,
+ ).get("error")
+
+ df = pd.DataFrame()
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert dispatch_tool("get_schema_coverage", {}, context=ctx, conn=conn)["error"]
+
+
+def test_context_loaders(conn: MagicMock) -> None:
+ ctx = Ctx(property_id=1, report_id=2)
+ with patch("website_profiling.tools.audit_tools.context.read_latest_gsc_links_data", return_value={"x": 1}):
+ assert ctx.load_gsc_links(conn)["x"] == 1
+ with patch("website_profiling.tools.audit_tools.context.read_latest_gsc_links_data", return_value=None), patch.object(
+ Ctx, "load_payload", return_value={"gsc_links": {"y": 2}},
+ ):
+ assert ctx.load_gsc_links(conn)["y"] == 2
+ with patch("website_profiling.tools.audit_tools.context.read_report_payload", return_value={"a": 1}):
+ assert ctx.load_report_payload_by_id(conn, 5)["a"] == 1
+ with patch("website_profiling.tools.audit_tools.context.get_property_by_id", return_value={"canonical_domain": "ex.com"}):
+ assert ctx.resolve_property_domain(conn) == "ex.com"
+ merged_bad = ctx.with_args({"property_id": "x", "report_id": "y"})
+ assert merged_bad.property_id == 1
+
+
+def test_list_report_history_and_workflow(conn: MagicMock, ctx: AuditToolContext) -> None:
+ from tests.db_test_fakes import FakeConn, FakeCursor
+
+ now = datetime.now(timezone.utc)
+ fake = FakeConn()
+ fake.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[{
+ "id": 10,
+ "site_name": "Ex",
+ "canonical_domain": "ex.com",
+ "generated_at": now,
+ }],
+ ),
+ )
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"):
+ hist = dispatch_tool("list_report_history", {"property_id": 1}, conn=fake)
+ assert hist["count"] == 1
+
+ fake2 = FakeConn()
+ fake2.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[{
+ "issue_key": "k1",
+ "url": "https://ex.com",
+ "category": "Tech",
+ "priority": "High",
+ "message": "msg",
+ "status": "open",
+ "assignee": None,
+ "note": None,
+ "updated_at": now,
+ }],
+ ),
+ )
+ wf = dispatch_tool("list_issue_workflow", {"property_id": 1}, conn=fake2)
+ assert wf["count"] == 1
+
+
+def test_compare_reports(conn: MagicMock, ctx: AuditToolContext) -> None:
+ current = _full_payload()
+ baseline = {**current, "summary": {**current["summary"], "total_urls": 8}}
+ with patch("website_profiling.tools.audit_tools.compare.read_report_payload", side_effect=[current, baseline]):
+ result = dispatch_tool("compare_reports", {"baseline_report_id": 1}, context=ctx, conn=conn)
+ assert "health_score" in result
+ with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", side_effect=[current, baseline]):
+ diff = dispatch_tool("compare_url_set_diff", {"baseline_report_id": 1}, context=ctx, conn=conn)
+ assert "new_count" in diff
+ assert dispatch_tool("compare_reports", {}, context=ctx, conn=conn)["error"]
+ with patch(
+ "website_profiling.tools.audit_tools.compare.read_report_payload",
+ side_effect=[None, baseline],
+ ):
+ assert "not found" in dispatch_tool("compare_reports", {"baseline_report_id": 1}, context=ctx, conn=conn)["error"]
diff --git a/tests/test_chat_agent.py b/tests/test_chat_agent.py
new file mode 100644
index 00000000..a15c15e4
--- /dev/null
+++ b/tests/test_chat_agent.py
@@ -0,0 +1,90 @@
+"""Tests for chat agent loop."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from website_profiling.llm.agent import MAX_TOOL_ROUNDS, run_agent_turn
+from website_profiling.llm.base import ChatResult, ToolCall
+from website_profiling.tools.audit_tools import AuditToolContext
+
+
+class FakeToolClient:
+ def __init__(self, steps: list[ChatResult]) -> None:
+ self._steps = list(steps)
+ self._calls = 0
+
+ def chat_with_tools(self, messages, tools, *, on_token=None):
+ result = self._steps[min(self._calls, len(self._steps) - 1)]
+ self._calls += 1
+ if on_token and result.content:
+ on_token(result.content)
+ return result
+
+
+def test_agent_tool_then_answer() -> None:
+ client = FakeToolClient([
+ ChatResult(tool_calls=[ToolCall(id="tc1", name="list_issues", arguments={"limit": 5})]),
+ ChatResult(content="Found 3 critical issues."),
+ ])
+ events: list[dict] = []
+ ctx = AuditToolContext(property_id=1)
+
+ with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={
+ "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test",
+ }):
+ with patch("website_profiling.llm.agent.get_llm_client", return_value=client):
+ with patch(
+ "website_profiling.llm.agent.dispatch_tool",
+ return_value={"issues": [], "total": 0},
+ ) as mock_dispatch:
+ result = run_agent_turn(
+ [{"role": "user", "content": "What are the top issues?"}],
+ ctx,
+ on_event=events.append,
+ )
+
+ assert result["ok"] is True
+ assert "critical" in result["message"].lower()
+ mock_dispatch.assert_called_once()
+ types = [e["type"] for e in events]
+ assert "tool_start" in types
+ assert "tool_end" in types
+ assert "done" in types
+
+
+def test_agent_disabled_llm() -> None:
+ events: list[dict] = []
+ with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={
+ "llm_enabled": False, "llm_provider": "none",
+ }):
+ result = run_agent_turn(
+ [{"role": "user", "content": "Hi"}],
+ AuditToolContext(),
+ on_event=events.append,
+ )
+ assert result["ok"] is False
+ assert events[-1]["type"] == "error"
+
+
+def test_max_tool_rounds() -> None:
+ always_tool = ChatResult(
+ tool_calls=[ToolCall(id="x", name="list_properties", arguments={})],
+ )
+ client = FakeToolClient([always_tool] * (MAX_TOOL_ROUNDS + 1))
+ ctx = AuditToolContext()
+
+ with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={
+ "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test",
+ }):
+ with patch("website_profiling.llm.agent.get_llm_client", return_value=client):
+ with patch(
+ "website_profiling.llm.agent.dispatch_tool",
+ return_value={"properties": []},
+ ):
+ result = run_agent_turn(
+ [{"role": "user", "content": "List properties"}],
+ ctx,
+ )
+
+ assert result["ok"] is False
+ assert "maximum tool rounds" in result["error"].lower()
diff --git a/tests/test_chat_cmd.py b/tests/test_chat_cmd.py
new file mode 100644
index 00000000..50f5a5c7
--- /dev/null
+++ b/tests/test_chat_cmd.py
@@ -0,0 +1,99 @@
+"""CLI chat command tests."""
+from __future__ import annotations
+
+import argparse
+import io
+import json
+from unittest.mock import patch
+
+import pytest
+
+from website_profiling.commands import chat_cmd
+
+
+def test_chat_cmd_requires_stdin_json() -> None:
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=False))
+ assert exc.value.code == 1
+
+
+def test_chat_cmd_invalid_stdin_json(capsys) -> None:
+ with patch("sys.stdin", io.StringIO("not-json")):
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 1
+ assert "error" in capsys.readouterr().out
+
+
+def test_chat_cmd_success(capsys) -> None:
+ payload = json.dumps({"messages": [{"role": "user", "content": "Hi"}], "property_id": 1})
+ with patch("sys.stdin", io.StringIO(payload)):
+ with patch(
+ "website_profiling.commands.chat_cmd.run_agent_turn",
+ return_value={"ok": True, "message": "Done"},
+ ) as mock_turn:
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 0
+ mock_turn.assert_called_once()
+ assert mock_turn.call_args[0][1].property_id == 1
+
+
+def test_chat_cmd_streams_sanitized_events(capsys) -> None:
+ payload = json.dumps({"messages": [{"role": "user", "content": "Hi"}]})
+
+ def fake_turn(_messages, _ctx, on_event=None):
+ if on_event:
+ on_event({"type": "token", "content": "bad\udc9d"})
+ return {"ok": True}
+
+ with patch("sys.stdin", io.StringIO(payload)):
+ with patch("website_profiling.commands.chat_cmd.run_agent_turn", side_effect=fake_turn):
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 0
+ out = capsys.readouterr().out
+ assert "\udc9d" not in out
+ assert "token" in out
+
+
+def test_chat_cmd_coerces_invalid_ids_and_messages(capsys) -> None:
+ payload = json.dumps({"messages": "bad", "property_id": "x", "report_id": "y"})
+ with patch("sys.stdin", io.StringIO(payload)):
+ with patch(
+ "website_profiling.commands.chat_cmd.run_agent_turn",
+ return_value={"ok": True},
+ ) as mock_turn:
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 0
+ ctx = mock_turn.call_args[0][1]
+ assert ctx.property_id is None
+ assert ctx.report_id is None
+ assert mock_turn.call_args[0][0] == []
+
+
+def test_chat_cmd_agent_failure(capsys) -> None:
+ payload = json.dumps({"messages": []})
+ with patch("sys.stdin", io.StringIO(payload)):
+ with patch(
+ "website_profiling.commands.chat_cmd.run_agent_turn",
+ return_value={"ok": False, "error": "LLM disabled"},
+ ):
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 1
+ assert "LLM disabled" in capsys.readouterr().out
+
+
+def test_chat_cmd_exception(capsys) -> None:
+ payload = json.dumps({"messages": []})
+ with patch("sys.stdin", io.StringIO(payload)):
+ with patch(
+ "website_profiling.commands.chat_cmd.run_agent_turn",
+ side_effect=RuntimeError("boom"),
+ ):
+ with pytest.raises(SystemExit) as exc:
+ chat_cmd.run({}, argparse.Namespace(stdin_json=True))
+ assert exc.value.code == 1
+ assert "boom" in capsys.readouterr().out
diff --git a/tests/test_chat_store.py b/tests/test_chat_store.py
new file mode 100644
index 00000000..cf37b7a7
--- /dev/null
+++ b/tests/test_chat_store.py
@@ -0,0 +1,157 @@
+"""Unit tests for chat_store."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from tests.db_test_fakes import FakeConn, FakeCursor
+
+from website_profiling.db.chat_store import (
+ append_message,
+ create_session,
+ delete_session,
+ get_messages,
+ get_session,
+ list_sessions,
+ update_session_title,
+)
+
+
+def test_create_session_returns_id() -> None:
+ conn = FakeConn()
+ conn.set_next_cursor(FakeCursor(fetchone_value={"id": 42}))
+ sid = create_session(conn, 7, "Test chat")
+ assert sid == 42
+ assert conn.commits == 1
+
+
+def test_list_sessions() -> None:
+ conn = FakeConn()
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[
+ {"id": 1, "property_id": 7, "title": "Chat A", "created_at": now, "updated_at": now},
+ ],
+ ),
+ )
+ rows = list_sessions(conn, 7)
+ assert len(rows) == 1
+ assert rows[0]["title"] == "Chat A"
+ assert rows[0]["property_id"] == 7
+
+
+def test_get_session_found() -> None:
+ conn = FakeConn()
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor(
+ FakeCursor(
+ fetchone_value={
+ "id": 5,
+ "property_id": 7,
+ "title": "Found",
+ "created_at": now,
+ "updated_at": now,
+ },
+ ),
+ )
+ row = get_session(conn, 5)
+ assert row is not None
+ assert row["title"] == "Found"
+
+
+def test_get_session_missing() -> None:
+ conn = FakeConn()
+ conn.set_next_cursor(FakeCursor(fetchone_value=None))
+ assert get_session(conn, 99) is None
+
+
+def test_get_messages_parses_json_fields() -> None:
+ conn = FakeConn()
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[
+ {
+ "id": 1,
+ "role": "tool",
+ "content": "",
+ "tool_name": "list_issues",
+ "tool_args": '{"limit": 5}',
+ "tool_result": "not-json",
+ "created_at": now,
+ },
+ ],
+ ),
+ )
+ msgs = get_messages(conn, 5)
+ assert msgs[0]["tool_args"] == {"limit": 5}
+ assert msgs[0]["tool_result"] == "not-json"
+
+
+def test_get_messages_keeps_invalid_tool_args_json() -> None:
+ conn = FakeConn()
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[
+ {
+ "id": 2,
+ "role": "tool",
+ "content": "",
+ "tool_name": "list_issues",
+ "tool_args": "not-json",
+ "tool_result": None,
+ "created_at": now,
+ },
+ ],
+ ),
+ )
+ msgs = get_messages(conn, 5)
+ assert msgs[0]["tool_args"] == "not-json"
+
+
+def test_get_messages() -> None:
+ conn = FakeConn()
+ now = datetime.now(timezone.utc)
+ conn.set_next_cursor(
+ FakeCursor(
+ fetchall_value=[
+ {
+ "id": 1,
+ "role": "user",
+ "content": "Hello",
+ "tool_name": None,
+ "tool_args": None,
+ "tool_result": None,
+ "created_at": now,
+ },
+ ],
+ ),
+ )
+ msgs = get_messages(conn, 5)
+ assert msgs[0]["content"] == "Hello"
+ assert msgs[0]["role"] == "user"
+
+
+def test_append_message_and_touch() -> None:
+ conn = FakeConn()
+ conn.set_next_cursor(FakeCursor(fetchone_value={"id": 10}))
+ mid = append_message(conn, 3, "user", "Hi there")
+ assert mid == 10
+ assert conn.commits == 1
+ touch_sql = [sql for sql, _ in conn.executed if "UPDATE chat_sessions" in sql]
+ assert touch_sql
+
+
+def test_update_session_title() -> None:
+ conn = FakeConn()
+ update_session_title(conn, 3, "Renamed")
+ assert conn.commits == 1
+
+
+def test_delete_session() -> None:
+ conn = FakeConn()
+ conn.set_next_cursor(FakeCursor(fetchone_value={"id": 3}))
+ assert delete_session(conn, 3) is True
+ conn.set_next_cursor(FakeCursor(fetchone_value=None))
+ assert delete_session(conn, 99) is False
diff --git a/tests/test_compare_payload.py b/tests/test_compare_payload.py
new file mode 100644
index 00000000..0d6723a9
--- /dev/null
+++ b/tests/test_compare_payload.py
@@ -0,0 +1,278 @@
+"""Tests for reporting/compare_payload.py — parity with web compare."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from website_profiling.reporting.compare_payload import (
+ build_category_scores,
+ build_content_metrics,
+ build_duplicate_deltas,
+ build_full_compare,
+ build_google_metrics,
+ build_issue_deltas,
+ build_lighthouse_url_deltas,
+ build_link_metric_deltas,
+ build_priority_counts,
+ build_redirect_deltas,
+ build_security_deltas,
+ build_seo_health_deltas,
+ build_tech_deltas,
+ build_url_set_diff,
+ norm_report_url,
+)
+
+
+def _payload(**overrides) -> dict:
+ base = {
+ "report_generated_at": "2026-06-01",
+ "summary": {"total_urls": 10, "count_2xx": 9, "count_4xx": 1, "success_rate": 90},
+ "seo_health": {"missing_title": 1, "title_ok": 9, "thin_content": 2},
+ "categories": [
+ {"id": "tech", "name": "Technical", "score": 80, "issues": [
+ {"priority": "High", "message": "Slow", "url": "https://ex.com/slow"},
+ ]},
+ ],
+ "redirects": [{"url": "https://ex.com/old", "status": "301", "final_url": "https://ex.com/new"}],
+ "security_findings": [{"url": "https://ex.com", "severity": "high", "finding_type": "hsts", "message": "x"}],
+ "content_duplicates": [{"id": "d1", "representative_url": "https://ex.com/a", "member_count": 2}],
+ "tech_stack_summary": {"technologies": [{"name": "WP", "count": 5}]},
+ "lighthouse_by_url": {
+ "https://ex.com/slow": {"performance": 40, "median_metrics": {"performance_score": 40, "seo_score": 80}},
+ },
+ "links": [
+ {"url": "https://ex.com/slow", "status": "200", "inlinks": 2, "outlinks": 3, "word_count": 100, "response_time_ms": 200},
+ ],
+ "google": {"gsc": {"summary": {"clicks": 10, "impressions": 100}}, "ga4": {"summary": {"sessions": 5}}},
+ "content_analytics": {"word_count_stats": {"mean": 300}},
+ "social_coverage": {"og_coverage_pct": 80},
+ "response_time_stats": {"p50": 100},
+ }
+ base.update(overrides)
+ return base
+
+
+def test_norm_report_url() -> None:
+ assert norm_report_url("https://Ex.COM/page/") == "ex.com/page"
+ assert norm_report_url("") == ""
+ assert norm_report_url(" ") == ""
+ assert norm_report_url("relative/path/") == "relative/path"
+ with patch("website_profiling.reporting.compare_payload.urlparse", side_effect=ValueError("bad")):
+ assert norm_report_url("https://ex.com/x") == "https://ex.com/x"
+
+
+def test_issue_and_priority_deltas() -> None:
+ cur = _payload()
+ base = _payload(categories=[
+ {"id": "tech", "name": "Technical", "score": 85, "issues": []},
+ ])
+ issues = build_issue_deltas(cur, base)
+ assert any(i["kind"] == "new" for i in issues)
+ counts = build_priority_counts(cur, base)
+ assert counts[0]["priority"] == "Critical"
+
+
+def test_lighthouse_redirect_security_dup_tech() -> None:
+ cur = _payload()
+ base = _payload(
+ lighthouse_by_url={"https://ex.com/slow": {"performance": 90, "median_metrics": {"performance_score": 90}}},
+ redirects=[],
+ security_findings=[],
+ content_duplicates=[],
+ tech_stack_summary={"technologies": []},
+ )
+ assert build_lighthouse_url_deltas(cur, base)
+ assert build_redirect_deltas(cur, base)
+ assert build_security_deltas(cur, base)
+ assert build_duplicate_deltas(cur, base)
+ assert build_tech_deltas(cur, base)
+
+
+def test_content_google_seo_health_full_compare() -> None:
+ cur = _payload()
+ base = _payload(seo_health={"missing_title": 0, "title_ok": 10, "thin_content": 1})
+ assert build_content_metrics(cur, base)
+ g = build_google_metrics(cur, base)
+ assert g["available"] is True
+ assert build_seo_health_deltas(cur, base)
+ full = build_full_compare(cur, base, current_report_id=2, baseline_report_id=1)
+ assert full["health_score"]["delta"] is not None
+ assert "issue_deltas" in full
+
+
+def test_url_set_diff() -> None:
+ cur = {"links": [{"url": "https://ex.com/new"}, {"url": "https://ex.com/"}]}
+ base = {"links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/old"}]}
+ diff = build_url_set_diff(cur, base)
+ assert diff["new_count"] >= 1
+ assert diff["removed_count"] >= 1
+ assert diff["new_urls"][0].startswith("https://")
+ assert diff["removed_urls"][0].startswith("https://")
+ assert build_url_set_diff({"links": ["not-a-dict"]}, {}) == {
+ "new_urls": [],
+ "removed_urls": [],
+ "new_count": 0,
+ "removed_count": 0,
+ }
+
+
+def test_issue_deltas_edge_cases() -> None:
+ cur = {
+ "categories": [
+ "skip",
+ {"name": "SEO", "issues": [
+ "skip",
+ {"url": "", "message": ""},
+ {"url": "https://ex.com/a", "message": "Fix", "priority": "Critical"},
+ ]},
+ ],
+ }
+ base = {
+ "categories": [
+ {"name": "SEO", "issues": [
+ {"url": "https://ex.com/b", "message": "Gone", "priority": "Low"},
+ ]},
+ ],
+ }
+ issues = build_issue_deltas(cur, base)
+ kinds = {i["kind"] for i in issues}
+ assert "new" in kinds
+ assert "resolved" in kinds
+
+
+def test_priority_counts_skips_invalid_entries() -> None:
+ cur = {"categories": ["skip", {"issues": ["skip", {"priority": "High"}]}]}
+ base = {"categories": []}
+ counts = build_priority_counts(cur, base)
+ assert counts[1]["current"] == 1
+
+
+def test_lighthouse_from_links_and_skips() -> None:
+ cur = {
+ "lighthouse_by_url": {
+ "": {"performance": 50},
+ "https://ex.com/a": "skip",
+ },
+ "links": [
+ {"url": "https://ex.com/b", "lighthouse": {"median_metrics": {"performance_score": 70, "seo_score": 90}}},
+ "skip",
+ {"url": "https://ex.com/a", "lighthouse": {"median_metrics": {"performance_score": 80}}},
+ ],
+ }
+ base = {"lighthouse_by_url": {"https://ex.com/c": {"median_metrics": {"performance_score": 50, "seo_score": 50}}}}
+ assert build_lighthouse_url_deltas(cur, base) == []
+
+
+def test_link_metric_deltas_edge_cases() -> None:
+ cur = {"links": [{"url": "https://ex.com/a", "inlinks": 10, "outlinks": 5}]}
+ base = {
+ "links": [
+ "skip",
+ {"url": ""},
+ {"url": "https://ex.com/missing"},
+ {"url": "https://ex.com/a", "inlinks": 5, "outlinks": 5, "word_count": "x"},
+ {"url": "https://ex.com/b", "inlinks": 1, "outlinks": 1},
+ ],
+ }
+ deltas = build_link_metric_deltas(cur, base)
+ assert len(deltas) == 1
+ assert deltas[0]["metric"] == "inlinks"
+ assert build_link_metric_deltas({"links": []}, {"links": []}) == []
+
+
+def test_redirect_deltas_removed_and_skips() -> None:
+ cur = {"redirects": ["skip", {"url": "", "from": ""}]}
+ base = {"redirects": [{"url": "https://ex.com/gone", "status": "301"}]}
+ deltas = build_redirect_deltas(cur, base)
+ assert any(d["kind"] == "removed" for d in deltas)
+
+
+def test_security_deltas_resolved_and_skips() -> None:
+ cur = {"security_findings": []}
+ base = {
+ "security_findings": [
+ "skip",
+ {"url": "https://ex.com", "finding_type": "csp", "message": "missing"},
+ ],
+ }
+ deltas = build_security_deltas(cur, base)
+ assert len(deltas) == 1
+ assert deltas[0]["kind"] == "resolved"
+
+
+def test_duplicate_deltas_all_kinds() -> None:
+ cur = {
+ "content_duplicates": [
+ "skip",
+ {"representative_url": ""},
+ {"id": "new1", "representative_url": "https://ex.com/n", "member_count": 3},
+ {"id": "chg", "representative_url": "https://ex.com/c", "member_urls": ["a", "b", "c"]},
+ ],
+ }
+ base = {
+ "content_duplicates": [
+ {"id": "chg", "representative_url": "https://ex.com/c", "member_count": 2},
+ {"id": "rm", "representative_url": "https://ex.com/r", "member_count": 4},
+ ],
+ }
+ deltas = build_duplicate_deltas(cur, base)
+ kinds = {d["kind"] for d in deltas}
+ assert kinds == {"new", "changed", "removed"}
+
+
+def test_tech_deltas_removed_and_skips() -> None:
+ cur = {"tech_stack_summary": {"technologies": ["skip", {"name": "React", "count": 2}]}}
+ base = {"tech_stack_summary": {"technologies": [{"name": "jQuery", "count": 1}]}}
+ deltas = build_tech_deltas(cur, base)
+ kinds = {d["kind"] for d in deltas}
+ assert kinds == {"added", "removed"}
+
+
+def test_google_metrics_unavailable() -> None:
+ assert build_google_metrics({}, {}) == {"available": False, "metrics": []}
+
+
+def test_category_scores_skips_invalid() -> None:
+ cur = {"categories": ["skip", {"id": "", "score": 90}, {"id": "perf", "name": "Performance", "score": 75}]}
+ base = {"categories": [{"id": "perf", "score": 80}]}
+ scores = build_category_scores(cur, base)
+ assert len(scores) == 1
+ assert scores[0]["id"] == "perf"
+ assert scores[0]["delta"] == -5
+
+
+def test_full_compare_truncation() -> None:
+ many_issues = [
+ {"priority": "Low", "message": f"issue-{i}", "url": f"https://ex.com/p{i}"}
+ for i in range(105)
+ ]
+ cur = {"categories": [{"id": "x", "name": "X", "score": 50, "issues": many_issues}]}
+ base = {"categories": []}
+ full = build_full_compare(cur, base)
+ assert full["truncated_sections"].get("issue_deltas") is True
+ assert len(full["issue_deltas"]) == 100
+
+ links = []
+ for i in range(210):
+ links.append({
+ "url": f"https://ex.com/l{i}",
+ "inlinks": i + 10,
+ "outlinks": 1,
+ "word_count": 100,
+ "response_time_ms": 100,
+ })
+ cur_links = {"links": links}
+ base_links = {
+ "links": [
+ {
+ "url": f"https://ex.com/l{i}",
+ "inlinks": i,
+ "outlinks": 1,
+ "word_count": 100,
+ "response_time_ms": 100,
+ }
+ for i in range(210)
+ ],
+ }
+ full_links = build_full_compare(cur_links, base_links)
+ assert full_links["truncated_sections"].get("link_metric_deltas") is True
+ assert len(full_links["link_metric_deltas"]) == 200
diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py
new file mode 100644
index 00000000..95aada69
--- /dev/null
+++ b/tests/test_mcp_registry.py
@@ -0,0 +1,27 @@
+"""MCP tool registry validation (no live MCP process)."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool
+
+
+def test_tool_definitions_schema() -> None:
+ assert len(TOOL_DEFINITIONS) == 123
+ for tool in TOOL_DEFINITIONS:
+ assert tool.get("name")
+ assert tool.get("description")
+ schema = tool.get("inputSchema")
+ assert isinstance(schema, dict)
+ assert schema.get("type") == "object"
+
+
+def test_dispatch_list_properties_roundtrip() -> None:
+ conn = MagicMock()
+ props = [{"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}]
+ with patch(
+ "website_profiling.tools.audit_tools.properties.list_properties_public",
+ return_value=props,
+ ):
+ result = dispatch_tool("list_properties", {}, conn=conn)
+ assert result["count"] == 1
diff --git a/tests/test_mcp_resources.py b/tests/test_mcp_resources.py
new file mode 100644
index 00000000..55abd6f4
--- /dev/null
+++ b/tests/test_mcp_resources.py
@@ -0,0 +1,44 @@
+"""MCP resource URI resolution tests."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from website_profiling.mcp import server as mcp_server
+
+
+def test_resolve_properties_resource() -> None:
+ with patch("website_profiling.mcp.server.dispatch_tool", return_value={"count": 0, "properties": []}):
+ text = mcp_server._resolve_resource("audit://properties")
+ assert "properties" in text
+
+
+def test_resolve_report_latest_missing_payload() -> None:
+ with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object(
+ mcp_server.AuditToolContext, "load_payload", return_value=None,
+ ):
+ mock_db.return_value.__enter__.return_value = object()
+ text = mcp_server._resolve_resource("audit://property/1/report/latest")
+ assert "error" in text
+
+
+def test_resolve_glossary_and_tools() -> None:
+ text = mcp_server._resolve_resource("audit://tools")
+ assert "tool_count" in text
+ unknown = mcp_server._resolve_resource("audit://unknown")
+ assert "error" in unknown
+
+
+def test_resolve_property_and_report() -> None:
+ with patch("website_profiling.mcp.server.dispatch_tool", side_effect=[
+ {"property": {"id": 1}},
+ {"health_score": 80},
+ ]):
+ text = mcp_server._resolve_resource("audit://property/1")
+ assert "property" in text
+
+ with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object(
+ mcp_server.AuditToolContext, "load_payload", return_value={"summary": {}, "categories": []},
+ ):
+ mock_db.return_value.__enter__.return_value = object()
+ text = mcp_server._resolve_resource("audit://property/1/report/latest")
+ assert "summary" in text or "type" in text
diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py
new file mode 100644
index 00000000..9e19e99d
--- /dev/null
+++ b/tests/test_mcp_server_helpers.py
@@ -0,0 +1,190 @@
+"""MCP server helper and main() coverage."""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import runpy
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from website_profiling.mcp import server as mcp_server
+
+
+def test_default_property_id_env() -> None:
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "12"}):
+ assert mcp_server._default_property_id() == 12
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "0"}):
+ assert mcp_server._default_property_id() is None
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "bad"}):
+ assert mcp_server._default_property_id() is None
+ with patch.dict(os.environ, {}, clear=True):
+ assert mcp_server._default_property_id() is None
+
+
+def test_merge_context() -> None:
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "3"}):
+ ctx = mcp_server._merge_context({"property_id": 9, "report_id": 4})
+ assert ctx.property_id == 9
+ assert ctx.report_id == 4
+
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "3"}):
+ ctx = mcp_server._merge_context({"property_id": "bad", "report_id": "bad"})
+ assert ctx.property_id == 3
+ assert ctx.report_id is None
+
+
+def test_payload_index_variants() -> None:
+ index = mcp_server._payload_index({
+ "items": [1, 2],
+ "meta": {"a": 1},
+ "score": 88,
+ })
+ assert index["items"]["count"] == 2
+ assert index["meta"]["type"] == "object"
+ assert index["score"]["type"] == "int"
+
+
+def test_read_glossary_excerpt() -> None:
+ text = mcp_server._read_glossary_excerpt()
+ assert isinstance(text, str)
+ assert text
+
+
+def test_read_glossary_excerpt_missing(monkeypatch) -> None:
+ monkeypatch.setattr(Path, "is_file", lambda _self: False)
+ assert mcp_server._read_glossary_excerpt() == "Glossary file not found."
+
+
+def test_tools_catalog_json_includes_security_tools() -> None:
+ catalog = json.loads(mcp_server._tools_catalog_json())
+ assert catalog["tool_count"] >= 123
+ assert "get_security_findings" in catalog["domains"]["security"]
+
+
+def test_tools_catalog_json_backlinks_domain() -> None:
+ fake_tools = [
+ {
+ "name": "get_bing_overview",
+ "description": "Bing overview without link in name.",
+ "inputSchema": {"type": "object", "properties": {}},
+ },
+ ]
+ with patch("website_profiling.mcp.server.TOOL_DEFINITIONS", fake_tools):
+ catalog = json.loads(mcp_server._tools_catalog_json())
+ assert catalog["domains"]["backlinks"] == ["get_bing_overview"]
+
+
+def test_resolve_glossary_and_report_by_id() -> None:
+ glossary = mcp_server._resolve_resource("audit://glossary")
+ assert isinstance(glossary, str)
+
+ with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object(
+ mcp_server.AuditToolContext, "load_payload", return_value=None,
+ ):
+ mock_db.return_value.__enter__.return_value = object()
+ missing = mcp_server._resolve_resource("audit://property/1/report/99")
+ assert "error" in missing
+
+ with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object(
+ mcp_server.AuditToolContext, "load_payload", return_value={"summary": {"score": 1}, "pages": [1, 2]},
+ ):
+ mock_db.return_value.__enter__.return_value = object()
+ found = mcp_server._resolve_resource("audit://property/1/report/99")
+ payload = json.loads(found)
+ assert payload["summary"]["type"] == "object"
+
+
+def test_mcp_main_missing_sdk() -> None:
+ with patch.dict(sys.modules, {"mcp.server": None, "mcp.server.stdio": None, "mcp.types": None}):
+ with pytest.raises(SystemExit, match="MCP SDK"):
+ mcp_server.main()
+
+
+def test_mcp_main_registers_handlers(monkeypatch) -> None:
+ captured: dict[str, object] = {}
+
+ class FakeServer:
+ def __init__(self, name: str) -> None:
+ captured["name"] = name
+
+ def list_tools(self):
+ def decorator(fn):
+ captured["list_tools"] = fn
+ return fn
+ return decorator
+
+ def call_tool(self):
+ def decorator(fn):
+ captured["call_tool"] = fn
+ return fn
+ return decorator
+
+ def list_resources(self):
+ def decorator(fn):
+ captured["list_resources"] = fn
+ return fn
+ return decorator
+
+ def read_resource(self):
+ def decorator(fn):
+ captured["read_resource"] = fn
+ return fn
+ return decorator
+
+ def create_initialization_options(self):
+ return {}
+
+ async def run(self, *_args, **_kwargs) -> None:
+ captured["ran"] = True
+
+ class FakeStdioCM:
+ async def __aenter__(self):
+ return (MagicMock(), MagicMock())
+
+ async def __aexit__(self, *_args):
+ return False
+
+ fake_server_mod = MagicMock()
+ fake_server_mod.Server = FakeServer
+ fake_stdio_mod = MagicMock()
+ fake_stdio_mod.stdio_server = MagicMock(return_value=FakeStdioCM())
+ fake_types_mod = MagicMock()
+ fake_types_mod.Tool = lambda **kwargs: kwargs
+ fake_types_mod.TextContent = lambda **kwargs: kwargs
+ fake_types_mod.Resource = lambda **kwargs: kwargs
+
+ monkeypatch.setitem(sys.modules, "mcp", MagicMock())
+ monkeypatch.setitem(sys.modules, "mcp.server", fake_server_mod)
+ monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod)
+ monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod)
+
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "7"}, clear=False):
+ mcp_server.main()
+
+ assert captured["name"] == "site-audit"
+ assert captured["ran"] is True
+ tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type]
+ assert len(tools) >= 123
+ resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type]
+ assert any(r["uri"] == "audit://property/7" for r in resources)
+
+ with patch("website_profiling.mcp.server.dispatch_tool", return_value={"ok": True}):
+ content = asyncio.run(captured["call_tool"]("list_properties", {"property_id": 1})) # type: ignore[arg-type]
+ assert content[0]["text"] == json.dumps({"ok": True}, indent=2, default=str)
+ read_text = asyncio.run(captured["read_resource"]("audit://tools")) # type: ignore[arg-type]
+ assert read_text.startswith("{")
+
+
+def test_mcp_package_main(monkeypatch) -> None:
+ with patch("website_profiling.mcp.server.main") as mock_main:
+ runpy.run_module("website_profiling.mcp", run_name="__main__")
+ mock_main.assert_called_once()
+
+
+def test_mcp_server_main_guard() -> None:
+ with pytest.raises(SystemExit, match="MCP SDK"):
+ runpy.run_module("website_profiling.mcp.server", run_name="__main__")
diff --git a/tests/test_ollama_messages.py b/tests/test_ollama_messages.py
new file mode 100644
index 00000000..69b2b718
--- /dev/null
+++ b/tests/test_ollama_messages.py
@@ -0,0 +1,59 @@
+"""Ollama chat message normalization for tool calling."""
+from __future__ import annotations
+
+import json
+
+from website_profiling.llm.providers.ollama import normalize_messages_for_ollama
+
+
+def test_tool_message_uses_tool_name_not_tool_call_id():
+ msgs = normalize_messages_for_ollama([
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [{
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "list_properties", "arguments": "{}"},
+ }],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": '{"ok": true}',
+ },
+ ])
+ tool_msg = msgs[-1]
+ assert tool_msg["role"] == "tool"
+ assert "tool_call_id" not in tool_msg
+ assert tool_msg.get("tool_name") == "tool"
+ assert tool_msg["content"] == '{"ok": true}'
+
+
+def test_assistant_tool_call_arguments_are_objects():
+ msgs = normalize_messages_for_ollama([
+ {
+ "role": "assistant",
+ "tool_calls": [{
+ "type": "function",
+ "function": {"name": "list_issues", "arguments": '{"limit": 5}'},
+ }],
+ },
+ ])
+ fn = msgs[0]["tool_calls"][0]["function"]
+ assert fn["arguments"] == {"limit": 5}
+
+
+def test_assistant_tool_call_preserves_dict_arguments():
+ msgs = normalize_messages_for_ollama([
+ {
+ "role": "assistant",
+ "tool_calls": [{
+ "type": "function",
+ "function": {"name": "list_issues", "arguments": {"limit": 3}},
+ }],
+ },
+ ])
+ fn = msgs[0]["tool_calls"][0]["function"]
+ assert fn["arguments"] == {"limit": 3}
diff --git a/tests/test_text_sanitize.py b/tests/test_text_sanitize.py
new file mode 100644
index 00000000..a73d18ea
--- /dev/null
+++ b/tests/test_text_sanitize.py
@@ -0,0 +1,89 @@
+"""Tests for surrogate stripping in chat/JSON paths."""
+from __future__ import annotations
+
+import json
+from unittest.mock import patch
+
+from website_profiling.llm.agent import run_agent_turn
+from website_profiling.llm.base import ChatResult, ToolCall
+from website_profiling.text_sanitize import sanitize_unicode_deep, strip_surrogates
+from website_profiling.tools.audit_tools import AuditToolContext
+
+
+def test_strip_surrogates_replaces_lone_surrogate() -> None:
+ bad = "URL issue\udc9d here"
+ cleaned = strip_surrogates(bad)
+ assert "\udc9d" not in cleaned
+ cleaned.encode("utf-8")
+
+
+def test_sanitize_unicode_deep_nested() -> None:
+ payload = {
+ "issues": [{"message": "broken\udc9d", "url": "https://example.com"}],
+ }
+ cleaned = sanitize_unicode_deep(payload)
+ serialized = json.dumps(cleaned, ensure_ascii=False)
+ serialized.encode("utf-8")
+
+
+def test_sanitize_unicode_deep_tuple() -> None:
+ cleaned = sanitize_unicode_deep(("ok\udc9d", {"nested": "x\udc9d"}))
+ assert isinstance(cleaned, tuple)
+ assert "\udc9d" not in cleaned[0]
+ assert "\udc9d" not in cleaned[1]["nested"]
+
+
+def test_agent_surrogate_tool_result_does_not_break_llm_request() -> None:
+ surrogate = "\udc9d"
+ tool_payload = {
+ "issues": [
+ {
+ "category": "Technical SEO",
+ "priority": "High",
+ "message": f"URL in sitemap but not crawled: https://codefrydev.in/2048{surrogate}",
+ "url": "https://codefrydev.in/2048",
+ },
+ ],
+ "total": 1,
+ "truncated": False,
+ }
+
+ class RecordingClient:
+ def __init__(self) -> None:
+ self.last_messages: list[dict] | None = None
+
+ def chat_with_tools(self, messages, tools, *, on_token=None):
+ self.last_messages = messages
+ if self.last_messages and any(
+ m.get("role") == "tool" for m in self.last_messages
+ ):
+ return ChatResult(content="Summary with no further tools.")
+ return ChatResult(
+ tool_calls=[ToolCall(id="tc1", name="list_issues", arguments={"priority": "High"})],
+ )
+
+ client = RecordingClient()
+ events: list[dict] = []
+
+ with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={
+ "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test",
+ }):
+ with patch("website_profiling.llm.agent.get_llm_client", return_value=client):
+ with patch(
+ "website_profiling.llm.agent.dispatch_tool",
+ return_value=tool_payload,
+ ):
+ result = run_agent_turn(
+ [{"role": "user", "content": "high risk audit issues"}],
+ AuditToolContext(property_id=1),
+ on_event=events.append,
+ )
+
+ assert result["ok"] is True
+ assert client.last_messages is not None
+ json.dumps(
+ {"messages": client.last_messages, "tools": [], "stream": True},
+ ensure_ascii=False,
+ ).encode("utf-8")
+ tool_end = next(e for e in events if e["type"] == "tool_end")
+ assert "\udc9d" not in json.dumps(tool_end["result"])
diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.ts
new file mode 100644
index 00000000..89d60f8d
--- /dev/null
+++ b/web/app/api/chat/route.ts
@@ -0,0 +1,277 @@
+import { type NextRequest } from 'next/server';
+import { spawn } from 'child_process';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { requireApiAuthForChat } from '@/server/auth';
+import { getPipelineSpawnEnv, getRepoRoot } from '@/server/pipelineSpawnEnv';
+import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython';
+import {
+ appendChatMessage,
+ getChatMessages,
+ getChatSession,
+ messagesForAgentContext,
+ updateChatSessionTitle,
+} from '@/server/chatDb';
+import { loadLlmConfig } from '@/server/llmConfig';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+const DEFAULT_CHAT_TIMEOUT_MS = 120_000;
+const OLLAMA_MIN_TIMEOUT_MS = 300_000;
+
+async function resolveChatTimeoutMs(): Promise {
+ try {
+ const cfg = await loadLlmConfig();
+ const provider = String(cfg.state.llm_provider || 'none');
+ const timeoutS = Number(cfg.state.llm_timeout_s) || 120;
+ const baseMs = Math.max(timeoutS, 30) * 1000;
+ if (provider === 'ollama') {
+ return Math.max(baseMs, OLLAMA_MIN_TIMEOUT_MS);
+ }
+ return baseMs;
+ } catch {
+ return DEFAULT_CHAT_TIMEOUT_MS;
+ }
+}
+
+interface ChatBody {
+ sessionId?: number;
+ message?: string;
+ propertyId?: number;
+ reportId?: number;
+}
+
+function sseLine(event: string, data: Record): string {
+ return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+}
+
+/** POST /api/chat — stream agent response via SSE. */
+export const POST: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ let body: ChatBody;
+ try {
+ body = await request.json();
+ } catch {
+ return new Response(JSON.stringify({ error: 'Invalid JSON' }), { status: 400 });
+ }
+
+ const sessionId = Number(body.sessionId || 0);
+ const propertyId = Number(body.propertyId || 0);
+ const message = String(body.message || '').trim();
+ const reportId = body.reportId != null ? Number(body.reportId) : undefined;
+
+ if (!sessionId || !propertyId || !message) {
+ return new Response(
+ JSON.stringify({ error: 'sessionId, propertyId, and message are required' }),
+ { status: 400 },
+ );
+ }
+
+ const session = await getChatSession(sessionId);
+ if (!session || session.property_id !== propertyId) {
+ return new Response(JSON.stringify({ error: 'session not found' }), { status: 404 });
+ }
+
+ await appendChatMessage(sessionId, 'user', message);
+
+ const history = await getChatMessages(sessionId);
+ const agentMessages = messagesForAgentContext(history, 20);
+
+ const repoRoot = getRepoRoot();
+ const pythonExe = resolvePythonExecutable(null, repoRoot);
+ const stdinPayload = JSON.stringify({
+ messages: agentMessages,
+ property_id: propertyId,
+ report_id: Number.isFinite(reportId) ? reportId : undefined,
+ });
+
+ const chatTimeoutMs = await resolveChatTimeoutMs();
+ const timeoutSec = Math.round(chatTimeoutMs / 1000);
+
+ const stream = new ReadableStream({
+ start(controller) {
+ const encoder = new TextEncoder();
+ let assistantText = '';
+ let buffer = '';
+ let stderrAcc = '';
+ const toolEvents: Array<{
+ name: string;
+ args?: Record;
+ result?: Record;
+ }> = [];
+ let sawError = false;
+ let timedOut = false;
+ let closed = false;
+ let exitCode: number | null = null;
+
+ const closeStream = () => {
+ if (closed) return;
+ closed = true;
+ try {
+ controller.close();
+ } catch {
+ /* stream may already be closed (client disconnect, timeout race) */
+ }
+ };
+
+ const push = (event: string, data: Record) => {
+ if (closed) return;
+ if (event === 'error') sawError = true;
+ try {
+ controller.enqueue(encoder.encode(sseLine(event, data)));
+ } catch {
+ closed = true;
+ }
+ };
+
+ const proc = spawn(
+ pythonExe,
+ ['-m', 'src', 'chat', '--stdin-json'],
+ {
+ cwd: repoRoot,
+ env: getPipelineSpawnEnv(repoRoot, propertyId),
+ shell: false,
+ },
+ );
+
+ const timer = setTimeout(() => {
+ timedOut = true;
+ try {
+ proc.kill();
+ } catch {
+ /* ignore */
+ }
+ push('error', { message: `Chat timed out after ${timeoutSec}s` });
+ closeStream();
+ }, chatTimeoutMs);
+
+ proc.stdin?.write(stdinPayload);
+ proc.stdin?.end();
+
+ proc.stdout?.on('data', (chunk: Buffer) => {
+ buffer += chunk.toString();
+ const lines = buffer.split('\n');
+ buffer = lines.pop() || '';
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed) continue;
+ try {
+ const evt = JSON.parse(trimmed) as {
+ type?: string;
+ text?: string;
+ message?: string;
+ phase?: string;
+ detail?: string;
+ name?: string;
+ args?: Record;
+ result?: Record;
+ };
+ if (evt.type === 'token' && evt.text) {
+ assistantText += evt.text;
+ push('token', { text: evt.text });
+ } else if (evt.type === 'status') {
+ push('status', {
+ phase: evt.phase || 'working',
+ detail: evt.detail || evt.message || '',
+ });
+ } else if (evt.type === 'tool_start') {
+ toolEvents.push({
+ name: String(evt.name || ''),
+ args: evt.args || {},
+ });
+ push('tool_start', evt as Record);
+ } else if (evt.type === 'tool_end') {
+ const name = String(evt.name || '');
+ const existing = toolEvents.findIndex((t) => t.name === name && t.result == null);
+ if (existing >= 0) {
+ toolEvents[existing] = {
+ ...toolEvents[existing],
+ result: evt.result || {},
+ };
+ } else {
+ toolEvents.push({ name, result: evt.result || {} });
+ }
+ push('tool_end', evt as Record);
+ } else if (evt.type === 'done' && evt.message) {
+ assistantText = evt.message;
+ push('done', { message: evt.message });
+ } else if (evt.type === 'error') {
+ push('error', { message: evt.message || 'Agent error' });
+ }
+ } catch {
+ /* ignore non-JSON log lines */
+ }
+ }
+ });
+
+ proc.stderr?.on('data', (chunk: Buffer) => {
+ stderrAcc += chunk.toString();
+ if (stderrAcc.length > 8000) {
+ stderrAcc = stderrAcc.slice(-8000);
+ }
+ });
+
+ proc.on('error', (err: Error) => {
+ clearTimeout(timer);
+ push('error', { message: formatPythonSpawnError(err, pythonExe, repoRoot) });
+ closeStream();
+ });
+
+ proc.on('close', async (code: number | null) => {
+ clearTimeout(timer);
+ if (timedOut) return;
+ exitCode = code;
+
+ if (!sawError && !assistantText.trim()) {
+ const stderrLine = stderrAcc
+ .split('\n')
+ .map((l) => l.trim())
+ .find((l) => l && !l.startsWith('['));
+ const fallback =
+ stderrLine ||
+ (exitCode != null && exitCode !== 0
+ ? `Assistant process exited with code ${exitCode}.`
+ : 'No response from the assistant.');
+ push('error', { message: fallback });
+ } else if (!sawError && exitCode != null && exitCode !== 0) {
+ const stderrLine = stderrAcc
+ .split('\n')
+ .map((l) => l.trim())
+ .find((l) => l && !l.startsWith('['));
+ if (stderrLine) {
+ push('error', { message: stderrLine });
+ }
+ }
+
+ if (assistantText.trim()) {
+ try {
+ await appendChatMessage(sessionId, 'assistant', assistantText.trim(), {
+ toolResult: toolEvents.length ? { tool_events: toolEvents } : null,
+ });
+ if (session.title === 'New chat') {
+ const title = message.slice(0, 60) + (message.length > 60 ? '…' : '');
+ await updateChatSessionTitle(sessionId, title);
+ }
+ } catch {
+ /* persistence failure should not break stream */
+ }
+ }
+
+ closeStream();
+ });
+ },
+ });
+
+ return new Response(stream, {
+ headers: {
+ 'Content-Type': 'text/event-stream',
+ 'Cache-Control': 'no-cache',
+ Connection: 'keep-alive',
+ },
+ });
+};
diff --git a/web/app/api/chat/sessions/[id]/messages/route.ts b/web/app/api/chat/sessions/[id]/messages/route.ts
new file mode 100644
index 00000000..e4f17d4c
--- /dev/null
+++ b/web/app/api/chat/sessions/[id]/messages/route.ts
@@ -0,0 +1,33 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { requireApiAuthForChat } from '@/server/auth';
+import { getChatMessages } from '@/server/chatDb';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET /api/chat/sessions/[id]/messages */
+export const GET: ApiRouteHandler = async (
+ request: NextRequest,
+ context?: { params?: Promise<{ id: string }> },
+): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ const params = context?.params ? await context.params : { id: '' };
+ const sessionId = Number(params.id || '0');
+ if (!sessionId) {
+ return NextResponse.json({ error: 'invalid session id' }, { status: 400 });
+ }
+
+ try {
+ const messages = await getChatMessages(sessionId);
+ return NextResponse.json({ messages });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
diff --git a/web/app/api/chat/sessions/[id]/route.ts b/web/app/api/chat/sessions/[id]/route.ts
new file mode 100644
index 00000000..366d1397
--- /dev/null
+++ b/web/app/api/chat/sessions/[id]/route.ts
@@ -0,0 +1,64 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { requireApiAuthForChat } from '@/server/auth';
+import { deleteChatSession, getChatSession } from '@/server/chatDb';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET /api/chat/sessions/[id] */
+export const GET: ApiRouteHandler = async (
+ request: NextRequest,
+ context?: { params?: Promise<{ id: string }> },
+): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ const params = context?.params ? await context.params : { id: '' };
+ const sessionId = Number(params.id || '0');
+ if (!sessionId) {
+ return NextResponse.json({ error: 'invalid session id' }, { status: 400 });
+ }
+
+ try {
+ const session = await getChatSession(sessionId);
+ if (!session) {
+ return NextResponse.json({ error: 'session not found' }, { status: 404 });
+ }
+ return NextResponse.json({ session });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
+
+/** DELETE /api/chat/sessions/[id] */
+export const DELETE: ApiRouteHandler = async (
+ request: NextRequest,
+ context?: { params?: Promise<{ id: string }> },
+): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ const params = context?.params ? await context.params : { id: '' };
+ const sessionId = Number(params.id || '0');
+ if (!sessionId) {
+ return NextResponse.json({ error: 'invalid session id' }, { status: 400 });
+ }
+
+ try {
+ const deleted = await deleteChatSession(sessionId);
+ if (!deleted) {
+ return NextResponse.json({ error: 'session not found' }, { status: 404 });
+ }
+ return NextResponse.json({ ok: true });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
diff --git a/web/app/api/chat/sessions/route.ts b/web/app/api/chat/sessions/route.ts
new file mode 100644
index 00000000..01cb9195
--- /dev/null
+++ b/web/app/api/chat/sessions/route.ts
@@ -0,0 +1,57 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { requireApiAuthForChat } from '@/server/auth';
+import { createChatSession, listChatSessions } from '@/server/chatDb';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET /api/chat/sessions?propertyId= — list chat sessions for a property. */
+export const GET: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ const propertyId = Number(request.nextUrl.searchParams.get('propertyId') || '0');
+ if (!propertyId) {
+ return NextResponse.json({ error: 'propertyId required' }, { status: 400 });
+ }
+
+ try {
+ const sessions = await listChatSessions(propertyId);
+ return NextResponse.json({ sessions });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
+
+/** POST /api/chat/sessions — create session { propertyId, title? }. */
+export const POST: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuthForChat(request);
+ if (authDenied) return authDenied;
+
+ let body: { propertyId?: number; title?: string };
+ try {
+ body = await request.json();
+ } catch {
+ return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 });
+ }
+
+ const propertyId = Number(body.propertyId || 0);
+ if (!propertyId) {
+ return NextResponse.json({ error: 'propertyId required' }, { status: 400 });
+ }
+
+ try {
+ const id = await createChatSession(propertyId, body.title);
+ return NextResponse.json({ id, propertyId, title: body.title?.trim() || 'New chat' });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
diff --git a/web/app/api/jobs/[id]/cancel/route.ts b/web/app/api/jobs/[id]/cancel/route.ts
new file mode 100644
index 00000000..44f99111
--- /dev/null
+++ b/web/app/api/jobs/[id]/cancel/route.ts
@@ -0,0 +1,32 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { requireApiAuth } from '@/server/auth';
+import { cancelPipelineJob } from '@/server/pipelineJobs';
+import type { ApiRouteHandlerWithParams } from '@/types/api';
+
+export const runtime = 'nodejs';
+
+/**
+ * POST /api/jobs/:id/cancel — stop a running pipeline job.
+ */
+export const POST: ApiRouteHandlerWithParams<{ id: string }> = async (
+ request: NextRequest,
+ { params }: { params: Promise<{ id: string }> },
+): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+ const authDenied = requireApiAuth(request);
+ if (authDenied) return authDenied;
+
+ const { id } = await params;
+ const result = await cancelPipelineJob(id);
+ if (!result.ok) {
+ const status = result.error === 'Job not found' ? 404 : 409;
+ return NextResponse.json({ error: result.error || 'Unable to cancel job' }, { status });
+ }
+ return NextResponse.json({
+ ok: true,
+ status: result.status,
+ error: result.error ?? null,
+ });
+};
diff --git a/web/app/api/ollama/status/route.ts b/web/app/api/ollama/status/route.ts
new file mode 100644
index 00000000..e742e9ab
--- /dev/null
+++ b/web/app/api/ollama/status/route.ts
@@ -0,0 +1,61 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { loadLlmConfig } from '@/server/llmConfig';
+import {
+ fetchOllamaModels,
+ modelIsConfigured,
+ modelsSupportTools,
+} from '@/server/ollamaModels';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+
+const DEFAULT_BASE = 'http://127.0.0.1:11434';
+
+/** GET /api/ollama/status — local install + full Ollama cloud catalog. */
+export const GET: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+
+ let baseUrl = DEFAULT_BASE;
+ let configuredModel = '';
+ try {
+ const cfg = await loadLlmConfig();
+ baseUrl = String(cfg.state.llm_base_url || DEFAULT_BASE).replace(/\/$/, '');
+ configuredModel = String(cfg.state.llm_model || '').trim();
+ } catch {
+ /* use defaults */
+ }
+
+ const result = await fetchOllamaModels(baseUrl);
+
+ if (!result.ok) {
+ return NextResponse.json({
+ ok: false,
+ baseUrl: result.baseUrl,
+ configuredModel,
+ error: result.error || 'Cannot reach Ollama. Is it running?',
+ models: [],
+ cloudCatalogOk: false,
+ localOk: false,
+ });
+ }
+
+ const modelInstalled = modelIsConfigured(result.models, configuredModel);
+ const configuredEntry = result.models.find(
+ (m) => m.name.toLowerCase() === configuredModel.toLowerCase(),
+ );
+
+ return NextResponse.json({
+ ok: true,
+ baseUrl: result.baseUrl,
+ configuredModel,
+ modelInstalled,
+ supportsTools: configuredEntry?.capabilities?.includes('tools') ?? modelsSupportTools(result.models),
+ cloudCatalogOk: result.cloudCatalogOk,
+ localOk: result.localOk,
+ catalogSource: 'live',
+ cloudModelCount: result.models.filter((m) => m.source === 'cloud').length,
+ models: result.models,
+ });
+};
diff --git a/web/app/chat/page.tsx b/web/app/chat/page.tsx
new file mode 100644
index 00000000..3fba8e0c
--- /dev/null
+++ b/web/app/chat/page.tsx
@@ -0,0 +1,12 @@
+import { Suspense } from 'react';
+import ChatPage from '@/views/Chat';
+
+export const dynamic = 'force-dynamic';
+
+export default function ChatRoutePage() {
+ return (
+
+
+
+ );
+}
diff --git a/web/app/globals.css b/web/app/globals.css
index c9f676d8..bbf227d5 100644
--- a/web/app/globals.css
+++ b/web/app/globals.css
@@ -1,9 +1,7 @@
@import "tailwindcss";
-@source "../src/**/*.jsx";
-@source "../src/**/*.js";
-@source "./**/*.jsx";
-@source "./**/*.js";
+@source "../src/**/*.{js,jsx,ts,tsx}";
+@source "./**/*.{js,jsx,ts,tsx}";
/* Class-based dark mode (html.dark) — toggled by ThemeProvider + inline FOUC script */
@custom-variant dark (&:where(.dark, .dark *));
@@ -45,6 +43,12 @@
/* Links / interactive blue (readable on light surfaces) */
--app-link: #1d4ed8;
--app-link-soft: #1e3a8a;
+
+ --chat-bg: var(--app-bg);
+ --chat-surface: var(--app-bg-elevated);
+ --chat-surface-hover: var(--app-bg-muted);
+ --chat-glow: rgba(66, 97, 255, 0.08);
+ --chat-glow-secondary: rgba(138, 79, 255, 0.04);
}
html.dark {
@@ -81,6 +85,12 @@ html.dark {
--shadow: var(--shadow-elevated);
--app-link: #60a5fa;
--app-link-soft: #93c5fd;
+
+ --chat-bg: #131314;
+ --chat-surface: #1e1f20;
+ --chat-surface-hover: #28292a;
+ --chat-glow: rgba(66, 97, 255, 0.12);
+ --chat-glow-secondary: rgba(138, 79, 255, 0.06);
}
@theme {
@@ -233,6 +243,233 @@ code {
background: var(--code-bg);
}
+/* Chat shell layout — explicit breakpoints so sidebar participates in flex on desktop */
+.chat-sidebar-rail {
+ position: relative;
+ z-index: 40;
+ display: flex;
+ height: 100%;
+ width: 3.5rem;
+ flex-shrink: 0;
+ flex-direction: column;
+ align-items: center;
+ gap: 0.25rem;
+ border-right: 1px solid color-mix(in srgb, var(--app-border-muted) 70%, transparent);
+ background: var(--chat-surface);
+ padding: 0.75rem 0;
+}
+
+.chat-sidebar-backdrop {
+ position: fixed;
+ inset: 0;
+ z-index: 30;
+ border: 0;
+ background: var(--app-overlay);
+}
+
+.chat-sidebar-panel {
+ z-index: 40;
+ display: flex;
+ height: 100%;
+ width: 17.5rem;
+ flex-shrink: 0;
+ flex-direction: column;
+ border-right: 1px solid color-mix(in srgb, var(--app-border-muted) 70%, transparent);
+ background: var(--chat-surface);
+}
+
+@media (max-width: 767px) {
+ .chat-sidebar-backdrop {
+ display: block;
+ }
+
+ .chat-sidebar-panel {
+ position: fixed;
+ inset: 0 auto 0 0;
+ }
+}
+
+@media (min-width: 768px) {
+ .chat-sidebar-backdrop {
+ display: none;
+ }
+
+ .chat-sidebar-panel {
+ position: relative;
+ }
+}
+
+/* Chat shell — explicit height chain (flex-1 alone was not filling the viewport) */
+.chat-shell-main {
+ position: relative;
+ display: flex;
+ height: 100%;
+ min-height: 0;
+ flex: 1 1 0%;
+ flex-direction: column;
+ overflow: hidden;
+}
+
+.chat-main-panel {
+ display: flex;
+ height: 100%;
+ min-height: 0;
+ flex: 1 1 0%;
+ flex-direction: column;
+ overflow: hidden;
+}
+
+.chat-context-bar {
+ position: relative;
+ z-index: 10;
+ flex-shrink: 0;
+}
+
+/* Grid pins composer to the bottom; messages scroll in the top row */
+.chat-conversation {
+ display: grid;
+ min-height: 0;
+ flex: 1 1 0%;
+ grid-template-rows: minmax(0, 1fr) auto;
+ overflow: hidden;
+}
+
+.chat-messages-scroll {
+ min-height: 0;
+ overflow-x: hidden;
+ overflow-y: auto;
+ overscroll-behavior: contain;
+}
+
+.chat-composer-dock {
+ border-top: 1px solid color-mix(in srgb, var(--app-border-muted) 55%, transparent);
+ background: color-mix(in srgb, var(--chat-bg) 96%, transparent);
+ backdrop-filter: blur(8px);
+}
+
+/* Chat page ambient glow */
+.chat-glow {
+ background: radial-gradient(
+ ellipse 90% 60% at 50% 42%,
+ var(--chat-glow) 0%,
+ var(--chat-glow-secondary) 40%,
+ transparent 72%
+ );
+}
+
+.chat-hero-input {
+ box-shadow:
+ 0 0 0 1px rgba(255, 255, 255, 0.06),
+ 0 8px 32px rgba(0, 0, 0, 0.35);
+}
+
+.chat-hero-input:focus-within {
+ box-shadow:
+ 0 0 0 1px rgba(255, 255, 255, 0.1),
+ 0 8px 40px rgba(0, 0, 0, 0.45);
+}
+
+/* Chat markdown prose */
+.chat-prose {
+ color: var(--app-text);
+ font-size: 0.875rem;
+ line-height: 1.6;
+ overflow-wrap: anywhere;
+ word-break: break-word;
+}
+
+.chat-prose-h1,
+.chat-prose-h2,
+.chat-prose-h3 {
+ color: var(--app-text-heading);
+ font-weight: 600;
+ margin: 1rem 0 0.5rem;
+}
+
+.chat-prose-h1 { font-size: 1.125rem; }
+.chat-prose-h2 { font-size: 1rem; }
+.chat-prose-h3 {
+ font-size: 0.9375rem;
+ margin-top: 1.25rem;
+ padding-top: 0.25rem;
+ color: var(--app-text-heading);
+}
+
+.chat-prose > h3:first-child,
+.chat-prose > h4:first-child,
+.chat-prose > h5:first-child {
+ margin-top: 0;
+}
+
+.chat-prose-p {
+ margin: 0.5rem 0;
+}
+
+.chat-prose-ul,
+.chat-prose-ol {
+ margin: 0.5rem 0;
+ padding-left: 1.25rem;
+}
+
+.chat-prose-li {
+ margin: 0.25rem 0;
+}
+
+.chat-prose-a {
+ color: var(--app-link);
+ text-decoration: underline;
+ text-underline-offset: 2px;
+ word-break: break-all;
+}
+
+.chat-prose-blockquote {
+ margin: 0.75rem 0;
+ padding-left: 0.75rem;
+ border-left: 3px solid var(--app-border);
+ color: var(--app-text-subtle);
+}
+
+.chat-prose-hr {
+ margin: 1rem 0;
+ border: 0;
+ border-top: 1px solid var(--app-border-muted);
+}
+
+.chat-prose-table-wrap {
+ margin: 0.75rem 0;
+ overflow-x: auto;
+}
+
+.chat-prose-table {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 0.8125rem;
+}
+
+.chat-prose-th,
+.chat-prose-td {
+ border: 1px solid var(--app-border-muted);
+ padding: 0.375rem 0.5rem;
+ text-align: left;
+}
+
+.chat-prose-th {
+ background: var(--app-bg-muted);
+ font-weight: 600;
+}
+
+.chat-prose-pre {
+ margin: 0.75rem 0;
+}
+
+.chat-inline-code {
+ border-radius: 0.25rem;
+ background: var(--code-bg);
+ padding: 0.125rem 0.375rem;
+ font-family: var(--mono, ui-monospace, monospace);
+ font-size: 0.8125rem;
+}
+
/* Focus visibility for keyboard users */
.nav-btn:focus-visible,
button:focus-visible,
diff --git a/web/global.d.ts b/web/global.d.ts
index 561f3fab..1a01682c 100644
--- a/web/global.d.ts
+++ b/web/global.d.ts
@@ -2,6 +2,23 @@ declare module '*.css';
declare module '@/patchConsole';
+declare module 'react-syntax-highlighter' {
+ import type { ComponentType } from 'react';
+
+ export interface SyntaxHighlighterProps {
+ language?: string;
+ style?: Record>;
+ customStyle?: Record;
+ children?: string;
+ }
+
+ export const Prism: ComponentType;
+}
+
+declare module 'react-syntax-highlighter/dist/esm/styles/prism' {
+ export const oneDark: Record>;
+}
+
declare module 'react-chartjs-2' {
import type { ChartProps } from 'react-chartjs-2/dist/types';
import type { ChartData, ChartOptions, DefaultDataPoint } from 'chart.js';
diff --git a/web/src/components/charts/CategoryScoreGauge.tsx b/web/src/components/charts/CategoryScoreGauge.tsx
new file mode 100644
index 00000000..8dcb67d2
--- /dev/null
+++ b/web/src/components/charts/CategoryScoreGauge.tsx
@@ -0,0 +1,93 @@
+'use client';
+
+import { format, strings } from '@/lib/strings';
+import { scoreBandColor } from '@/utils/chartPalette';
+
+const vo = strings.views.overview;
+const sj = strings.common;
+
+export interface CategoryScoreGaugeProps {
+ name: string;
+ score?: number | null;
+ size?: 'sm' | 'md';
+ onClick?: () => void;
+}
+
+export function CategoryScoreGauge({ name, score, size = 'md', onClick }: CategoryScoreGaugeProps) {
+ const clamped = score != null ? Math.min(100, Math.max(0, score)) : 0;
+ const label =
+ score == null ? sj.na : score >= 80 ? vo.scoreGood : score >= 50 ? vo.scoreNeeds : vo.scoreCritical;
+ const labelCls =
+ score == null
+ ? 'text-muted-foreground'
+ : score >= 80
+ ? 'text-green-700 dark:text-green-400'
+ : score >= 50
+ ? 'text-yellow-700 dark:text-yellow-400'
+ : 'text-red-600 dark:text-red-500';
+ const color = scoreBandColor(score);
+ const isCritical = score != null && score < 50;
+ const dim = size === 'sm' ? 'w-16 h-16' : 'w-20 h-20';
+ const textSize = size === 'sm' ? 'text-lg' : 'text-xl';
+
+ const inner = (
+ <>
+
+
+
+ {score != null ? score : sj.na}
+
+
+
+ >
+ );
+
+ if (onClick) {
+ return (
+
+ );
+ }
+
+ return {inner}
;
+}
diff --git a/web/src/components/charts/ScoreDelta.tsx b/web/src/components/charts/ScoreDelta.tsx
new file mode 100644
index 00000000..7d97781c
--- /dev/null
+++ b/web/src/components/charts/ScoreDelta.tsx
@@ -0,0 +1,27 @@
+'use client';
+
+import { Minus, TrendingDown, TrendingUp } from 'lucide-react';
+
+export interface ScoreDeltaProps {
+ delta: number | null | undefined;
+}
+
+export function ScoreDelta({ delta }: ScoreDeltaProps) {
+ if (delta == null || delta === 0) {
+ return (
+
+ 0
+
+ );
+ }
+ const up = delta > 0;
+ const Icon = up ? TrendingUp : TrendingDown;
+ const color = up ? 'text-emerald-700 dark:text-emerald-400' : 'text-rose-700 dark:text-rose-400';
+ return (
+
+
+ {up ? '+' : ''}
+ {delta}
+
+ );
+}
diff --git a/web/src/components/charts/SimpleBarChart.tsx b/web/src/components/charts/SimpleBarChart.tsx
new file mode 100644
index 00000000..865b22dc
--- /dev/null
+++ b/web/src/components/charts/SimpleBarChart.tsx
@@ -0,0 +1,44 @@
+'use client';
+
+import { useMemo } from 'react';
+import { Bar } from 'react-chartjs-2';
+import { palette } from '@/utils/chartPalette';
+import { barOptionsHorizontal, registerChartJsBase } from '@/utils/chartJsDefaults';
+
+registerChartJsBase();
+
+export interface SimpleBarChartProps {
+ labels: string[];
+ values: number[];
+ horizontal?: boolean;
+ ariaLabel?: string;
+ heightClass?: string;
+}
+
+export function SimpleBarChart({
+ labels,
+ values,
+ horizontal = true,
+ ariaLabel,
+ heightClass = 'h-48',
+}: SimpleBarChartProps) {
+ const data = useMemo(
+ () => ({
+ labels,
+ datasets: [
+ {
+ data: values,
+ backgroundColor: palette(labels.length),
+ borderRadius: 4,
+ },
+ ],
+ }),
+ [labels, values],
+ );
+
+ return (
+
+
+
+ );
+}
diff --git a/web/src/components/charts/index.ts b/web/src/components/charts/index.ts
index 0d22c3ca..beb35b0e 100644
--- a/web/src/components/charts/index.ts
+++ b/web/src/components/charts/index.ts
@@ -3,3 +3,6 @@ export { RatioBar, CoverageBar } from './RatioBar';
export { LighthouseScoreGrid } from './LighthouseScoreGrid';
export { StatusDistributionChart } from './DistributionChart';
export { RankedBarChart } from './RankedBarChart';
+export { CategoryScoreGauge } from './CategoryScoreGauge';
+export { ScoreDelta } from './ScoreDelta';
+export { SimpleBarChart } from './SimpleBarChart';
diff --git a/web/src/components/chat/ChatActivityBar.tsx b/web/src/components/chat/ChatActivityBar.tsx
new file mode 100644
index 00000000..4800cc9b
--- /dev/null
+++ b/web/src/components/chat/ChatActivityBar.tsx
@@ -0,0 +1,30 @@
+'use client';
+
+import { Loader2 } from 'lucide-react';
+import { format, strings } from '@/lib/strings';
+
+const c = strings.components.chat;
+
+export interface ChatActivityBarProps {
+ busy: boolean;
+ statusText?: string;
+ elapsedSec?: number;
+}
+
+export default function ChatActivityBar({ busy, statusText, elapsedSec }: ChatActivityBarProps) {
+ if (!busy && !statusText) return null;
+
+ return (
+
+ {busy ? : null}
+ {statusText || c.thinking}
+ {busy && elapsedSec != null && elapsedSec > 0 ? (
+ · {format(c.elapsed, { seconds: elapsedSec })}
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/ChatComposer.tsx b/web/src/components/chat/ChatComposer.tsx
new file mode 100644
index 00000000..cc1e7dc9
--- /dev/null
+++ b/web/src/components/chat/ChatComposer.tsx
@@ -0,0 +1,123 @@
+'use client';
+
+import { useCallback, useEffect, useRef, useState, type FormEvent, type ReactNode } from 'react';
+import { Loader2, Plus, Send } from 'lucide-react';
+import { strings } from '@/lib/strings';
+
+const c = strings.components.chat;
+
+export interface ChatComposerProps {
+ disabled?: boolean;
+ busy?: boolean;
+ onSend: (message: string) => void;
+ trailing?: ReactNode;
+ variant?: 'hero' | 'dock';
+ draftMessage?: string;
+ onDraftApplied?: () => void;
+}
+
+export default function ChatComposer({
+ disabled,
+ busy,
+ onSend,
+ trailing,
+ variant = 'dock',
+ draftMessage,
+ onDraftApplied,
+}: ChatComposerProps) {
+ const [text, setText] = useState('');
+ const textareaRef = useRef(null);
+
+ const submitDisabled = disabled || busy || !text.trim();
+
+ const resizeTextarea = useCallback(() => {
+ const el = textareaRef.current;
+ if (!el) return;
+ el.style.height = 'auto';
+ el.style.height = `${Math.min(el.scrollHeight, 160)}px`;
+ }, []);
+
+ useEffect(() => {
+ if (!draftMessage) return;
+ setText(draftMessage);
+ resizeTextarea();
+ onDraftApplied?.();
+ textareaRef.current?.focus();
+ }, [draftMessage, onDraftApplied, resizeTextarea]);
+
+ const handleSubmit = (e: FormEvent) => {
+ e.preventDefault();
+ const msg = text.trim();
+ if (!msg || submitDisabled) return;
+ onSend(msg);
+ setText('');
+ if (textareaRef.current) {
+ textareaRef.current.style.height = 'auto';
+ }
+ };
+
+ const isHero = variant === 'hero';
+
+ return (
+
+ );
+}
diff --git a/web/src/components/chat/ChatContextBar.tsx b/web/src/components/chat/ChatContextBar.tsx
new file mode 100644
index 00000000..2fdb3242
--- /dev/null
+++ b/web/src/components/chat/ChatContextBar.tsx
@@ -0,0 +1,59 @@
+'use client';
+
+import { Globe, PanelLeft } from 'lucide-react';
+import { formatChatPropertyLabel } from '@/lib/chatPropertyLabel';
+import { strings } from '@/lib/strings';
+import type { PropertyOption } from '@/components/chat/ChatSidebar';
+
+const c = strings.components.chat;
+
+export interface ChatContextBarProps {
+ property: PropertyOption | null;
+ propertyId: number | null;
+ sessionTitle?: string | null;
+ loading?: boolean;
+ onExpandSidebar?: () => void;
+}
+
+export default function ChatContextBar({
+ property,
+ propertyId,
+ sessionTitle,
+ loading,
+ onExpandSidebar,
+}: ChatContextBarProps) {
+ const domainLabel = property
+ ? formatChatPropertyLabel(property)
+ : propertyId
+ ? `#${propertyId}`
+ : c.noProperties;
+
+ return (
+
+ {onExpandSidebar ? (
+
+ ) : null}
+
+
+
+
+ {loading && !property ? c.loadingProperty : domainLabel}
+
+ {sessionTitle ? (
+
+ {sessionTitle}
+
+ ) : null}
+
+
+
+ );
+}
diff --git a/web/src/components/chat/ChatFollowUpContext.tsx b/web/src/components/chat/ChatFollowUpContext.tsx
new file mode 100644
index 00000000..bfdde745
--- /dev/null
+++ b/web/src/components/chat/ChatFollowUpContext.tsx
@@ -0,0 +1,26 @@
+'use client';
+
+import { createContext, useContext, type ReactNode } from 'react';
+
+export interface ChatFollowUpContextValue {
+ suggestFollowUp: (prompt: string) => void;
+}
+
+const ChatFollowUpContext = createContext(null);
+
+export function ChatFollowUpProvider({
+ children,
+ suggestFollowUp,
+}: {
+ children: ReactNode;
+ suggestFollowUp: (prompt: string) => void;
+}) {
+ return (
+ {children}
+ );
+}
+
+export function useChatFollowUp(): ChatFollowUpContextValue {
+ const ctx = useContext(ChatFollowUpContext);
+ return ctx ?? { suggestFollowUp: () => {} };
+}
diff --git a/web/src/components/chat/ChatMarkdown.tsx b/web/src/components/chat/ChatMarkdown.tsx
new file mode 100644
index 00000000..e02af26f
--- /dev/null
+++ b/web/src/components/chat/ChatMarkdown.tsx
@@ -0,0 +1,85 @@
+'use client';
+
+import { useMemo } from 'react';
+import ReactMarkdown from 'react-markdown';
+import remarkBreaks from 'remark-breaks';
+import remarkGfm from 'remark-gfm';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import type { Components } from 'react-markdown';
+import { preprocessChatMarkdown } from '@/components/chat/preprocessChatMarkdown';
+
+export interface ChatMarkdownProps {
+ content: string;
+ streaming?: boolean;
+}
+
+export default function ChatMarkdown({ content, streaming }: ChatMarkdownProps) {
+ const normalized = useMemo(() => preprocessChatMarkdown(content), [content]);
+
+ const components = useMemo(
+ () => ({
+ h1: ({ children }) => {children}
,
+ h2: ({ children }) => {children}
,
+ h3: ({ children }) => {children}
,
+ p: ({ children }) => {children}
,
+ ul: ({ children }) => ,
+ ol: ({ children }) => {children}
,
+ li: ({ children }) => {children},
+ a: ({ href, children }) => (
+
+ {children}
+
+ ),
+ strong: ({ children }) => {children},
+ em: ({ children }) => {children},
+ blockquote: ({ children }) => (
+ {children}
+ ),
+ hr: () =>
,
+ table: ({ children }) => (
+
+ ),
+ thead: ({ children }) => {children},
+ tbody: ({ children }) => {children},
+ tr: ({ children }) => {children}
,
+ th: ({ children }) => {children} | ,
+ td: ({ children }) => {children} | ,
+ pre: ({ children }) => {children}
,
+ code: ({ className, children }) => {
+ const text = String(children).replace(/\n$/, '');
+ const lang = /language-(\w+)/.exec(className || '')?.[1];
+ if (lang) {
+ return (
+
+ {text}
+
+ );
+ }
+ return {children};
+ },
+ }),
+ [],
+ );
+
+ if (!normalized.trim()) return null;
+
+ return (
+
+
+ {normalized}
+
+
+ );
+}
diff --git a/web/src/components/chat/ChatMessageList.tsx b/web/src/components/chat/ChatMessageList.tsx
new file mode 100644
index 00000000..37c85b2f
--- /dev/null
+++ b/web/src/components/chat/ChatMessageList.tsx
@@ -0,0 +1,104 @@
+'use client';
+
+import { useEffect, useRef } from 'react';
+import { Sparkles } from 'lucide-react';
+import ChatBlocks from '@/components/chat/blocks/ChatBlocks';
+import ChatMarkdown from '@/components/chat/ChatMarkdown';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { stripRedundantMarkdown } from '@/components/chat/stripRedundantMarkdown';
+import ChatToolActivity, { type ToolActivityItem } from './ChatToolActivity';
+import { strings } from '@/lib/strings';
+
+const c = strings.components.chat;
+
+export interface ChatMessage {
+ id: string | number;
+ role: 'user' | 'assistant';
+ content: string;
+ toolActivity?: ToolActivityItem[];
+ blocks?: ChatBlock[];
+ streaming?: boolean;
+ error?: boolean;
+ statusText?: string;
+}
+
+export interface ChatMessageListProps {
+ messages: ChatMessage[];
+ empty?: boolean;
+}
+
+export default function ChatMessageList({ messages, empty }: ChatMessageListProps) {
+ const scrollRef = useRef(null);
+
+ useEffect(() => {
+ const node = scrollRef.current;
+ if (!node) return;
+ const raf = requestAnimationFrame(() => {
+ node.scrollTop = node.scrollHeight;
+ });
+ return () => cancelAnimationFrame(raf);
+ }, [messages]);
+
+ if (empty) return null;
+
+ return (
+
+
+ {messages.map((msg) => (
+
+
+ {msg.role === 'assistant' && (msg.streaming || !msg.content) && !msg.error ? (
+
+ ) : null}
+ {msg.toolActivity?.length ? (
+
+ ) : null}
+ {msg.role === 'assistant' && msg.blocks?.length ? (
+
+ ) : null}
+ {msg.content ? (
+ msg.role === 'user' ? (
+
{msg.content}
+ ) : msg.streaming ? (
+
{msg.content}
+ ) : (
+
+ )
+ ) : msg.streaming ? (
+
+ {msg.statusText ||
+ (msg.toolActivity?.some((t) => t.status === 'running')
+ ? c.queryingData
+ : c.thinking)}
+
+ ) : msg.error && !msg.content ? (
+
{c.responseFailed}
+ ) : null}
+
+
+ ))}
+
+
+
+ );
+}
diff --git a/web/src/components/chat/ChatModelPicker.tsx b/web/src/components/chat/ChatModelPicker.tsx
new file mode 100644
index 00000000..9ed6a229
--- /dev/null
+++ b/web/src/components/chat/ChatModelPicker.tsx
@@ -0,0 +1,278 @@
+'use client';
+
+import { useEffect, useMemo, useRef, useState } from 'react';
+import Link from 'next/link';
+import { Check, ChevronDown, Circle, Loader2, RefreshCw } from 'lucide-react';
+import { useOllamaModels, type OllamaModelEntry } from '@/hooks/useOllamaModels';
+import {
+ ollamaBillingLabel,
+ ollamaModelOptionLabel,
+ ollamaModelShortLabel,
+} from '@/lib/ollamaModelLabels';
+import { format, strings } from '@/lib/strings';
+import { usePipeline } from '@/context/PipelineContext';
+
+const c = strings.components.chat;
+const s = strings.pipelineRunner.ollama;
+
+export interface ChatModelPickerProps {
+ provider: string;
+ model: string;
+ baseUrl?: string;
+ disabled?: boolean;
+}
+
+function groupModels(models: OllamaModelEntry[]) {
+ return {
+ installed: models.filter((m) => m.installed),
+ cloud: models.filter((m) => m.source === 'cloud' && !m.installed),
+ local: models.filter((m) => m.source === 'local' && !m.installed),
+ };
+}
+
+function ModelRow({
+ entry,
+ active,
+ onSelect,
+}: {
+ entry: OllamaModelEntry;
+ active: boolean;
+ onSelect: (name: string) => void;
+}) {
+ return (
+
+ );
+}
+
+export default function ChatModelPicker({
+ provider,
+ model,
+ baseUrl = 'http://127.0.0.1:11434',
+ disabled,
+}: ChatModelPickerProps) {
+ const { saveLlmModel, saving } = usePipeline();
+ const isOllama = provider === 'ollama';
+ const { status, loading, refresh, models, connected } = useOllamaModels(baseUrl, isOllama);
+ const [open, setOpen] = useState(false);
+ const [saveError, setSaveError] = useState('');
+ const [query, setQuery] = useState('');
+ const rootRef = useRef(null);
+
+ const filtered = useMemo(() => {
+ const q = query.trim().toLowerCase();
+ if (!q) return models;
+ return models.filter((m) => m.name.toLowerCase().includes(q));
+ }, [models, query]);
+
+ const { installed, cloud, local } = groupModels(filtered);
+ const selected = models.find((m) => m.name === model);
+ const busy = disabled || saving || loading;
+
+ const triggerLabel = isOllama
+ ? model
+ ? ollamaModelShortLabel(model)
+ : c.ollamaNoModel
+ : ollamaModelShortLabel(model || provider || 'AI');
+
+ useEffect(() => {
+ if (!open) return;
+ const onDocClick = (e: MouseEvent) => {
+ if (rootRef.current && !rootRef.current.contains(e.target as Node)) {
+ setOpen(false);
+ }
+ };
+ const onKey = (e: KeyboardEvent) => {
+ if (e.key === 'Escape') setOpen(false);
+ };
+ document.addEventListener('mousedown', onDocClick);
+ document.addEventListener('keydown', onKey);
+ return () => {
+ document.removeEventListener('mousedown', onDocClick);
+ document.removeEventListener('keydown', onKey);
+ };
+ }, [open]);
+
+ const handleModelChange = async (nextModel: string) => {
+ if (!nextModel || nextModel === model) {
+ setOpen(false);
+ return;
+ }
+ setSaveError('');
+ const ok = await saveLlmModel(nextModel);
+ if (!ok) setSaveError(c.modelSaveFailed);
+ else setOpen(false);
+ };
+
+ return (
+
+
+
+ {open ? (
+
+ {isOllama && connected && models.length ? (
+ <>
+
+ setQuery(e.target.value)}
+ placeholder={s.searchPlaceholder}
+ aria-label={c.findModel}
+ className="w-full rounded-xl border border-default bg-[var(--chat-bg)] px-3 py-2 text-xs text-foreground placeholder:text-muted-foreground focus:border-violet-500/50 focus:outline-none"
+ />
+
+
+ {installed.length ? (
+
+
+ {s.groupInstalled}
+
+ {installed.map((m) => (
+
void handleModelChange(name)}
+ />
+ ))}
+
+ ) : null}
+ {cloud.length ? (
+
+
+ {s.groupCloud}
+
+ {cloud.map((m) => (
+
void handleModelChange(name)}
+ />
+ ))}
+
+ ) : null}
+ {local.length ? (
+
+
+ {s.groupLocal}
+
+ {local.map((m) => (
+
void handleModelChange(name)}
+ />
+ ))}
+
+ ) : null}
+
+ >
+ ) : (
+
+ {isOllama ? (
+ connected ? (
+
{model || c.ollamaNoModel}
+ ) : (
+
{c.ollamaUnreachable}
+ )
+ ) : (
+
+ {model || provider}
+
+ )}
+
+ )}
+
+
+ {isOllama && connected ? (
+
+ {status?.supportsTools ? (
+ {c.ollamaToolsMode}
+ ) : (
+ {c.ollamaReactMode}
+ )}
+ {status?.cloudCatalogOk ? (
+ {format(c.ollamaCloudCount, { count: status.cloudModelCount ?? 0 })}
+ ) : null}
+ {selected ? (
+
+ {ollamaBillingLabel(selected.billing)}
+
+ ) : null}
+
+ ) : null}
+ {saveError ?
{saveError}
: null}
+
+ {isOllama ? (
+
+ ) : (
+
+ )}
+ setOpen(false)}
+ >
+ {c.aiSettingsLink}
+
+
+
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/ChatShell.tsx b/web/src/components/chat/ChatShell.tsx
new file mode 100644
index 00000000..68b4bad3
--- /dev/null
+++ b/web/src/components/chat/ChatShell.tsx
@@ -0,0 +1,39 @@
+'use client';
+
+import { useCallback, useState, type ReactNode } from 'react';
+
+export interface ChatLayoutState {
+ expanded: boolean;
+ toggle: () => void;
+ setExpanded: (value: boolean) => void;
+}
+
+export interface ChatShellProps {
+ sidebar: (layout: ChatLayoutState) => ReactNode;
+ children: ReactNode | ((layout: ChatLayoutState) => ReactNode);
+}
+
+export default function ChatShell({ sidebar, children }: ChatShellProps) {
+ const [expanded, setExpanded] = useState(false);
+
+ const toggle = useCallback(() => {
+ setExpanded((current) => !current);
+ }, []);
+
+ const layout: ChatLayoutState = {
+ expanded,
+ toggle,
+ setExpanded,
+ };
+
+ return (
+
+ {sidebar(layout)}
+
+
+
+ {typeof children === 'function' ? children(layout) : children}
+
+
+ );
+}
diff --git a/web/src/components/chat/ChatSidebar.tsx b/web/src/components/chat/ChatSidebar.tsx
new file mode 100644
index 00000000..c7c3f805
--- /dev/null
+++ b/web/src/components/chat/ChatSidebar.tsx
@@ -0,0 +1,312 @@
+'use client';
+
+import { useEffect, useRef, useState, type ReactNode } from 'react';
+import Link from 'next/link';
+import {
+ ChevronLeft,
+ History,
+ Home,
+ Link as LinkIcon,
+ MessageSquarePlus,
+ Settings,
+ Terminal,
+ Trash2,
+ TrendingUp,
+} from 'lucide-react';
+import AppLogo from '@/components/AppLogo';
+import ThemeToggle from '@/components/ThemeToggle';
+import type { ChatLayoutState } from '@/components/chat/ChatShell';
+import { formatChatPropertyOption } from '@/lib/chatPropertyLabel';
+import { strings } from '@/lib/strings';
+
+const c = strings.components.chat;
+
+export interface ChatSessionItem {
+ id: number;
+ title: string;
+}
+
+export interface PropertyOption {
+ id: number;
+ name: string;
+ canonical_domain: string;
+}
+
+export interface ChatSidebarProps extends ChatLayoutState {
+ sessions: ChatSessionItem[];
+ activeSessionId: number | null;
+ properties: PropertyOption[];
+ propertyId: number | null;
+ onPropertyChange: (id: number | null) => void;
+ onNewChat: () => void;
+ onSelect: (id: number) => void;
+ onDelete: (id: number) => void;
+ loading?: boolean;
+}
+
+const NAV_LINKS = [
+ { href: '/home', label: c.navHome, icon: Home },
+ { href: '/search-performance', label: c.navGsc, icon: TrendingUp },
+ { href: '/links', label: c.navLinks, icon: LinkIcon },
+ { href: '/pipeline', label: c.navPipeline, icon: Terminal },
+] as const;
+
+function RailButton({
+ label,
+ onClick,
+ children,
+ active,
+}: {
+ label: string;
+ onClick?: () => void;
+ children: ReactNode;
+ active?: boolean;
+}) {
+ return (
+
+ );
+}
+
+function SettingsMenu({ onClose }: { onClose: () => void }) {
+ return (
+
+
{c.settingsTitle}
+
+ Theme
+
+
+
+ {c.aiSettingsLink}
+
+
+ );
+}
+
+export default function ChatSidebar({
+ sessions,
+ activeSessionId,
+ properties,
+ propertyId,
+ onPropertyChange,
+ onNewChat,
+ onSelect,
+ onDelete,
+ loading,
+ expanded,
+ toggle,
+ setExpanded,
+}: ChatSidebarProps) {
+ const [settingsOpen, setSettingsOpen] = useState(false);
+ const settingsRef = useRef(null);
+
+ useEffect(() => {
+ if (!settingsOpen) return;
+ const onDocClick = (e: MouseEvent) => {
+ if (settingsRef.current && !settingsRef.current.contains(e.target as Node)) {
+ setSettingsOpen(false);
+ }
+ };
+ const onKey = (e: KeyboardEvent) => {
+ if (e.key === 'Escape') setSettingsOpen(false);
+ };
+ document.addEventListener('mousedown', onDocClick);
+ document.addEventListener('keydown', onKey);
+ return () => {
+ document.removeEventListener('mousedown', onDocClick);
+ document.removeEventListener('keydown', onKey);
+ };
+ }, [settingsOpen]);
+
+ const sessionList = (
+ <>
+ {loading ? (
+ {c.loadingSessions}
+ ) : sessions.length === 0 ? (
+ {c.noSessions}
+ ) : (
+
+ {sessions.map((s) => (
+ -
+
+
+
+ ))}
+
+ )}
+ >
+ );
+
+ if (!expanded) {
+ return (
+
+
+
+
+
+
+
+
+
+
setExpanded(true)}>
+
+
+
+
+
setSettingsOpen((v) => !v)}
+ active={settingsOpen}
+ >
+
+
+ {settingsOpen ? (
+
+ setSettingsOpen(false)} />
+
+ ) : null}
+
+
+ );
+ }
+
+ return (
+ <>
+
+
+
+ >
+ );
+}
diff --git a/web/src/components/chat/ChatToolActivity.tsx b/web/src/components/chat/ChatToolActivity.tsx
new file mode 100644
index 00000000..5e4de47b
--- /dev/null
+++ b/web/src/components/chat/ChatToolActivity.tsx
@@ -0,0 +1,54 @@
+'use client';
+
+import { useState } from 'react';
+import { ChevronDown, ChevronRight, Wrench } from 'lucide-react';
+import { format, strings } from '@/lib/strings';
+
+const c = strings.components.chat;
+
+export interface ToolActivityItem {
+ id: string;
+ name: string;
+ args?: Record;
+ result?: Record;
+ status: 'running' | 'done';
+}
+
+export interface ChatToolActivityProps {
+ items: ToolActivityItem[];
+}
+
+export default function ChatToolActivity({ items }: ChatToolActivityProps) {
+ const [open, setOpen] = useState(false);
+
+ if (!items.length) return null;
+
+ return (
+
+
+ {open ? (
+
+ {items.map((item) => (
+ -
+ {item.name}
+ {item.status === 'running' ? (
+ {c.toolRunning}
+ ) : (
+ {c.toolDone}
+ )}
+
+ ))}
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/SuggestedPrompts.tsx b/web/src/components/chat/SuggestedPrompts.tsx
new file mode 100644
index 00000000..ee34782a
--- /dev/null
+++ b/web/src/components/chat/SuggestedPrompts.tsx
@@ -0,0 +1,59 @@
+'use client';
+
+import {
+ AlertTriangle,
+ FileSearch,
+ Gauge,
+ GitBranch,
+ Link2,
+ Search,
+} from 'lucide-react';
+import { strings } from '@/lib/strings';
+import type { LucideIcon } from 'lucide-react';
+
+const c = strings.components.chat;
+
+const PROMPT_ICONS: LucideIcon[] = [
+ AlertTriangle,
+ Link2,
+ GitBranch,
+ FileSearch,
+ Search,
+ Gauge,
+];
+
+export interface SuggestedPromptsProps {
+ onSelect: (prompt: string) => void;
+ disabled?: boolean;
+}
+
+export default function SuggestedPrompts({ onSelect, disabled }: SuggestedPromptsProps) {
+ const prompts = c.suggestedPrompts.slice(0, 6);
+
+ return (
+
+
+ {prompts.map((prompt, i) => {
+ const Icon = PROMPT_ICONS[i % PROMPT_ICONS.length];
+ return (
+
+ );
+ })}
+
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatBlocks.tsx b/web/src/components/chat/blocks/ChatBlocks.tsx
new file mode 100644
index 00000000..1f79dee9
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatBlocks.tsx
@@ -0,0 +1,51 @@
+'use client';
+
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { blockKey } from '@/components/chat/deriveChatBlocks';
+import ChatCategoryScoresBlock from './ChatCategoryScoresBlock';
+import ChatCompareCategoryBlock from './ChatCompareCategoryBlock';
+import ChatGoogleSummaryBlock from './ChatGoogleSummaryBlock';
+import ChatHealthTrendBlock from './ChatHealthTrendBlock';
+import ChatIssueSummaryBlock from './ChatIssueSummaryBlock';
+import ChatIssueTableBlock from './ChatIssueTableBlock';
+import ChatLabelValueChartBlock from './ChatLabelValueChartBlock';
+import ChatLighthouseBlock from './ChatLighthouseBlock';
+import ChatStatusBreakdownBlock from './ChatStatusBreakdownBlock';
+
+export interface ChatBlocksProps {
+ blocks: ChatBlock[];
+}
+
+export default function ChatBlocks({ blocks }: ChatBlocksProps) {
+ if (!blocks.length) return null;
+
+ return (
+
+ {blocks.map((block) => {
+ const key = blockKey(block);
+ switch (block.type) {
+ case 'issue_summary':
+ return ;
+ case 'issue_table':
+ return ;
+ case 'category_scores':
+ return ;
+ case 'label_value_chart':
+ return ;
+ case 'status_breakdown':
+ return ;
+ case 'health_trend':
+ return ;
+ case 'compare_category_deltas':
+ return ;
+ case 'lighthouse_scores':
+ return ;
+ case 'google_summary':
+ return ;
+ default:
+ return null;
+ }
+ })}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatCategoryScoresBlock.tsx b/web/src/components/chat/blocks/ChatCategoryScoresBlock.tsx
new file mode 100644
index 00000000..73c3e9b7
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatCategoryScoresBlock.tsx
@@ -0,0 +1,94 @@
+'use client';
+
+import { useState } from 'react';
+import { CategoryScoreGauge } from '@/components/charts/CategoryScoreGauge';
+import { useChatFollowUp } from '@/components/chat/ChatFollowUpContext';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { format, strings } from '@/lib/strings';
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+export default function ChatCategoryScoresBlock({ block }: { block: Block }) {
+ const { suggestFollowUp } = useChatFollowUp();
+ const [view, setView] = useState<'gauges' | 'bars'>('gauges');
+
+ return (
+
+
+
+ {block.healthScore != null ? (
+
+ Health score:{' '}
+ {block.healthScore}
+
+ ) : null}
+
+
+
+
+
+
+
+ {view === 'gauges' ? (
+
+ {block.categories.map((cat) => (
+
+ suggestFollowUp(format(cb.askCategoryIssues, { category: cat.name }))
+ }
+ />
+ ))}
+
+ ) : (
+
+ )}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatCompareCategoryBlock.tsx b/web/src/components/chat/blocks/ChatCompareCategoryBlock.tsx
new file mode 100644
index 00000000..acbb0911
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatCompareCategoryBlock.tsx
@@ -0,0 +1,64 @@
+'use client';
+
+import { ScoreDelta } from '@/components/charts/ScoreDelta';
+import { useChatFollowUp } from '@/components/chat/ChatFollowUpContext';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { format, strings } from '@/lib/strings';
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+export default function ChatCompareCategoryBlock({ block }: { block: Block }) {
+ const { suggestFollowUp } = useChatFollowUp();
+ const maxAbs = Math.max(...block.rows.map((r) => Math.abs(r.delta ?? 0)), 0);
+
+ return (
+
+
+ {cb.categoryCompare}
+
+
+
+
+
+ | Category |
+ Current |
+ Baseline |
+ Δ |
+
+
+
+ {block.rows.map((row) => {
+ const highlight = Math.abs(row.delta ?? 0) === maxAbs && maxAbs > 0;
+ return (
+
+ |
+
+ |
+ {row.current ?? '—'} |
+
+ {row.baseline ?? '—'}
+ |
+
+
+ |
+
+ );
+ })}
+
+
+
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatGoogleSummaryBlock.tsx b/web/src/components/chat/blocks/ChatGoogleSummaryBlock.tsx
new file mode 100644
index 00000000..bb3a6a46
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatGoogleSummaryBlock.tsx
@@ -0,0 +1,95 @@
+'use client';
+
+import { SimpleBarChart } from '@/components/charts/SimpleBarChart';
+import { useChatFollowUp } from '@/components/chat/ChatFollowUpContext';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { format, strings } from '@/lib/strings';
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+export default function ChatGoogleSummaryBlock({ block }: { block: Block }) {
+ const { suggestFollowUp } = useChatFollowUp();
+ const hasKpis =
+ block.clicks != null || block.impressions != null || block.ctr != null;
+
+ return (
+
+
{cb.googleSummary}
+
+ {hasKpis ? (
+
+ {block.clicks != null ? (
+
+ {cb.clicks}{' '}
+ {block.clicks.toLocaleString()}
+
+ ) : null}
+ {block.impressions != null ? (
+
+ {cb.impressions}{' '}
+
+ {block.impressions.toLocaleString()}
+
+
+ ) : null}
+ {block.ctr != null ? (
+
+ {cb.ctr}{' '}
+
+ {(block.ctr * 100).toFixed(2)}%
+
+
+ ) : null}
+
+ ) : null}
+
+ {block.queries.length > 0 ? (
+
+
{cb.topQueries}
+
+ q.query.length > 28 ? `${q.query.slice(0, 28)}…` : q.query,
+ )}
+ values={block.queries.map((q) => q.clicks ?? 0)}
+ ariaLabel={cb.topQueries}
+ />
+
+ {block.queries.slice(0, 5).map((q) => (
+ -
+
+
+ {q.clicks ?? 0} clicks
+
+
+ ))}
+
+
+ ) : null}
+
+ {block.pages.length > 0 ? (
+
+
{cb.topPages}
+
+ {block.pages.map((p) => (
+ -
+
+ {p.page}
+
+
+ {p.clicks ?? 0}
+
+
+ ))}
+
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatHealthTrendBlock.tsx b/web/src/components/chat/blocks/ChatHealthTrendBlock.tsx
new file mode 100644
index 00000000..2f2a2223
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatHealthTrendBlock.tsx
@@ -0,0 +1,68 @@
+'use client';
+
+import { useMemo } from 'react';
+import { Line } from 'react-chartjs-2';
+import {
+ Chart as ChartJS,
+ CategoryScale,
+ LinearScale,
+ PointElement,
+ LineElement,
+ Title,
+ Tooltip,
+ Legend,
+} from 'chart.js';
+import { getChartTitleColor, getGridColor } from '@/utils/chartJsDefaults';
+import { strings } from '@/lib/strings';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend);
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+export default function ChatHealthTrendBlock({ block }: { block: Block }) {
+ const chartData = useMemo(
+ () => ({
+ labels: block.points.map((p) => p.label),
+ datasets: [
+ {
+ label: cb.healthScoreAxis,
+ data: block.points.map((p) => p.score),
+ borderColor: 'rgb(59, 130, 246)',
+ backgroundColor: 'rgba(59, 130, 246, 0.15)',
+ tension: 0.25,
+ fill: true,
+ },
+ ],
+ }),
+ [block.points],
+ );
+
+ const options = useMemo(
+ () => ({
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: { legend: { display: false } },
+ scales: {
+ y: {
+ min: 0,
+ max: 100,
+ grid: { color: getGridColor() },
+ title: { display: true, text: cb.healthScoreAxis, color: getChartTitleColor() },
+ },
+ x: { grid: { color: getGridColor() } },
+ },
+ }),
+ [],
+ );
+
+ return (
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatIssueSummaryBlock.tsx b/web/src/components/chat/blocks/ChatIssueSummaryBlock.tsx
new file mode 100644
index 00000000..95c61d79
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatIssueSummaryBlock.tsx
@@ -0,0 +1,54 @@
+'use client';
+
+import { PRIORITY_ORDER } from '@/lib/issuePriority';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+type Block = Extract;
+
+function formatSuccessRate(rate: number): string {
+ const pct = rate > 1 ? rate : rate * 100;
+ return `${pct.toFixed(pct % 1 === 0 ? 0 : 1)}%`;
+}
+
+export default function ChatIssueSummaryBlock({ block }: { block: Block }) {
+ const total =
+ block.totalIssues ??
+ PRIORITY_ORDER.reduce((sum, p) => sum + (block.counts[p] || 0), 0);
+
+ return (
+
+
+ {block.siteName ? (
+
{block.siteName}
+ ) : null}
+ {block.healthScore != null ? (
+
+ {block.healthScore}
+ /100
+
+ ) : null}
+
{total} issues
+ {block.totalUrls != null ? (
+
+ {block.totalUrls} URLs
+ {block.successRate != null ? ` · ${formatSuccessRate(block.successRate)} success` : ''}
+
+ ) : null}
+
+
+ {PRIORITY_ORDER.map((p) => {
+ const n = block.counts[p] || 0;
+ if (!n) return null;
+ return (
+
+ {p}: {n}
+
+ );
+ })}
+
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatIssueTableBlock.tsx b/web/src/components/chat/blocks/ChatIssueTableBlock.tsx
new file mode 100644
index 00000000..5787ede8
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatIssueTableBlock.tsx
@@ -0,0 +1,77 @@
+'use client';
+
+import Badge from '@/components/Badge';
+import { useChatFollowUp } from '@/components/chat/ChatFollowUpContext';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+import { formatChatUrlDisplay } from '@/lib/formatChatUrl';
+import { strings } from '@/lib/strings';
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+const DISPLAY_LIMIT = 15;
+
+export default function ChatIssueTableBlock({ block }: { block: Block }) {
+ const { suggestFollowUp } = useChatFollowUp();
+ const shown = block.issues.slice(0, DISPLAY_LIMIT);
+ const remaining = (block.total ?? block.issues.length) - shown.length;
+
+ return (
+
+
+
+
+
+ | Priority |
+ Category |
+ URL |
+ Issue |
+
+
+
+ {shown.map((issue, i) => (
+
+ |
+
+ |
+ {issue.category || '—'} |
+
+ {issue.url ? (
+
+ {formatChatUrlDisplay(issue.url)}
+
+ ) : (
+ '—'
+ )}
+ |
+
+ {issue.message || '—'}
+ |
+
+ ))}
+
+
+
+ {remaining > 0 || block.truncated ? (
+
+
+ {remaining > 0 ? `${remaining} more issues not shown` : 'Results truncated'}
+
+
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatLabelValueChartBlock.tsx b/web/src/components/chat/blocks/ChatLabelValueChartBlock.tsx
new file mode 100644
index 00000000..1363439e
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatLabelValueChartBlock.tsx
@@ -0,0 +1,54 @@
+'use client';
+
+import { useMemo } from 'react';
+import { Doughnut } from 'react-chartjs-2';
+import { SimpleBarChart } from '@/components/charts/SimpleBarChart';
+import { palette } from '@/utils/chartPalette';
+import { registerChartJsBase } from '@/utils/chartJsDefaults';
+import { doughnutOptionsWithPercentTooltip } from '@/lib/chartDoughnutUtils';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+registerChartJsBase();
+
+type Block = Extract;
+
+function preferBarChart(items: { label: string; value: number }[]): boolean {
+ if (items.length > 6) return true;
+ return items.some((i) => i.label.length > 20);
+}
+
+export default function ChatLabelValueChartBlock({ block }: { block: Block }) {
+ const useBar = preferBarChart(block.items);
+
+ const doughnutData = useMemo(() => {
+ const colors = palette(block.items.length);
+ return {
+ labels: block.items.map((i) => i.label),
+ datasets: [
+ {
+ data: block.items.map((i) => i.value),
+ backgroundColor: colors,
+ borderWidth: 0,
+ },
+ ],
+ };
+ }, [block.items]);
+
+ return (
+
+
{block.title}
+ {useBar ? (
+
i.label)}
+ values={block.items.map((i) => i.value)}
+ ariaLabel={block.title}
+ heightClass={block.items.length > 8 ? 'h-64' : 'h-48'}
+ />
+ ) : (
+
+
+
+ )}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatLighthouseBlock.tsx b/web/src/components/chat/blocks/ChatLighthouseBlock.tsx
new file mode 100644
index 00000000..c04e628d
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatLighthouseBlock.tsx
@@ -0,0 +1,47 @@
+'use client';
+
+import { LighthouseScoreGrid } from '@/components/charts';
+import { strings } from '@/lib/strings';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+const lhLabels = strings.lighthouse.categoryLabels as Record;
+
+export default function ChatLighthouseBlock({ block }: { block: Block }) {
+ const ariaParts = Object.entries(block.scores)
+ .filter(([, v]) => v != null)
+ .map(([k, v]) => `${lhLabels[k] || k}: ${v}`);
+
+ return (
+
+
{cb.lighthouseScores}
+
+ {block.poorPages.length > 0 ? (
+
+
{cb.poorPerformance}
+
+ {block.poorPages.map((p) => (
+ -
+
+ {p.url}
+
+ {p.performance}
+
+ ))}
+
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/chat/blocks/ChatStatusBreakdownBlock.tsx b/web/src/components/chat/blocks/ChatStatusBreakdownBlock.tsx
new file mode 100644
index 00000000..3f6872ee
--- /dev/null
+++ b/web/src/components/chat/blocks/ChatStatusBreakdownBlock.tsx
@@ -0,0 +1,40 @@
+'use client';
+
+import { useMemo } from 'react';
+import { Doughnut } from 'react-chartjs-2';
+import { palette } from '@/utils/chartPalette';
+import { registerChartJsBase } from '@/utils/chartJsDefaults';
+import { doughnutOptionsWithPercentTooltip } from '@/lib/chartDoughnutUtils';
+import { strings } from '@/lib/strings';
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+registerChartJsBase();
+
+type Block = Extract;
+const cb = strings.components.chat.blocks;
+
+export default function ChatStatusBreakdownBlock({ block }: { block: Block }) {
+ const chartData = useMemo(() => {
+ const colors = palette(block.items.length);
+ return {
+ labels: block.items.map((i) => i.label),
+ datasets: [{ data: block.items.map((i) => i.value), backgroundColor: colors, borderWidth: 0 }],
+ };
+ }, [block.items]);
+
+ return (
+
+
{cb.statusBreakdown}
+ {block.successRate != null ? (
+
+ {cb.successRate}:{' '}
+ {(block.successRate > 1 ? block.successRate : block.successRate * 100).toFixed(1)}%
+ {block.totalUrls != null ? ` · ${block.totalUrls} URLs` : ''}
+
+ ) : null}
+
+
+
+
+ );
+}
diff --git a/web/src/components/chat/deriveChatBlocks.test.ts b/web/src/components/chat/deriveChatBlocks.test.ts
new file mode 100644
index 00000000..836380f6
--- /dev/null
+++ b/web/src/components/chat/deriveChatBlocks.test.ts
@@ -0,0 +1,199 @@
+import { describe, expect, it } from 'vitest';
+import { blockKey, deriveChatBlocks, toolEventsToActivity } from './deriveChatBlocks';
+import type { ToolActivityItem } from './ChatToolActivity';
+
+function doneTool(name: string, result: Record): ToolActivityItem {
+ return { id: name, name, status: 'done', result };
+}
+
+describe('deriveChatBlocks', () => {
+ it('builds issue_summary from get_report_summary', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_report_summary', {
+ health_score: 74,
+ issue_counts: { Critical: 1, High: 29, Medium: 14, Low: 2 },
+ total_issues: 46,
+ site_name: 'codefrydev.in',
+ crawl_summary: {
+ total_urls: 30,
+ count_2xx: 30,
+ success_rate: 1,
+ },
+ }),
+ ]);
+ expect(blocks.map((b) => b.type)).toContain('issue_summary');
+ expect(blocks.map((b) => b.type)).toContain('status_breakdown');
+ const summary = blocks.find((b) => b.type === 'issue_summary');
+ if (summary?.type === 'issue_summary') {
+ expect(summary.healthScore).toBe(74);
+ expect(summary.counts.Critical).toBe(1);
+ expect(summary.totalUrls).toBe(30);
+ expect(summary.successRate).toBe(1);
+ }
+ });
+
+ it('builds category_scores from get_report_summary categories', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_report_summary', {
+ health_score: 80,
+ issue_counts: { High: 2 },
+ categories: [{ name: 'Crawl', score: 0, issue_count: 5 }],
+ }),
+ ]);
+ expect(blocks.map((b) => b.type)).toContain('issue_summary');
+ expect(blocks.map((b) => b.type)).toContain('category_scores');
+ });
+
+ it('builds issue_table from list_issues', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('list_issues', {
+ issues: [
+ {
+ priority: 'Critical',
+ category: 'Mobile SEO',
+ url: 'https://example.com',
+ message: 'Missing viewport',
+ },
+ ],
+ total: 1,
+ truncated: false,
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('issue_table');
+ });
+
+ it('builds category_scores from get_category_scores', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_category_scores', {
+ health_score: 80,
+ categories: [{ name: 'Technical SEO', score: 85, issue_count: 3 }],
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('category_scores');
+ });
+
+ it('builds multiple label_value_chart blocks', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_mime_type_breakdown', {
+ items: [
+ { label: 'text/html', value: 120 },
+ { label: 'image/png', value: 40 },
+ ],
+ }),
+ doneTool('get_title_length_distribution', {
+ items: [{ label: '0-30', value: 10 }],
+ }),
+ ]);
+ const charts = blocks.filter((b) => b.type === 'label_value_chart');
+ expect(charts).toHaveLength(2);
+ });
+
+ it('builds status_breakdown from get_status_code_breakdown', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_status_code_breakdown', {
+ status_counts: { '2xx': 100, '4xx': 5 },
+ summary: { success_rate: 95.2, total_urls: 105 },
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('status_breakdown');
+ });
+
+ it('builds health_trend from get_health_history', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_health_history', {
+ snapshots: [
+ { health_score: 70, generated_at: '2026-01-01T00:00:00' },
+ { health_score: 75, generated_at: '2026-02-01T00:00:00' },
+ ],
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('health_trend');
+ });
+
+ it('builds priority chart from get_report_summary counts', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_report_summary', {
+ health_score: 70,
+ issue_counts: { Critical: 2, High: 5 },
+ }),
+ ]);
+ const chart = blocks.find((b) => b.type === 'label_value_chart');
+ expect(chart).toBeDefined();
+ if (chart?.type === 'label_value_chart') {
+ expect(chart.title).toBe('Issues by priority');
+ }
+ });
+
+ it('builds issue_table from get_critical_issues', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_critical_issues', {
+ issues: [
+ {
+ priority: 'Critical',
+ category: 'Mobile SEO',
+ url: 'https://example.com/page',
+ message: 'Missing viewport',
+ },
+ ],
+ total: 1,
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('issue_table');
+ });
+
+ it('builds compare_category_deltas block', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('compare_category_deltas', {
+ category_scores: [
+ { id: 'crawl', name: 'Crawl', current: 50, baseline: 80, delta: -30 },
+ ],
+ }),
+ ]);
+ expect(blocks[0]?.type).toBe('compare_category_deltas');
+ });
+
+ it('dedupes block types across multiple tools', () => {
+ const blocks = deriveChatBlocks([
+ doneTool('get_report_summary', {
+ health_score: 70,
+ issue_counts: { High: 2 },
+ }),
+ doneTool('list_issues', {
+ issues: [{ priority: 'High', category: 'SEO', url: '', message: 'x' }],
+ }),
+ ]);
+ expect(blocks.map((b) => b.type)).toEqual([
+ 'issue_summary',
+ 'label_value_chart',
+ 'issue_table',
+ ]);
+ });
+
+ it('ignores tools without results or with errors', () => {
+ expect(deriveChatBlocks([{ id: '1', name: 'list_issues', status: 'running' }])).toEqual([]);
+ expect(deriveChatBlocks([doneTool('list_issues', { error: 'no report' })])).toEqual([]);
+ });
+
+ it('blockKey distinguishes label_value charts by title', () => {
+ expect(blockKey({ type: 'label_value_chart', title: 'MIME types', items: [] })).toBe(
+ 'label_value:MIME types',
+ );
+ });
+});
+
+describe('toolEventsToActivity', () => {
+ it('restores persisted tool_events', () => {
+ const activity = toolEventsToActivity({
+ tool_events: [
+ {
+ name: 'list_issues',
+ args: { limit: 5 },
+ result: { total: 0, issues: [] },
+ },
+ ],
+ });
+ expect(activity).toHaveLength(1);
+ expect(activity[0]?.name).toBe('list_issues');
+ expect(activity[0]?.status).toBe('done');
+ });
+});
diff --git a/web/src/components/chat/deriveChatBlocks.ts b/web/src/components/chat/deriveChatBlocks.ts
new file mode 100644
index 00000000..d2a45629
--- /dev/null
+++ b/web/src/components/chat/deriveChatBlocks.ts
@@ -0,0 +1,579 @@
+import type { ToolActivityItem } from '@/components/chat/ChatToolActivity';
+
+export interface IssueRow {
+ priority: string;
+ category: string;
+ url: string;
+ message: string;
+}
+
+export interface CategoryScoreRow {
+ name: string;
+ score?: number;
+ issue_count?: number;
+}
+
+export interface CompareCategoryRow {
+ id: string;
+ name: string;
+ current?: number | null;
+ baseline?: number | null;
+ delta?: number | null;
+}
+
+export interface HealthTrendPoint {
+ label: string;
+ score: number;
+}
+
+export interface GoogleQueryRow {
+ query: string;
+ clicks?: number;
+ impressions?: number;
+}
+
+export interface GooglePageRow {
+ page: string;
+ clicks?: number;
+}
+
+export type ChatBlock =
+ | {
+ type: 'issue_summary';
+ healthScore?: number;
+ counts: Record;
+ siteName?: string;
+ totalIssues?: number;
+ totalUrls?: number;
+ successRate?: number;
+ }
+ | {
+ type: 'issue_table';
+ issues: IssueRow[];
+ total?: number;
+ truncated?: boolean;
+ }
+ | {
+ type: 'category_scores';
+ healthScore?: number;
+ categories: CategoryScoreRow[];
+ }
+ | {
+ type: 'label_value_chart';
+ title: string;
+ items: { label: string; value: number }[];
+ }
+ | {
+ type: 'status_breakdown';
+ items: { label: string; value: number }[];
+ successRate?: number;
+ totalUrls?: number;
+ }
+ | {
+ type: 'health_trend';
+ title: string;
+ points: HealthTrendPoint[];
+ categoryId?: string;
+ }
+ | {
+ type: 'compare_category_deltas';
+ rows: CompareCategoryRow[];
+ }
+ | {
+ type: 'lighthouse_scores';
+ scores: Record;
+ poorPages: { url: string; performance: number }[];
+ }
+ | {
+ type: 'google_summary';
+ clicks?: number;
+ impressions?: number;
+ ctr?: number;
+ queries: GoogleQueryRow[];
+ pages: GooglePageRow[];
+ };
+
+const SUMMARY_TOOLS = new Set(['get_report_summary', 'get_executive_summary']);
+const ISSUE_TABLE_TOOLS = new Set([
+ 'list_issues',
+ 'list_issues_by_category',
+ 'get_category_issues',
+ 'get_critical_issues',
+ 'list_seo_onpage_issues',
+ 'list_issues_with_ai_fixes',
+]);
+const CATEGORY_SCORE_TOOLS = new Set([
+ 'get_category_scores',
+ 'get_report_summary',
+ 'list_audit_categories',
+]);
+const CHART_TOOLS: Record = {
+ get_issue_priority_breakdown: 'Issues by priority',
+ get_mime_type_breakdown: 'MIME types',
+ get_title_length_distribution: 'Title length',
+ get_domain_link_distribution: 'Top domains',
+ get_outlink_distribution: 'Outlinks',
+};
+const GOOGLE_SUMMARY_TOOLS = new Set([
+ 'get_google_summary',
+ 'get_gsc_top_queries',
+ 'get_gsc_top_pages',
+]);
+
+function asRecord(v: unknown): Record | null {
+ return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record) : null;
+}
+
+export function blockKey(block: ChatBlock): string {
+ switch (block.type) {
+ case 'label_value_chart':
+ return `label_value:${block.title}`;
+ case 'health_trend':
+ return block.categoryId ? `health_trend:${block.categoryId}` : 'health_trend';
+ default:
+ return block.type;
+ }
+}
+
+function humanizeIssueType(type: string): string {
+ return type
+ .replace(/_/g, ' ')
+ .replace(/\b\w/g, (c) => c.toUpperCase());
+}
+
+function parseIssueRow(raw: unknown, defaults?: Partial): IssueRow | null {
+ const row = asRecord(raw);
+ if (!row) return null;
+ const type = String(row.type || row.issue_type || '');
+ const category =
+ String(row.category || row.name || defaults?.category || '') ||
+ (type ? 'On-page SEO' : '');
+ const message =
+ String(row.message || row.description || defaults?.message || '') ||
+ (type ? humanizeIssueType(type) : '');
+ const url = String(row.url || row.page || defaults?.url || '');
+ if (!url && !message) return null;
+ return {
+ priority: String(row.priority || defaults?.priority || 'Medium'),
+ category,
+ url,
+ message,
+ };
+}
+
+function parseCounts(raw: unknown): Record {
+ const obj = asRecord(raw);
+ if (!obj) return {};
+ const out: Record = {};
+ for (const key of ['Critical', 'High', 'Medium', 'Low']) {
+ const n = Number(obj[key]);
+ if (Number.isFinite(n)) out[key] = n;
+ }
+ return out;
+}
+
+function parseCategories(catsRaw: unknown): CategoryScoreRow[] {
+ if (!Array.isArray(catsRaw) || !catsRaw.length) return [];
+ const out: CategoryScoreRow[] = [];
+ for (const c of catsRaw) {
+ const row = asRecord(c);
+ if (!row) continue;
+ const name = String(row.name || row.id || '');
+ if (!name) continue;
+ out.push({
+ name,
+ score: typeof row.score === 'number' ? row.score : undefined,
+ issue_count: typeof row.issue_count === 'number' ? row.issue_count : undefined,
+ });
+ }
+ return out;
+}
+
+function parseLabelValueItems(raw: unknown): { label: string; value: number }[] {
+ if (!Array.isArray(raw)) return [];
+ return raw
+ .map((item) => {
+ const row = asRecord(item);
+ if (!row) return null;
+ const value = Number(row.value);
+ if (!Number.isFinite(value)) return null;
+ return { label: String(row.label ?? ''), value };
+ })
+ .filter((i): i is { label: string; value: number } => i != null && i.label !== '');
+}
+
+function parseCrawlSummary(raw: unknown): {
+ totalUrls?: number;
+ successRate?: number;
+ statusItems: { label: string; value: number }[];
+} {
+ const crawl = asRecord(raw);
+ if (!crawl) return { statusItems: [] };
+ const totalUrls =
+ crawl.total_urls != null && Number.isFinite(Number(crawl.total_urls))
+ ? Number(crawl.total_urls)
+ : undefined;
+ const successRate =
+ crawl.success_rate != null && Number.isFinite(Number(crawl.success_rate))
+ ? Number(crawl.success_rate)
+ : undefined;
+ const statusItems: { label: string; value: number }[] = [];
+ for (const [key, label] of [
+ ['count_2xx', '2xx'],
+ ['count_3xx', '3xx'],
+ ['count_4xx', '4xx'],
+ ['count_5xx', '5xx'],
+ ] as const) {
+ const n = Number(crawl[key]);
+ if (Number.isFinite(n) && n > 0) statusItems.push({ label, value: n });
+ }
+ return { totalUrls, successRate, statusItems };
+}
+
+function blockFromSummary(name: string, result: Record): ChatBlock | null {
+ if (!SUMMARY_TOOLS.has(name)) return null;
+ if (result.error) return null;
+ const counts = parseCounts(result.issue_counts);
+ const crawl = parseCrawlSummary(result.crawl_summary);
+ if (
+ !Object.keys(counts).length &&
+ result.total_issues == null &&
+ result.health_score == null &&
+ crawl.totalUrls == null
+ ) {
+ return null;
+ }
+ return {
+ type: 'issue_summary',
+ healthScore: typeof result.health_score === 'number' ? result.health_score : undefined,
+ counts,
+ siteName: result.site_name != null ? String(result.site_name) : undefined,
+ totalIssues: typeof result.total_issues === 'number' ? result.total_issues : undefined,
+ totalUrls: crawl.totalUrls,
+ successRate: crawl.successRate,
+ };
+}
+
+function blockFromReportCrawlStatus(name: string, result: Record): ChatBlock | null {
+ if (name !== 'get_report_summary') return null;
+ if (result.error) return null;
+ const crawl = parseCrawlSummary(result.crawl_summary);
+ if (!crawl.statusItems.length) return null;
+ return {
+ type: 'status_breakdown',
+ items: crawl.statusItems,
+ successRate: crawl.successRate,
+ totalUrls: crawl.totalUrls,
+ };
+}
+
+function blockFromIssueTable(name: string, result: Record): ChatBlock | null {
+ if (!ISSUE_TABLE_TOOLS.has(name)) return null;
+ if (result.error) return null;
+ const issuesRaw = result.issues;
+ if (!Array.isArray(issuesRaw) || !issuesRaw.length) return null;
+ const defaultPriority = name === 'get_critical_issues' ? 'Critical' : undefined;
+ const issues = issuesRaw
+ .map((row) => parseIssueRow(row, defaultPriority ? { priority: defaultPriority } : undefined))
+ .filter((r): r is IssueRow => r != null);
+ if (!issues.length) return null;
+ return {
+ type: 'issue_table',
+ issues,
+ total: typeof result.total === 'number' ? result.total : issues.length,
+ truncated: Boolean(result.truncated),
+ };
+}
+
+function blockFromPriorityChart(name: string, result: Record): ChatBlock | null {
+ if (name !== 'get_report_summary') return null;
+ if (result.error) return null;
+ const counts = parseCounts(result.issue_counts);
+ const items = (['Critical', 'High', 'Medium', 'Low'] as const)
+ .map((label) => ({ label, value: counts[label] || 0 }))
+ .filter((i) => i.value > 0);
+ if (!items.length) return null;
+ return { type: 'label_value_chart', title: 'Issues by priority', items };
+}
+
+function blockFromCategoryScores(name: string, result: Record): ChatBlock | null {
+ if (!CATEGORY_SCORE_TOOLS.has(name)) return null;
+ if (result.error) return null;
+ const categories = parseCategories(result.categories);
+ if (!categories.length) return null;
+ return {
+ type: 'category_scores',
+ healthScore: typeof result.health_score === 'number' ? result.health_score : undefined,
+ categories,
+ };
+}
+
+function blockFromLabelValue(name: string, result: Record): ChatBlock | null {
+ const title = CHART_TOOLS[name];
+ if (!title) return null;
+ if (result.error) return null;
+ const items = parseLabelValueItems(result.items);
+ if (!items.length) return null;
+ return { type: 'label_value_chart', title, items };
+}
+
+function blockFromDepthDistribution(name: string, result: Record): ChatBlock | null {
+ if (name !== 'get_depth_distribution') return null;
+ if (result.error || result.missing) return null;
+ const data = asRecord(result.data) ?? result;
+ const byDepth = asRecord(data.by_depth) ?? data;
+ const entries = Object.entries(byDepth)
+ .map(([k, v]) => ({ depth: Number(k), value: Number(v) }))
+ .filter((e) => Number.isFinite(e.depth) && Number.isFinite(e.value))
+ .sort((a, b) => a.depth - b.depth);
+ if (!entries.length) return null;
+ return {
+ type: 'label_value_chart',
+ title: 'Crawl depth',
+ items: entries.map((e) => ({ label: `Depth ${e.depth}`, value: e.value })),
+ };
+}
+
+function blockFromStatusBreakdown(name: string, result: Record): ChatBlock | null {
+ if (name !== 'get_status_code_breakdown') return null;
+ if (result.error) return null;
+ const counts = asRecord(result.status_counts);
+ if (!counts) return null;
+ const items = Object.entries(counts)
+ .map(([label, value]) => ({ label, value: Number(value) }))
+ .filter((i) => Number.isFinite(i.value) && i.value > 0);
+ if (!items.length) return null;
+ const summary = asRecord(result.summary);
+ const successRate = summary?.success_rate != null ? Number(summary.success_rate) : undefined;
+ const totalUrls = summary?.total_urls != null ? Number(summary.total_urls) : undefined;
+ return {
+ type: 'status_breakdown',
+ items,
+ successRate: Number.isFinite(successRate) ? successRate : undefined,
+ totalUrls: Number.isFinite(totalUrls) ? totalUrls : undefined,
+ };
+}
+
+function blockFromHealthTrend(name: string, result: Record): ChatBlock | null {
+ if (name === 'get_health_history') {
+ if (result.error) return null;
+ const snapshots = result.snapshots;
+ if (!Array.isArray(snapshots) || !snapshots.length) return null;
+ const points: HealthTrendPoint[] = snapshots
+ .map((raw) => {
+ const row = asRecord(raw);
+ if (!row) return null;
+ const score = Number(row.health_score);
+ if (!Number.isFinite(score)) return null;
+ const at = String(row.generated_at || '');
+ const label = at ? at.slice(0, 10) : '—';
+ return { label, score };
+ })
+ .filter((p): p is HealthTrendPoint => p != null)
+ .reverse();
+ if (!points.length) return null;
+ return { type: 'health_trend', title: 'Health score trend', points };
+ }
+
+ if (name === 'get_category_health_history') {
+ if (result.error) return null;
+ const pointsRaw = result.points;
+ if (!Array.isArray(pointsRaw) || !pointsRaw.length) return null;
+ const categoryId = result.category_id != null ? String(result.category_id) : undefined;
+ const points: HealthTrendPoint[] = pointsRaw
+ .map((raw) => {
+ const row = asRecord(raw);
+ if (!row) return null;
+ const score = Number(categoryId ? row.category_score : row.health_score);
+ if (!Number.isFinite(score)) return null;
+ const at = String(row.generated_at || '');
+ return { label: at ? at.slice(0, 10) : '—', score };
+ })
+ .filter((p): p is HealthTrendPoint => p != null)
+ .reverse();
+ if (!points.length) return null;
+ return {
+ type: 'health_trend',
+ title: categoryId ? `Category trend (${categoryId})` : 'Health score trend',
+ points,
+ categoryId,
+ };
+ }
+
+ return null;
+}
+
+function blockFromCompareCategory(name: string, result: Record): ChatBlock | null {
+ if (name !== 'compare_category_deltas') return null;
+ if (result.error) return null;
+ const rowsRaw = result.category_scores;
+ if (!Array.isArray(rowsRaw) || !rowsRaw.length) return null;
+ const rows: CompareCategoryRow[] = [];
+ for (const raw of rowsRaw) {
+ const row = asRecord(raw);
+ if (!row) continue;
+ const name = String(row.name || row.id || '');
+ if (!name) continue;
+ rows.push({
+ id: String(row.id || row.name || ''),
+ name,
+ current: row.current != null ? Number(row.current) : null,
+ baseline: row.baseline != null ? Number(row.baseline) : null,
+ delta: row.delta != null ? Number(row.delta) : null,
+ });
+ }
+ if (!rows.length) return null;
+ return { type: 'compare_category_deltas', rows };
+}
+
+function blockFromLighthouse(name: string, result: Record): ChatBlock | null {
+ if (name !== 'get_lighthouse_summary') return null;
+ if (result.error) return null;
+ const summary = asRecord(result.summary);
+ const cs = asRecord(summary?.category_scores) ?? asRecord(result.category_scores);
+ if (!cs) return null;
+ const scores: Record = {};
+ for (const [key, val] of Object.entries(cs)) {
+ scores[key] = val != null && Number.isFinite(Number(val)) ? Number(val) : null;
+ }
+ if (!Object.keys(scores).length) return null;
+ const poorRaw = result.poor_performance_pages;
+ const poorPages: { url: string; performance: number }[] = [];
+ if (Array.isArray(poorRaw)) {
+ for (const raw of poorRaw.slice(0, 5)) {
+ const row = asRecord(raw);
+ if (!row) continue;
+ const perf = Number(row.performance);
+ if (!Number.isFinite(perf)) continue;
+ poorPages.push({ url: String(row.url || ''), performance: perf });
+ }
+ }
+ return { type: 'lighthouse_scores', scores, poorPages };
+}
+
+function blockFromGoogle(name: string, result: Record): ChatBlock | null {
+ if (!GOOGLE_SUMMARY_TOOLS.has(name)) return null;
+ if (result.error) return null;
+
+ if (name === 'get_gsc_top_queries') {
+ const queries = parseGoogleQueries(result.queries);
+ if (!queries.length) return null;
+ return { type: 'google_summary', queries, pages: [] };
+ }
+
+ if (name === 'get_gsc_top_pages') {
+ const pages = parseGooglePages(result.pages);
+ if (!pages.length) return null;
+ return { type: 'google_summary', queries: [], pages };
+ }
+
+ const gsc = asRecord(result.gsc);
+ const gscSummary = asRecord(gsc?.summary);
+ const queries = parseGoogleQueries(gsc?.top_queries);
+ const pages = parseGooglePages(gsc?.top_pages);
+ if (!gscSummary && !queries.length && !pages.length) return null;
+ return {
+ type: 'google_summary',
+ clicks: gscSummary?.clicks != null ? Number(gscSummary.clicks) : undefined,
+ impressions: gscSummary?.impressions != null ? Number(gscSummary.impressions) : undefined,
+ ctr: gscSummary?.ctr != null ? Number(gscSummary.ctr) : undefined,
+ queries,
+ pages,
+ };
+}
+
+function parseGoogleQueries(raw: unknown): GoogleQueryRow[] {
+ if (!Array.isArray(raw)) return [];
+ const out: GoogleQueryRow[] = [];
+ for (const item of raw) {
+ const row = asRecord(item);
+ if (!row) continue;
+ const query = String(row.query || '');
+ if (!query) continue;
+ out.push({
+ query,
+ clicks: row.clicks != null ? Number(row.clicks) : undefined,
+ impressions: row.impressions != null ? Number(row.impressions) : undefined,
+ });
+ if (out.length >= 10) break;
+ }
+ return out;
+}
+
+function parseGooglePages(raw: unknown): GooglePageRow[] {
+ if (!Array.isArray(raw)) return [];
+ const out: GooglePageRow[] = [];
+ for (const item of raw) {
+ const row = asRecord(item);
+ if (!row) continue;
+ const page = String(row.page || row.url || '');
+ if (!page) continue;
+ out.push({
+ page,
+ clicks: row.clicks != null ? Number(row.clicks) : undefined,
+ });
+ if (out.length >= 5) break;
+ }
+ return out;
+}
+
+type BlockParser = (name: string, result: Record) => ChatBlock | null;
+
+const BLOCK_PARSERS: BlockParser[] = [
+ blockFromSummary,
+ blockFromReportCrawlStatus,
+ blockFromIssueTable,
+ blockFromPriorityChart,
+ blockFromCategoryScores,
+ blockFromLabelValue,
+ blockFromDepthDistribution,
+ blockFromStatusBreakdown,
+ blockFromHealthTrend,
+ blockFromCompareCategory,
+ blockFromLighthouse,
+ blockFromGoogle,
+];
+
+export function deriveChatBlocks(toolActivity: ToolActivityItem[]): ChatBlock[] {
+ const blocks: ChatBlock[] = [];
+ const seen = new Set();
+
+ for (const item of toolActivity) {
+ if (item.status !== 'done' || !item.result) continue;
+ const result = item.result;
+ for (const parser of BLOCK_PARSERS) {
+ const block = parser(item.name, result);
+ if (!block) continue;
+ const key = blockKey(block);
+ if (seen.has(key)) continue;
+ seen.add(key);
+ blocks.push(block);
+ }
+ }
+
+ return blocks;
+}
+
+export interface PersistedToolEvent {
+ name: string;
+ args?: Record;
+ result?: Record;
+}
+
+export function toolEventsToActivity(
+ toolResult: Record | null | undefined,
+): ToolActivityItem[] {
+ if (!toolResult) return [];
+ const events = toolResult.tool_events;
+ if (!Array.isArray(events)) return [];
+ return events.map((raw, i) => {
+ const e = asRecord(raw) || {};
+ return {
+ id: `${String(e.name || 'tool')}-${i}`,
+ name: String(e.name || 'tool'),
+ args: asRecord(e.args) || undefined,
+ result: asRecord(e.result) || undefined,
+ status: 'done' as const,
+ };
+ });
+}
diff --git a/web/src/components/chat/parseChatSse.test.ts b/web/src/components/chat/parseChatSse.test.ts
new file mode 100644
index 00000000..108c8120
--- /dev/null
+++ b/web/src/components/chat/parseChatSse.test.ts
@@ -0,0 +1,31 @@
+import { describe, expect, it } from 'vitest';
+import { parseSseChunk } from './parseChatSse';
+
+describe('parseSseChunk', () => {
+ it('parses token and done events', () => {
+ const chunk =
+ 'event: token\ndata: {"text":"Hi"}\n\n' +
+ 'event: done\ndata: {"message":"Done"}\n\n';
+ const { events, rest } = parseSseChunk(chunk);
+ expect(events).toHaveLength(2);
+ expect(events[0]).toEqual({ type: 'token', text: 'Hi' });
+ expect(events[1]).toEqual({ type: 'done', message: 'Done' });
+ expect(rest).toBe('');
+ });
+
+ it('parses tool events', () => {
+ const chunk =
+ 'event: tool_start\ndata: {"name":"list_issues","args":{"limit":5}}\n\n';
+ const { events } = parseSseChunk(chunk);
+ expect(events[0]?.type).toBe('tool_start');
+ if (events[0]?.type === 'tool_start') {
+ expect(events[0].name).toBe('list_issues');
+ }
+ });
+
+ it('keeps incomplete block in rest buffer', () => {
+ const { events, rest } = parseSseChunk('event: token\ndata: {"text":');
+ expect(events).toHaveLength(0);
+ expect(rest).toContain('event: token');
+ });
+});
diff --git a/web/src/components/chat/parseChatSse.ts b/web/src/components/chat/parseChatSse.ts
new file mode 100644
index 00000000..4bb5c962
--- /dev/null
+++ b/web/src/components/chat/parseChatSse.ts
@@ -0,0 +1,88 @@
+export type ChatSseEvent =
+ | { type: 'token'; text: string }
+ | { type: 'status'; phase?: string; detail?: string }
+ | { type: 'tool_start'; name?: string; args?: Record }
+ | { type: 'tool_end'; name?: string; result?: Record }
+ | { type: 'done'; message?: string }
+ | { type: 'error'; message?: string };
+
+export function parseSseChunk(buffer: string): { events: ChatSseEvent[]; rest: string } {
+ const events: ChatSseEvent[] = [];
+ const parts = buffer.split('\n\n');
+ const rest = parts.pop() || '';
+
+ for (const block of parts) {
+ const lines = block.split('\n');
+ let eventType = 'message';
+ let dataLine = '';
+ for (const line of lines) {
+ if (line.startsWith('event: ')) {
+ eventType = line.slice(7).trim();
+ } else if (line.startsWith('data: ')) {
+ dataLine = line.slice(6);
+ }
+ }
+ if (!dataLine) continue;
+ try {
+ const data = JSON.parse(dataLine) as Record;
+ if (eventType === 'token') {
+ events.push({ type: 'token', text: String(data.text || '') });
+ } else if (eventType === 'status') {
+ events.push({
+ type: 'status',
+ phase: String(data.phase || ''),
+ detail: String(data.detail || ''),
+ });
+ } else if (eventType === 'tool_start') {
+ events.push({
+ type: 'tool_start',
+ name: String(data.name || ''),
+ args: (data.args as Record) || {},
+ });
+ } else if (eventType === 'tool_end') {
+ events.push({
+ type: 'tool_end',
+ name: String(data.name || ''),
+ result: (data.result as Record) || {},
+ });
+ } else if (eventType === 'done') {
+ events.push({ type: 'done', message: String(data.message || '') });
+ } else if (eventType === 'error') {
+ events.push({ type: 'error', message: String(data.message || 'Error') });
+ }
+ } catch {
+ /* ignore */
+ }
+ }
+
+ return { events, rest };
+}
+
+export async function consumeChatSse(
+ response: Response,
+ onEvent: (event: ChatSseEvent) => void,
+): Promise {
+ const reader = response.body?.getReader();
+ if (!reader) throw new Error('No response body');
+
+ const decoder = new TextDecoder();
+ let buffer = '';
+
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+ const parsed = parseSseChunk(buffer);
+ buffer = parsed.rest;
+ for (const evt of parsed.events) {
+ onEvent(evt);
+ }
+ }
+
+ if (buffer.trim()) {
+ const parsed = parseSseChunk(`${buffer}\n\n`);
+ for (const evt of parsed.events) {
+ onEvent(evt);
+ }
+ }
+}
diff --git a/web/src/components/chat/preprocessChatMarkdown.test.ts b/web/src/components/chat/preprocessChatMarkdown.test.ts
new file mode 100644
index 00000000..09638841
--- /dev/null
+++ b/web/src/components/chat/preprocessChatMarkdown.test.ts
@@ -0,0 +1,18 @@
+import { describe, expect, it } from 'vitest';
+import { preprocessChatMarkdown } from './preprocessChatMarkdown';
+
+describe('preprocessChatMarkdown', () => {
+ it('splits run-on section headings into markdown headings', () => {
+ const raw =
+ "Here's a quick read (health score: 74/100). What looks good Crawl health is clean — 30/30 URLs returned 2xx.";
+ const out = preprocessChatMarkdown(raw);
+ expect(out).toContain('### What looks good');
+ expect(out).toContain('Crawl health is clean');
+ });
+
+ it('normalizes em-dash bullets to list items', () => {
+ const raw = 'Fix these next.\n— Add viewport meta\n— Improve titles';
+ const out = preprocessChatMarkdown(raw);
+ expect(out).toContain('- Add viewport meta');
+ });
+});
diff --git a/web/src/components/chat/preprocessChatMarkdown.ts b/web/src/components/chat/preprocessChatMarkdown.ts
new file mode 100644
index 00000000..ce0d901b
--- /dev/null
+++ b/web/src/components/chat/preprocessChatMarkdown.ts
@@ -0,0 +1,42 @@
+/** Normalize assistant markdown so section titles and lists render with structure. */
+export function preprocessChatMarkdown(content: string): string {
+ let out = content.trim();
+ if (!out) return out;
+
+ const sectionHeadings = [
+ 'What looks good',
+ "What's working",
+ 'Strengths',
+ 'What needs attention',
+ 'Areas to improve',
+ 'Top priorities',
+ 'Issues to fix',
+ 'Recommendations',
+ 'Next steps',
+ ];
+
+ for (const title of sectionHeadings) {
+ const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ const re = new RegExp(`([.!?])\\s+(${escaped})\\s+`, 'gi');
+ out = out.replace(re, `$1\n\n### $2\n\n`);
+ const lineStart = new RegExp(`^(?!#{1,6}\\s)(${escaped})\\s+`, 'im');
+ out = out.replace(lineStart, `### $1\n\n`);
+ }
+
+ // Em-dash or hyphen bullets run together in one paragraph → list items
+ out = out.replace(/([.!?])\s+([—–-]\s+)/g, '$1\n$2');
+
+ const lines = out.split('\n');
+ const normalized: string[] = [];
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (/^[—–-]\s+/.test(trimmed) && !trimmed.startsWith('---')) {
+ normalized.push(trimmed.replace(/^[—–-]\s+/, '- '));
+ } else {
+ normalized.push(line);
+ }
+ }
+ out = normalized.join('\n');
+
+ return out.replace(/\n{3,}/g, '\n\n').trim();
+}
diff --git a/web/src/components/chat/stripRedundantMarkdown.test.ts b/web/src/components/chat/stripRedundantMarkdown.test.ts
new file mode 100644
index 00000000..38c0a652
--- /dev/null
+++ b/web/src/components/chat/stripRedundantMarkdown.test.ts
@@ -0,0 +1,91 @@
+import { describe, expect, it } from 'vitest';
+import { stripRedundantMarkdown } from './stripRedundantMarkdown';
+import type { ChatBlock } from './deriveChatBlocks';
+
+describe('stripRedundantMarkdown', () => {
+ it('removes category score table when category_scores block exists', () => {
+ const content = `## Summary
+
+| Category | Score |
+| --- | --- |
+| Crawl | 0 |
+| HTTPS | 100 |
+
+Focus on crawl coverage next.`;
+
+ const blocks: ChatBlock[] = [
+ {
+ type: 'category_scores',
+ categories: [{ name: 'Crawl', score: 0 }],
+ },
+ ];
+
+ const out = stripRedundantMarkdown(content, blocks);
+ expect(out).not.toContain('| Crawl | 0 |');
+ expect(out).toContain('Focus on crawl coverage next.');
+ });
+
+ it('leaves content unchanged when no matching blocks', () => {
+ const content = '| A | B |\n| --- | --- |\n| 1 | 2 |';
+ expect(stripRedundantMarkdown(content, [])).toBe(content);
+ });
+
+ it('strips audit opener when overview blocks exist', () => {
+ const content = `Here's a quick read on the latest audit for codefrydev.in (health score: 74/100, 30 URLs crawled, 100% success rate).
+
+### What needs attention
+Focus on mobile viewport tags.`;
+
+ const blocks: ChatBlock[] = [
+ {
+ type: 'issue_summary',
+ healthScore: 74,
+ counts: { High: 5 },
+ totalUrls: 30,
+ successRate: 1,
+ },
+ {
+ type: 'status_breakdown',
+ items: [{ label: '2xx', value: 30 }],
+ totalUrls: 30,
+ successRate: 1,
+ },
+ ];
+
+ const out = stripRedundantMarkdown(content, blocks);
+ expect(out).not.toContain('health score: 74');
+ expect(out).not.toContain('30 URLs crawled');
+ expect(out).toContain('What needs attention');
+ expect(out).toContain('mobile viewport');
+ });
+
+ it('strips issue details prose when issue_table block exists', () => {
+ const content = `## Recommendations
+Fix viewport tags on mobile templates.
+
+Issue details: Critical Issue 1. Pages missing viewport meta tag — Mobile SEO
+Priority: Critical
+Affected URLs (2):
+https://example.com/long-path/page-one
+https://example.com/long-path/page-two`;
+
+ const blocks: ChatBlock[] = [
+ {
+ type: 'issue_table',
+ issues: [
+ {
+ priority: 'Critical',
+ category: 'Mobile SEO',
+ url: 'https://example.com',
+ message: 'Missing viewport',
+ },
+ ],
+ },
+ ];
+
+ const out = stripRedundantMarkdown(content, blocks);
+ expect(out).toContain('Recommendations');
+ expect(out).not.toContain('Issue details');
+ expect(out).not.toContain('https://example.com/long-path');
+ });
+});
diff --git a/web/src/components/chat/stripRedundantMarkdown.ts b/web/src/components/chat/stripRedundantMarkdown.ts
new file mode 100644
index 00000000..6b0b0b88
--- /dev/null
+++ b/web/src/components/chat/stripRedundantMarkdown.ts
@@ -0,0 +1,128 @@
+import type { ChatBlock } from '@/components/chat/deriveChatBlocks';
+
+const GFM_TABLE_RE = /(\n|^)(\|[^\n]+\|\n\|[-:\s|]+\|\n(?:\|[^\n]+\|\n?)+)/g;
+
+function isCategoryScoreTable(headerRow: string): boolean {
+ const h = headerRow.toLowerCase();
+ return /category/.test(h) && /score/.test(h);
+}
+
+function isIssueTable(headerRow: string): boolean {
+ const h = headerRow.toLowerCase();
+ return (
+ (/priority/.test(h) || /severity/.test(h)) &&
+ (/category/.test(h) || /url/.test(h) || /issue/.test(h) || /message/.test(h))
+ );
+}
+
+function shouldStripTable(headerRow: string, blocks: ChatBlock[]): boolean {
+ for (const block of blocks) {
+ if (block.type === 'category_scores' && isCategoryScoreTable(headerRow)) return true;
+ if (block.type === 'issue_table' && isIssueTable(headerRow)) return true;
+ if (block.type === 'compare_category_deltas' && isCategoryScoreTable(headerRow)) return true;
+ if (block.type === 'google_summary' && /query|clicks|page/i.test(headerRow)) return true;
+ }
+ return false;
+}
+
+function stripTables(content: string, blocks: ChatBlock[]): string {
+ return content.replace(GFM_TABLE_RE, (match, prefix: string, table: string) => {
+ const firstLine = table.split('\n')[0] || '';
+ if (shouldStripTable(firstLine, blocks)) {
+ return prefix;
+ }
+ return match;
+ });
+}
+
+/** Remove prose issue dumps when structured issue blocks already render the data. */
+function stripIssueProse(content: string, blocks: ChatBlock[]): string {
+ const hasIssueViz = blocks.some(
+ (b) => b.type === 'issue_table' || b.type === 'issue_summary',
+ );
+ if (!hasIssueViz) return content;
+
+ let out = content;
+
+ // Drop "## Issue details" sections
+ out = out.replace(/\n?#{1,3}\s*Issue details[^\n]*\n[\s\S]*?(?=\n#{1,3}\s|$)/gi, '\n');
+
+ // Drop inline "Issue details:" preamble and following enumeration
+ const detailsIdx = out.search(/\bIssue details:/i);
+ if (detailsIdx >= 0) {
+ const before = out.slice(0, detailsIdx).trim();
+ if (before.length > 0) {
+ out = before;
+ }
+ }
+
+ const lines = out.split('\n');
+ const filtered = lines.filter((line) => {
+ const t = line.trim();
+ if (!t) return true;
+ if (/^https?:\/\//i.test(t)) return false;
+ if (/affected urls/i.test(t)) return false;
+ if (/^critical issue \d+/i.test(t)) return false;
+ if (/^\d+\.\s+.+(priority:|affected urls)/i.test(t)) return false;
+ if (/^priority:\s*(critical|high|medium|low)\s*$/i.test(t)) return false;
+ return true;
+ });
+
+ return filtered.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}
+
+function stripOverviewProse(content: string, blocks: ChatBlock[]): string {
+ const hasSummary = blocks.some(
+ (b) =>
+ b.type === 'issue_summary' ||
+ b.type === 'category_scores' ||
+ b.type === 'status_breakdown',
+ );
+ if (!hasSummary) return content;
+
+ let out = content;
+
+ // Drop opening audit recap (health score, crawl stats) when viz blocks cover it
+ out = out.replace(
+ /^[^\n#]*(?:here'?s|quick read|latest audit|overview)[^\n]*(?:health score|urls?\s+crawl|success rate)[^\n]*\n?/i,
+ '',
+ );
+ out = out.replace(
+ /^[^\n#]*health score:\s*\d+[^\n]*(?:urls?\s+crawl|success rate)[^\n]*\n?/i,
+ '',
+ );
+
+ if (blocks.some((b) => b.type === 'status_breakdown')) {
+ out = out.replace(
+ /\n?#{1,3}\s*what(?:'s| is)?\s+(?:working|good)[^\n]*\n[\s\S]*?(?=\n#{1,3}\s|$)/i,
+ '\n',
+ );
+ out = out.replace(
+ /^.*(?:urls?\s+(?:crawled|returned)|success rate|no\s+4xx|no\s+5xx|\d+\/\d+\s+urls?).*$/gim,
+ '',
+ );
+ }
+
+ if (blocks.some((b) => b.type === 'category_scores')) {
+ out = out.replace(
+ /^.*(?:category score|scored?\s+\d+\/100|\/100\s+in\s+\w+).*$/gim,
+ '',
+ );
+ }
+
+ if (blocks.some((b) => b.type === 'issue_summary')) {
+ out = out.replace(/^.*\b\d+\s+issues?\b.*$/gim, '');
+ out = out.replace(/^.*\b(?:critical|high|medium|low):\s*\d+.*$/gim, '');
+ }
+
+ return out.replace(/\n{3,}/g, '\n\n').trim();
+}
+
+/** Remove GFM tables and prose duplicated by structured chat blocks. */
+export function stripRedundantMarkdown(content: string, blocks: ChatBlock[]): string {
+ if (!content.trim() || !blocks.length) return content;
+ let out = stripTables(content, blocks);
+ out = stripIssueProse(out, blocks);
+ out = stripOverviewProse(out, blocks);
+ return out.replace(/\n{3,}/g, '\n\n').trim();
+}
diff --git a/web/src/components/compare/CompareTabPanels.tsx b/web/src/components/compare/CompareTabPanels.tsx
index e2f7701e..d95d92a6 100644
--- a/web/src/components/compare/CompareTabPanels.tsx
+++ b/web/src/components/compare/CompareTabPanels.tsx
@@ -12,31 +12,11 @@ const ComparePerformanceCharts = dynamic(
{ ssr: false },
);
import type { IssueDeltaRow } from '@/lib/reportCompareExtras';
-import { TrendingDown, TrendingUp, Minus } from 'lucide-react';
+import { ScoreDelta } from '@/components/charts/ScoreDelta';
import type { CompareMetricRow, ReportCompareSummary } from '@/lib/reportCompare';
import { Card, Table, TableHead, TableHeadCell, TableBody, TableRow, TableCell, Badge } from '@/components';
import { CompareMetricCard } from './CompareDeltaBadge';
-function ScoreDelta({ delta }: { delta: number | null }) {
- if (delta == null || delta === 0) {
- return (
-
- 0
-
- );
- }
- const up = delta > 0;
- const Icon = up ? TrendingUp : TrendingDown;
- const color = up ? 'text-emerald-700 dark:text-emerald-400' : 'text-rose-700 dark:text-rose-400';
- return (
-
-
- {up ? '+' : ''}
- {delta}
-
- );
-}
-
function matchesQuery(text: string, q: string): boolean {
return text.toLowerCase().includes(q);
}
diff --git a/web/src/components/overview/OverviewHealthTab.tsx b/web/src/components/overview/OverviewHealthTab.tsx
index c6bb0ec8..f8b37dd3 100644
--- a/web/src/components/overview/OverviewHealthTab.tsx
+++ b/web/src/components/overview/OverviewHealthTab.tsx
@@ -4,7 +4,7 @@ import { ChevronRight, Lightbulb } from 'lucide-react';
import type { ReportCategory, ReportPayload } from '@/types';
import { strings, format } from '@/lib/strings';
import { categoryDisplayName } from '@/lib/categoryDisplayNames';
-import { scoreBandColor } from '@/utils/chartPalette';
+import { CategoryScoreGauge } from '@/components/charts/CategoryScoreGauge';
import { Card } from '@/components';
import { OverviewTabPanel } from './OverviewTabPanel';
@@ -34,66 +34,14 @@ export function OverviewHealthTab({ data, categoriesFiltered, recommendationsFil
{data.categories && data.categories.length > 0 ? (
categoriesFiltered.length > 0 ? (
- {categoriesFiltered.map((cat, i) => {
- const score = cat.score != null ? Math.min(100, Math.max(0, cat.score)) : 0;
- const label = score >= 80 ? vo.scoreGood : score >= 50 ? vo.scoreNeeds : vo.scoreCritical;
- const labelCls =
- score >= 80
- ? 'text-green-700 dark:text-green-400'
- : score >= 50
- ? 'text-yellow-700 dark:text-yellow-400'
- : 'text-red-600 dark:text-red-500';
- const color = scoreBandColor(cat.score);
- const isCritical = score < 50;
- return (
-
-
-
-
- {cat.score != null ? cat.score : sj.na}
-
-
-
-
- {categoryDisplayName(String(cat.name ?? cat.id ?? ''))}
-
-
{label}
-
-
- );
- })}
+ {categoriesFiltered.map((cat, i) => (
+
+
+
+ ))}
) : (
{vo.noCategorySearch}
diff --git a/web/src/components/pipeline/OllamaModelPicker.tsx b/web/src/components/pipeline/OllamaModelPicker.tsx
new file mode 100644
index 00000000..082cd7b1
--- /dev/null
+++ b/web/src/components/pipeline/OllamaModelPicker.tsx
@@ -0,0 +1,267 @@
+'use client';
+
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { Circle, Cloud, HardDrive, Loader2, RefreshCw, Sparkles } from 'lucide-react';
+import { apiUrl } from '@/lib/publicBase';
+import { format, strings } from '@/lib/strings';
+
+const s = strings.pipelineRunner.ollama;
+
+export interface OllamaModelPickerProps {
+ model: string;
+ baseUrl: string;
+ disabled?: boolean;
+ onModelChange: (model: string) => void;
+}
+
+type OllamaBillingTier = 'free_local' | 'cloud_free' | 'cloud_pro';
+
+interface OllamaModelEntry {
+ name: string;
+ source: 'local' | 'cloud';
+ installed: boolean;
+ capabilities?: string[];
+ billing: OllamaBillingTier;
+ requires_subscription: boolean;
+}
+
+interface OllamaStatus {
+ ok: boolean;
+ error?: string;
+ cloudCatalogOk?: boolean;
+ catalogSource?: string;
+ cloudModelCount?: number;
+ models?: OllamaModelEntry[];
+}
+
+function billingLabel(tier: OllamaBillingTier): string {
+ if (tier === 'free_local') return s.billingFreeLocal;
+ if (tier === 'cloud_pro') return s.billingCloudPro;
+ return s.billingCloudFree;
+}
+
+function billingBadgeClass(tier: OllamaBillingTier): string {
+ if (tier === 'free_local') return 'border-emerald-500/30 bg-emerald-500/10 text-emerald-300';
+ if (tier === 'cloud_pro') return 'border-amber-500/30 bg-amber-500/10 text-amber-200';
+ return 'border-sky-500/30 bg-sky-500/10 text-sky-200';
+}
+
+function modelOptionLabel(m: OllamaModelEntry): string {
+ const parts = [m.name];
+ if (m.capabilities?.includes('tools')) parts.push('tools');
+ parts.push(billingLabel(m.billing));
+ return parts.join(' · ');
+}
+
+export default function OllamaModelPicker({
+ model,
+ baseUrl,
+ disabled,
+ onModelChange,
+}: OllamaModelPickerProps) {
+ const [status, setStatus] = useState(null);
+ const [loading, setLoading] = useState(false);
+ const [query, setQuery] = useState('');
+
+ const refresh = useCallback(async () => {
+ setLoading(true);
+ try {
+ const res = await fetch(apiUrl('/ollama/status'));
+ const data = (await res.json()) as OllamaStatus;
+ setStatus(data);
+ if (data.ok && data.models?.length && !model) {
+ const preferred =
+ data.models.find((m) => m.installed) ??
+ data.models.find((m) => m.source === 'cloud') ??
+ data.models[0];
+ onModelChange(preferred.name);
+ }
+ } catch {
+ setStatus({ ok: false, error: s.unreachable });
+ } finally {
+ setLoading(false);
+ }
+ }, [model, onModelChange]);
+
+ useEffect(() => {
+ void refresh();
+ }, [refresh, baseUrl]);
+
+ const connected = status?.ok === true;
+ const models = status?.models || [];
+ const selected = models.find((m) => m.name === model);
+
+ const filtered = useMemo(() => {
+ const q = query.trim().toLowerCase();
+ if (!q) return models;
+ return models.filter((m) => m.name.toLowerCase().includes(q));
+ }, [models, query]);
+
+ const installed = filtered.filter((m) => m.installed);
+ const cloud = filtered.filter((m) => m.source === 'cloud' && !m.installed);
+ const local = filtered.filter((m) => m.source === 'local' && !m.installed);
+
+ return (
+
+
+
+
+
{s.title}
+ {status?.catalogSource === 'live' ? (
+
+
+ {s.liveCatalog}
+
+ ) : null}
+
+
{s.hint}
+
+
+
+
+
+
+
+
+ {connected ? s.connected : status?.error || s.disconnected}
+
+
+ {status?.cloudCatalogOk ? (
+
+
+ {format(s.cloudCount, { count: status.cloudModelCount ?? cloud.length })}
+
+ ) : null}
+ {baseUrl ? (
+ ({baseUrl})
+ ) : null}
+
+
+
+
{s.billingLegendTitle}
+
+ -
+
+ {s.billingFreeLocal}
+
+ {s.billingLegendFreeLocal}
+
+ -
+
+ {s.billingCloudFree}
+
+ {s.billingLegendCloudFree}
+
+ -
+
+ {s.billingCloudPro}
+
+ {s.billingLegendCloudPro}
+
+
+
+
+
+
+ {connected && models.length ? (
+
+
setQuery(e.target.value)}
+ className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground placeholder:text-muted-foreground focus:border-blue-500/50 focus:outline-none focus:ring-2 focus:ring-blue-500/20"
+ />
+
+ {selected ? (
+
+ {s.selectedBilling}:
+
+ {billingLabel(selected.billing)}
+
+ {selected.capabilities?.includes('tools') ? (
+
+ tools
+
+ ) : null}
+
+ ) : null}
+ {!filtered.length ? (
+
{s.noMatches}
+ ) : null}
+
+ ) : (
+
onModelChange(e.target.value)}
+ className="w-full rounded-lg border border-default bg-brand-900 px-3 py-2 text-sm text-foreground focus:border-blue-500/50 focus:outline-none focus:ring-2 focus:ring-blue-500/20"
+ />
+ )}
+ {connected && !status?.cloudCatalogOk ? (
+
+
+ {s.cloudCatalogUnavailable}
+
+ ) : null}
+
+
+ );
+}
diff --git a/web/src/components/pipeline/PipelineRunPanel.tsx b/web/src/components/pipeline/PipelineRunPanel.tsx
index f728409e..8e228b29 100644
--- a/web/src/components/pipeline/PipelineRunPanel.tsx
+++ b/web/src/components/pipeline/PipelineRunPanel.tsx
@@ -19,6 +19,7 @@ import { usePipeline } from '@/context/PipelineContext';
import { useReadOnlySession } from '@/hooks/useReadOnlySession';
import {
PipelineStatusBadge,
+ PipelineStopButton,
PRESET_COPY,
PresetIcon,
} from './pipelineUi';
@@ -27,6 +28,7 @@ import { CRAWL_PRESETS, type CrawlPresetId } from '@/lib/crawlPresets';
import PipelineWizardProgress, { type WizardStep } from './PipelineWizardProgress';
import PipelineLogViewer from './PipelineLogViewer';
import CrawlAuthorizeCheckbox from './CrawlAuthorizeCheckbox';
+import PipelineRunPreviewCard from './PipelineRunPreviewCard';
const s = strings.pipelineRunner;
const crawlPresets = s.crawlPresets as Record;
@@ -56,6 +58,8 @@ export default function PipelineRunPanel() {
status,
log,
startUrl,
+ configState,
+ customCommand,
presetId,
handleStartUrlChange,
handlePresetChange,
@@ -63,6 +67,8 @@ export default function PipelineRunPanel() {
handleCrawlPresetChange,
setField,
run,
+ cancelJob,
+ stopping,
continueInBackground,
} = usePipeline();
const { readOnly } = useReadOnlySession();
@@ -318,6 +324,13 @@ export default function PipelineRunPanel() {
+
+
{busy ? (
-
+ <>
+
+
+ >
) : null}
{s.outputLabel}
-
+
+ {busy ? (
+
+ ) : null}
+
+
{log ? (
diff --git a/web/src/components/pipeline/PipelineRunPreviewCard.tsx b/web/src/components/pipeline/PipelineRunPreviewCard.tsx
new file mode 100644
index 00000000..75f19014
--- /dev/null
+++ b/web/src/components/pipeline/PipelineRunPreviewCard.tsx
@@ -0,0 +1,146 @@
+'use client';
+
+import { useState } from 'react';
+import { ChevronDown, ChevronUp, Clock, Layers, Settings2 } from 'lucide-react';
+import { strings } from '@/lib/strings';
+import {
+ buildPipelineRunPreview,
+ formatPipelineRunDuration,
+} from '@/lib/pipelineRunPreview';
+import type { PipelinePresetId } from '@/components/pipeline/pipelinePresets';
+import type { CrawlPresetId } from '@/lib/crawlPresets';
+import type { PipelineConfigState } from '@/types/api';
+
+const s = strings.pipelineRunner.runPreview;
+
+export interface PipelineRunPreviewCardProps {
+ presetId: PipelinePresetId;
+ configState: PipelineConfigState;
+ customCommand?: string;
+ crawlPresetId?: CrawlPresetId | '';
+}
+
+export default function PipelineRunPreviewCard({
+ presetId,
+ configState,
+ customCommand = '',
+ crawlPresetId = '',
+}: PipelineRunPreviewCardProps) {
+ const [configOpen, setConfigOpen] = useState(false);
+ const preview = buildPipelineRunPreview({
+ presetId,
+ configState,
+ customCommand,
+ crawlPresetId,
+ });
+
+ const duration = formatPipelineRunDuration(preview.timeMinSeconds, preview.timeMaxSeconds);
+
+ return (
+
+
+
+
+
+ {s.estimatedTime}: {duration}
+
+
+
+
+
+
+ {s.maxPagesLabel}
+
+
+ {preview.maxCrawlPages != null ? preview.maxCrawlPages.toLocaleString() : '—'}
+
+
+
+
+ {s.lighthousePagesLabel}
+
+
+ {preview.lighthousePages != null ? preview.lighthousePages.toLocaleString() : '—'}
+
+
+
+
+ {s.stepsLabel}
+
+
+ {preview.phases.length}
+
+
+
+
+
+
+
+ {s.whatRunsLabel}
+
+
+ {preview.phases.map((phase) => (
+ -
+
+
+
{phase.label}
+ {phase.detail ? (
+
{phase.detail}
+ ) : null}
+
+
+ ))}
+
+
+
+
+ {preview.summaryLines.map((line) => (
+ - {line}
+ ))}
+ - {s.estimateDisclaimer}
+
+
+ {preview.configRows.length > 0 ? (
+
+
+ {configOpen ? (
+
+ {preview.configRows.map((row) => (
+
+
-
+ {row.label}
+
+ - {row.value}
+
+ ))}
+
+ ) : null}
+
+ ) : null}
+
+ );
+}
diff --git a/web/src/components/pipeline/PipelineRunnerFab.tsx b/web/src/components/pipeline/PipelineRunnerFab.tsx
index 21f72586..1f8e47ba 100644
--- a/web/src/components/pipeline/PipelineRunnerFab.tsx
+++ b/web/src/components/pipeline/PipelineRunnerFab.tsx
@@ -1,6 +1,6 @@
'use client';
-import { Loader2, Maximize2, Terminal } from 'lucide-react';
+import { Loader2, Maximize2, Square, Terminal } from 'lucide-react';
import { usePathname, useRouter, useSearchParams } from 'next/navigation';
import { strings } from '@/lib/strings';
import { usePipeline } from '@/context/PipelineContext';
@@ -16,10 +16,11 @@ export default function PipelineRunnerFab() {
const pathname = usePathname();
const router = useRouter();
const searchParams = useSearchParams();
- const { busy, status, log, backgroundMode, openPipelinePage } = usePipeline();
+ const { busy, status, log, backgroundMode, stopping, cancelJob, openPipelinePage } = usePipeline();
const { loading: sessionLoading, canMutate } = useSession();
const onPipelinePage = pathname === '/pipeline' || pathname.startsWith('/pipeline/');
+ const isHomePage = pathname === '/home';
const showDock = backgroundMode && (busy || Boolean(status) || Boolean(log));
const goToPipeline = () => {
@@ -31,7 +32,7 @@ export default function PipelineRunnerFab() {
openPipelinePage('run');
};
- if (onPipelinePage || sessionLoading || !canMutate) {
+ if (!isHomePage || onPipelinePage || sessionLoading || !canMutate) {
return null;
}
@@ -65,6 +66,20 @@ export default function PipelineRunnerFab() {
: 'Idle'}
+