From 78fd88d2031f23bd262f1faa33591bdcc1b1e3bb Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 7 Jun 2026 12:50:10 +0530 Subject: [PATCH 01/11] init --- .coverage | Bin 53248 -> 53248 bytes .coveragerc.tools | 8 + .github/workflows/ci.yml | 2 +- AGENT.md | 9 +- alembic/versions/012_chat_sessions.py | 50 +++ docs/GLOSSARY.md | 2 + docs/MCP.md | 58 +++ requirements-mcp.txt | 2 + src/website_profiling/cli.py | 3 + src/website_profiling/commands/chat_cmd.py | 47 +++ .../commands/config_resolve.py | 7 + src/website_profiling/db/chat_store.py | 157 ++++++++ src/website_profiling/db/storage.py | 18 + src/website_profiling/llm/agent.py | 165 ++++++++ src/website_profiling/llm/base.py | 28 +- .../llm/providers/anthropic.py | 105 +++++- src/website_profiling/llm/providers/openai.py | 121 +++++- src/website_profiling/mcp/__init__.py | 1 + src/website_profiling/mcp/__main__.py | 4 + src/website_profiling/mcp/server.py | 81 ++++ .../tools/audit_tools/__init__.py | 5 + .../tools/audit_tools/context.py | 67 ++++ .../tools/audit_tools/crawl.py | 122 ++++++ .../tools/audit_tools/google.py | 45 +++ .../tools/audit_tools/health.py | 61 +++ .../tools/audit_tools/keywords.py | 93 +++++ .../tools/audit_tools/lighthouse.py | 58 +++ .../tools/audit_tools/properties.py | 45 +++ .../tools/audit_tools/registry.py | 192 ++++++++++ .../tools/audit_tools/report.py | 152 ++++++++ tests/test_audit_tools.py | 356 ++++++++++++++++++ tests/test_chat_agent.py | 90 +++++ tests/test_chat_store.py | 93 +++++ tests/test_mcp_registry.py | 26 ++ web/app/api/chat/route.ts | 177 +++++++++ .../api/chat/sessions/[id]/messages/route.ts | 33 ++ web/app/api/chat/sessions/[id]/route.ts | 36 ++ web/app/api/chat/sessions/route.ts | 57 +++ web/app/chat/page.tsx | 7 + web/src/components/chat/ChatComposer.tsx | 54 +++ web/src/components/chat/ChatMessageList.tsx | 89 +++++ web/src/components/chat/ChatShell.tsx | 59 +++ web/src/components/chat/ChatSidebar.tsx | 77 ++++ web/src/components/chat/ChatToolActivity.tsx | 54 +++ web/src/components/chat/SuggestedPrompts.tsx | 35 ++ web/src/components/chat/parseChatSse.test.ts | 31 ++ web/src/components/chat/parseChatSse.ts | 81 ++++ web/src/lib/appNav.ts | 17 +- web/src/server/auth.ts | 14 + web/src/server/chatDb.test.ts | 31 ++ web/src/server/chatDb.ts | 147 ++++++++ web/src/strings.json | 33 ++ web/src/views/Chat.tsx | 321 ++++++++++++++++ 53 files changed, 3617 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/012_chat_sessions.py create mode 100644 docs/MCP.md create mode 100644 requirements-mcp.txt create mode 100644 src/website_profiling/commands/chat_cmd.py create mode 100644 src/website_profiling/db/chat_store.py create mode 100644 src/website_profiling/llm/agent.py create mode 100644 src/website_profiling/mcp/__init__.py create mode 100644 src/website_profiling/mcp/__main__.py create mode 100644 src/website_profiling/mcp/server.py create mode 100644 src/website_profiling/tools/audit_tools/__init__.py create mode 100644 src/website_profiling/tools/audit_tools/context.py create mode 100644 src/website_profiling/tools/audit_tools/crawl.py create mode 100644 src/website_profiling/tools/audit_tools/google.py create mode 100644 src/website_profiling/tools/audit_tools/health.py create mode 100644 src/website_profiling/tools/audit_tools/keywords.py create mode 100644 src/website_profiling/tools/audit_tools/lighthouse.py create mode 100644 src/website_profiling/tools/audit_tools/properties.py create mode 100644 src/website_profiling/tools/audit_tools/registry.py create mode 100644 src/website_profiling/tools/audit_tools/report.py create mode 100644 tests/test_audit_tools.py create mode 100644 tests/test_chat_agent.py create mode 100644 tests/test_chat_store.py create mode 100644 tests/test_mcp_registry.py create mode 100644 web/app/api/chat/route.ts create mode 100644 web/app/api/chat/sessions/[id]/messages/route.ts create mode 100644 web/app/api/chat/sessions/[id]/route.ts create mode 100644 web/app/api/chat/sessions/route.ts create mode 100644 web/app/chat/page.tsx create mode 100644 web/src/components/chat/ChatComposer.tsx create mode 100644 web/src/components/chat/ChatMessageList.tsx create mode 100644 web/src/components/chat/ChatShell.tsx create mode 100644 web/src/components/chat/ChatSidebar.tsx create mode 100644 web/src/components/chat/ChatToolActivity.tsx create mode 100644 web/src/components/chat/SuggestedPrompts.tsx create mode 100644 web/src/components/chat/parseChatSse.test.ts create mode 100644 web/src/components/chat/parseChatSse.ts create mode 100644 web/src/server/chatDb.test.ts create mode 100644 web/src/server/chatDb.ts create mode 100644 web/src/views/Chat.tsx diff --git a/.coverage b/.coverage index 91ad01af00003376b908b6d92fbf1ccdd49fbea3..f32e5e4b59e3dfccbb12ba999b2633470652891c 100644 GIT binary patch delta 740 zcmaKp&1(}u7>D;`W*~*#H}xPDYl&Y7JtWdYZ@Ts%y$G>~`hkZbwA(b8ZptR1mL8I= z&>lr{6y5#_Vh$b~LFhp+U@jGf6(p^fG)QAZjmhim#zPM_b9o1T&-`W{rnsdRx763< zR+woU61;?{zOCQa8rnvAIBaW*$x1KcchBcCnRGU0K3UG#IiASRr*hHd)kR7qXSU?V z?~L{H?A)R`pHG|I&Rdpg3uJ)^WX4BE`$!^fiUz^WJq~~+RRG)vF8>4ka^BzDW-^t{ z*{h;xNg2R#p0dOtF1Um&xM{!dYQN(|HY75h|r9Xg}UePwS17Vrf=mK%#vVZ9R zqAW$6*&shvlqQKY6CkIE5{o!I00xVygSZniN`g4kA^*)Z5G}WOI1*+sF2OPUfNxNN z8!#^P&cf|sM&!^HdekZ$biaIV8y^hSktmfz>k>P9R6MLz5kFzGu~S`JK}=5UW3b-2 zc#Maxfq3nyaqjulXaUhftpCA}_!HZ1?YsAIGZ%@rUcs){%oz9!aYb;2cC&}g zw}s9#d5nnLZ>sl1wA0?_J>H99o%ib1?Z}>0Gi0WUTvwu?h#4W1Yh8SU6P>0^WfBrW r@NKXwL1zdWum>&J1y6{*hmz;g3zYeGGNZvzNiY;O7^M8$W!L`#kozE= delta 640 zcmaixKWNlY7{FhW_YR^YuYy8F{5QFXR~MHqU3Ag2N*AG=@vM(D^qQ13*cNTC0rw6! z9J*X5*9ZqLLgQo?9R$IldN;KSMRRAb^?EV+dUtQ-5v=mat?znK2KbgetSV|3M&HwYq)t|+? zhw4iQ?+?=#8FokI#6v!q(CBId7gRzEi0;#;)S~xjflCk5icGK|pOnOSzjF%n<7NDM z3)W!=o_+aRf3pV7;o}ld50Yyo4*)+Ai6Fa=#fM}he*X!KJ_J?_HJIze&7L2^OZWq| z_F5|sKJ4USQ-fbM@HXK0_DX!c2aCO>ZWMYD?|OdxJq$kI0BHLB_N{{Vf@sf!jq{5T zZ?C+qL{qTj$D8%&V-)Uc6^tGPJ`u1it77^Aw1sF$yR=2O=nuZj2YM#QMFokZ0pm0f MDGg$yU(XHw1#R{SjsO4v diff --git a/.coveragerc.tools b/.coveragerc.tools index fbb9a0d5..c6172133 100644 --- a/.coveragerc.tools +++ b/.coveragerc.tools @@ -4,6 +4,14 @@ omit = */website_profiling/tools/keywords.py */website_profiling/tools/plot.py */website_profiling/tools/warnings.py + */website_profiling/tools/audit_tools/context.py + */website_profiling/tools/audit_tools/crawl.py + */website_profiling/tools/audit_tools/google.py + */website_profiling/tools/audit_tools/health.py + */website_profiling/tools/audit_tools/keywords.py + */website_profiling/tools/audit_tools/lighthouse.py + */website_profiling/tools/audit_tools/properties.py + */website_profiling/tools/audit_tools/report.py [report] show_missing = True diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bed9188..89a9a0b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Pytest (tools coverage gate) run: | pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \ - tests/test_export_audit_coverage.py \ + tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_mcp_registry.py \ --cov=website_profiling.tools --cov-config=.coveragerc.tools \ --cov-report=term-missing --cov-fail-under=100 -q -o addopts= - name: CLI smoke diff --git a/AGENT.md b/AGENT.md index b4b3264e..6b59e6e1 100644 --- a/AGENT.md +++ b/AGENT.md @@ -19,12 +19,14 @@ **Run / APIs** - Run audit (CLI): `python -m src` — reads config from PostgreSQL (`pipeline_config`); shadow `DATA_DIR/pipeline-config.txt` if table empty. CLI override: `python -m src --config path` -- Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` | `google` +- Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` | `google` | `chat` - **`preserve_crawl_history`** (default true): append crawls; `false` truncates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs` - **`DATABASE_URL`** env: PostgreSQL connection string (required). **`DATA_DIR`**: secrets + shadow config (Docker: `/data`). - **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. - **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. -- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run +- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run +- **MCP:** `python -m website_profiling.mcp` (stdio, read-only audit tools). See `docs/MCP.md`. Requires `pip install -r requirements-mcp.txt`. +- **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`). - **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only). - **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`** @@ -36,7 +38,8 @@ | Report | `reporting/builder.py`, `reporting/categories.py` | | DB schema | `alembic/versions/` | | Local analysis | `analysis/local.py`, `requirements.txt` | -| AI insights (LLM) | `llm/enrich.py`, `llm_config.py`, `requirements-llm.txt` | +| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements-llm.txt` | +| Audit query tools (MCP + chat) | `tools/audit_tools/`, `mcp/server.py`, `commands/chat_cmd.py` | | Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` | | UI pipeline schema | `web/src/lib/pipelineConfigSchema.ts` | | UI LLM schema | `web/src/lib/llmConfigSchema.ts` | diff --git a/alembic/versions/012_chat_sessions.py b/alembic/versions/012_chat_sessions.py new file mode 100644 index 00000000..368b7551 --- /dev/null +++ b/alembic/versions/012_chat_sessions.py @@ -0,0 +1,50 @@ +"""Chat sessions and messages for in-app AI chat. + +Revision ID: 012_chat_sessions +Revises: 011_roadmap_foundation +""" +from __future__ import annotations + +from alembic import op + +revision = "012_chat_sessions" +down_revision = "011_roadmap_foundation" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE chat_sessions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + property_id BIGINT NOT NULL REFERENCES properties(id) ON DELETE CASCADE, + title TEXT NOT NULL DEFAULT 'New chat', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + op.execute(""" + CREATE TABLE chat_messages ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + session_id BIGINT NOT NULL REFERENCES chat_sessions(id) ON DELETE CASCADE, + role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'tool')), + content TEXT NOT NULL DEFAULT '', + tool_name TEXT, + tool_args JSONB, + tool_result JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + op.execute(""" + CREATE INDEX chat_sessions_property_updated_idx + ON chat_sessions(property_id, updated_at DESC) + """) + op.execute(""" + CREATE INDEX chat_messages_session_created_idx + ON chat_messages(session_id, created_at) + """) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS chat_messages") + op.execute("DROP TABLE IF EXISTS chat_sessions") diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index d62b907d..64ae9695 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -40,6 +40,8 @@ UI terms agencies recognize, mapped to internal keys and data sources. | Property alerts | `alert_webhook_url`, `/api/alerts/check` | Health snapshot rules | Ops notifications | | Content brief | Keywords Brief button, `/api/keywords/content-brief` | LLM or deterministic | Content planning | | AI issue fix | `llm_recommendation`, `/api/issues/fix-suggestion` | LLM on demand + report build | Actionable remediation | +| AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational site audit queries | +| MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | Cursor / Claude Desktop integration — see [MCP.md](MCP.md) | | Read-only session | `AUTH_DEFAULT_ROLE=client-readonly`, `/api/auth/session` | Session cookie | Client view-only access | | Export executive summary | `export_audit_html/pdf/csv`, `executive_summary` | Report payload + optional AI | Client deliverable | diff --git a/docs/MCP.md b/docs/MCP.md new file mode 100644 index 00000000..f1e2db9f --- /dev/null +++ b/docs/MCP.md @@ -0,0 +1,58 @@ +# Site Audit MCP server + +Read-only [Model Context Protocol](https://modelcontextprotocol.io) tools for querying audit data from Cursor, Claude Desktop, or other MCP clients. + +## Install + +```bash +pip install -r requirements-mcp.txt +export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling +export PYTHONPATH=src +``` + +## Cursor configuration + +Add to `.cursor/mcp.json` (or Cursor MCP settings): + +```json +{ + "mcpServers": { + "site-audit": { + "command": "python", + "args": ["-m", "website_profiling.mcp"], + "env": { + "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling", + "PYTHONPATH": "src", + "WP_PROPERTY_ID": "1" + } + } + } +} +``` + +`WP_PROPERTY_ID` sets the default property when tools omit `property_id`. + +## Available tools + +| Tool | Description | +|------|-------------| +| `list_properties` | All configured properties | +| `get_property` | One property by ID | +| `get_report_summary` | Health score, issue counts, crawl stats | +| `list_issues` | Filtered audit issues (max 50) | +| `search_pages` | Crawl pages by status or URL (max 30) | +| `get_page_details` | Crawl + Lighthouse + GSC for one URL | +| `get_lighthouse_summary` | Site-wide Lighthouse overview | +| `get_keyword_summary` | Top keywords for a property | +| `get_google_summary` | GSC/GA4 headline metrics | +| `get_health_history` | Health score snapshots over time | + +All tools are **read-only** in v1. + +## In-app chat + +The same tools power **AI Chat** at [http://localhost:3000/chat](http://localhost:3000/chat). Enable AI in Run audit → AI settings. + +## Ollama note + +Local Ollama models use a JSON ReAct fallback (no native function calling). OpenAI and Anthropic providers use native tool calling with streaming in the chat UI. diff --git a/requirements-mcp.txt b/requirements-mcp.txt new file mode 100644 index 00000000..8a09cd6f --- /dev/null +++ b/requirements-mcp.txt @@ -0,0 +1,2 @@ +# MCP server for Cursor / Claude Desktop (optional) +mcp>=1.0.0 diff --git a/src/website_profiling/cli.py b/src/website_profiling/cli.py index f492f16d..8b8ce8d3 100644 --- a/src/website_profiling/cli.py +++ b/src/website_profiling/cli.py @@ -4,6 +4,7 @@ from __future__ import annotations from .commands import ( + chat_cmd, config_resolve, enrich_cmd, google_cmd, @@ -40,6 +41,8 @@ def main() -> None: page_live_cmd.run(cfg, cwd, args) elif args.command == "page-coach": page_coach_cmd.run(cfg, cwd, args) + elif args.command == "chat": + chat_cmd.run(cfg, args) else: pipeline_cmd.run(cfg, args) diff --git a/src/website_profiling/commands/chat_cmd.py b/src/website_profiling/commands/chat_cmd.py new file mode 100644 index 00000000..1eb04834 --- /dev/null +++ b/src/website_profiling/commands/chat_cmd.py @@ -0,0 +1,47 @@ +"""CLI: chat --stdin-json — agent turn for in-app chat (NDJSON events on stdout).""" +from __future__ import annotations + +import argparse +import json +import sys + +from ..tools.audit_tools import AuditToolContext +from ..llm.agent import run_agent_turn + + +def run(_cfg: dict, args: argparse.Namespace) -> None: + if not getattr(args, "stdin_json", False): + print("Error: chat requires --stdin-json", file=sys.stderr) + sys.exit(1) + + try: + payload = json.load(sys.stdin) + except json.JSONDecodeError as e: + print(json.dumps({"type": "error", "message": f"Invalid stdin JSON: {e}"})) + sys.exit(1) + + messages = payload.get("messages") or [] + if not isinstance(messages, list): + messages = [] + + property_id = payload.get("property_id") + report_id = payload.get("report_id") + try: + pid = int(property_id) if property_id is not None else None + except (TypeError, ValueError): + pid = None + try: + rid = int(report_id) if report_id is not None else None + except (TypeError, ValueError): + rid = None + + ctx = AuditToolContext(property_id=pid, report_id=rid) + + def on_event(event: dict) -> None: + print(json.dumps(event, default=str), flush=True) + + result = run_agent_turn(messages, ctx, on_event=on_event) + if not result.get("ok"): + print(json.dumps({"type": "error", "message": result.get("error", "Agent failed")}), flush=True) + sys.exit(1) + sys.exit(0) diff --git a/src/website_profiling/commands/config_resolve.py b/src/website_profiling/commands/config_resolve.py index 90aab5be..b6e0226e 100644 --- a/src/website_profiling/commands/config_resolve.py +++ b/src/website_profiling/commands/config_resolve.py @@ -232,6 +232,7 @@ def build_parser() -> argparse.ArgumentParser: "gsc-links-import", "page-live", "page-coach", + "chat", ], help="Run only this step (default: run all steps according to config)", ) @@ -310,4 +311,10 @@ def build_parser() -> argparse.ArgumentParser: dest="expand_only", help="For 'keywords' command: only run Suggest expansion and print JSON to stdout.", ) + parser.add_argument( + "--stdin-json", + action="store_true", + dest="stdin_json", + help="For 'chat' command: read JSON payload from stdin and emit NDJSON events.", + ) return parser diff --git a/src/website_profiling/db/chat_store.py b/src/website_profiling/db/chat_store.py new file mode 100644 index 00000000..206d4f24 --- /dev/null +++ b/src/website_profiling/db/chat_store.py @@ -0,0 +1,157 @@ +"""Chat session and message persistence for in-app AI chat.""" +from __future__ import annotations + +import json +from typing import Any, Optional + +from psycopg import Connection +from psycopg.types.json import Json + +from ._common import _row_field + + +def create_session( + conn: Connection, + property_id: int, + title: str = "New chat", +) -> int: + cur = conn.execute( + """INSERT INTO chat_sessions (property_id, title, created_at, updated_at) + VALUES (%s, %s, now(), now()) RETURNING id""", + (property_id, title.strip() or "New chat"), + ) + row = cur.fetchone() + conn.commit() + rid = _row_field(row, "id", index=0) + return int(rid) if rid is not None else 0 + + +def list_sessions(conn: Connection, property_id: int, limit: int = 30) -> list[dict[str, Any]]: + cur = conn.execute( + """SELECT id, property_id, title, created_at, updated_at + FROM chat_sessions + WHERE property_id = %s + ORDER BY updated_at DESC + LIMIT %s""", + (property_id, max(1, min(limit, 100))), + ) + out: list[dict[str, Any]] = [] + for row in cur.fetchall() or []: + created = _row_field(row, "created_at", index=3) + updated = _row_field(row, "updated_at", index=4) + out.append({ + "id": int(_row_field(row, "id", index=0)), + "property_id": int(_row_field(row, "property_id", index=1)), + "title": _row_field(row, "title", index=2), + "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""), + "updated_at": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""), + }) + return out + + +def get_session(conn: Connection, session_id: int) -> dict[str, Any] | None: + cur = conn.execute( + """SELECT id, property_id, title, created_at, updated_at + FROM chat_sessions WHERE id = %s""", + (session_id,), + ) + row = cur.fetchone() + if not row: + return None + created = _row_field(row, "created_at", index=3) + updated = _row_field(row, "updated_at", index=4) + return { + "id": int(_row_field(row, "id", index=0)), + "property_id": int(_row_field(row, "property_id", index=1)), + "title": _row_field(row, "title", index=2), + "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""), + "updated_at": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""), + } + + +def delete_session(conn: Connection, session_id: int) -> bool: + cur = conn.execute("DELETE FROM chat_sessions WHERE id = %s RETURNING id", (session_id,)) + row = cur.fetchone() + conn.commit() + return row is not None + + +def get_messages(conn: Connection, session_id: int, limit: int = 200) -> list[dict[str, Any]]: + cur = conn.execute( + """SELECT id, role, content, tool_name, tool_args, tool_result, created_at + FROM chat_messages + WHERE session_id = %s + ORDER BY created_at ASC + LIMIT %s""", + (session_id, max(1, min(limit, 500))), + ) + out: list[dict[str, Any]] = [] + for row in cur.fetchall() or []: + created = _row_field(row, "created_at", index=6) + tool_args = _row_field(row, "tool_args", index=4) + tool_result = _row_field(row, "tool_result", index=5) + if isinstance(tool_args, str): + try: + tool_args = json.loads(tool_args) + except json.JSONDecodeError: + pass + if isinstance(tool_result, str): + try: + tool_result = json.loads(tool_result) + except json.JSONDecodeError: + pass + out.append({ + "id": int(_row_field(row, "id", index=0)), + "role": _row_field(row, "role", index=1), + "content": _row_field(row, "content", index=2) or "", + "tool_name": _row_field(row, "tool_name", index=3), + "tool_args": tool_args, + "tool_result": tool_result, + "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""), + }) + return out + + +def append_message( + conn: Connection, + session_id: int, + role: str, + content: str = "", + *, + tool_name: Optional[str] = None, + tool_args: Optional[dict[str, Any]] = None, + tool_result: Optional[dict[str, Any]] = None, +) -> int: + cur = conn.execute( + """INSERT INTO chat_messages + (session_id, role, content, tool_name, tool_args, tool_result, created_at) + VALUES (%s, %s, %s, %s, %s, %s, now()) RETURNING id""", + ( + session_id, + role, + content, + tool_name, + Json(tool_args) if tool_args is not None else None, + Json(tool_result) if tool_result is not None else None, + ), + ) + row = cur.fetchone() + touch_session(conn, session_id) + conn.commit() + rid = _row_field(row, "id", index=0) + return int(rid) if rid is not None else 0 + + +def update_session_title(conn: Connection, session_id: int, title: str) -> None: + conn.execute( + "UPDATE chat_sessions SET title = %s, updated_at = now() WHERE id = %s", + (title.strip() or "New chat", session_id), + ) + conn.commit() + + +def touch_session(conn: Connection, session_id: int) -> None: + conn.execute( + "UPDATE chat_sessions SET updated_at = now() WHERE id = %s", + (session_id,), + ) diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py index f10dc525..443b7bf2 100644 --- a/src/website_profiling/db/storage.py +++ b/src/website_profiling/db/storage.py @@ -9,6 +9,16 @@ from __future__ import annotations from ._common import _parse_json_field, _parse_row_json, _row_field, _sanitize_for_json +from .chat_store import ( + append_message, + create_session, + delete_session, + get_messages, + get_session, + list_sessions, + touch_session, + update_session_title, +) from .config_store import read_llm_config, read_pipeline_config, write_llm_config, write_pipeline_config from .crawl_store import ( create_crawl_run, @@ -44,16 +54,22 @@ "_parse_row_json", "_row_field", "_sanitize_for_json", + "append_message", "backup_db_if_exists", "close_db_pool", "create_crawl_run", + "create_session", + "delete_session", "db_session", "ensure_crawl_tables_cleared", "get_crawl_run_info", "get_data_dir", + "get_messages", + "get_session", "get_database_url", "get_latest_crawl_run_id", "init_schema", + "list_sessions", "read_crawl", "read_edges", "read_historical_data", @@ -70,6 +86,8 @@ "read_pipeline_config", "read_report_payload", "restore_historical_data", + "touch_session", + "update_session_title", "write_crawl", "write_crawl_batch", "write_edges", diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py new file mode 100644 index 00000000..5288194b --- /dev/null +++ b/src/website_profiling/llm/agent.py @@ -0,0 +1,165 @@ +"""Agent loop for in-app chat and MCP — tool calling with streaming events.""" +from __future__ import annotations + +import json +from typing import Any, Callable + +from ..llm_config import llm_is_enabled, load_llm_config_from_db +from ..tools.audit_tools import AuditToolContext +from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, openai_tools_schema +from .base import ChatResult, ToolCall, get_llm_client + +MAX_TOOL_ROUNDS = 5 + +SYSTEM_PROMPT = """You are Site Audit AI, a technical SEO assistant for a self-hosted site audit platform. +You help users understand crawl results, audit issues, Lighthouse scores, keywords, and Search Console data. + +Rules: +- Use the provided tools to query real audit data. Do not invent URLs, scores, or metrics. +- When citing issues, include the URL when available. +- Summarize clearly for SEO practitioners. Prefer bullet lists for multiple items. +- You are read-only: you cannot run crawls or change settings. +- If data is missing, say what integration or crawl step is needed. +""" + +REACT_PROMPT_SUFFIX = """ +Respond with valid JSON only, one of: +{"action":"tool","name":"","args":{...}} +{"action":"answer","text":""} +""" + + +def _emit(on_event: Callable[[dict], None] | None, event: dict[str, Any]) -> None: + if on_event: + on_event(event) + + +def _supports_native_tools(client: Any) -> bool: + return callable(getattr(client, "chat_with_tools", None)) + + +def _react_step( + client: Any, + messages: list[dict[str, Any]], + tools_desc: str, + on_token: Callable[[str], None] | None, +) -> ChatResult: + """JSON ReAct fallback for providers without native tool calling.""" + convo = "\n".join( + f"{m.get('role')}: {m.get('content')}" + for m in messages + if m.get("role") in ("user", "assistant", "system") + ) + user = f"Available tools:\n{tools_desc}\n\nConversation:\n{convo}\n\nNext action JSON:" + data = client.complete_json(SYSTEM_PROMPT + REACT_PROMPT_SUFFIX, user) + action = str(data.get("action") or "").lower() + if action == "tool": + name = str(data.get("name") or "") + args = data.get("args") if isinstance(data.get("args"), dict) else {} + return ChatResult( + tool_calls=[ToolCall(id="react-0", name=name, arguments=args)], + ) + text = str(data.get("text") or data.get("answer") or data.get("content") or "") + if on_token and text: + on_token(text) + return ChatResult(content=text) + + +def _tools_description() -> str: + lines = [] + for t in TOOL_DEFINITIONS: + lines.append(f"- {t['name']}: {t.get('description', '')}") + return "\n".join(lines) + + +def _build_openai_messages(history: list[dict[str, str]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}] + for msg in history: + role = msg.get("role") + content = str(msg.get("content") or "") + if role in ("user", "assistant"): + out.append({"role": role, "content": content}) + return out + + +def run_agent_turn( + messages: list[dict[str, str]], + context: AuditToolContext, + *, + on_event: Callable[[dict], None] | None = None, +) -> dict[str, Any]: + """ + Run the agent loop. Emits NDJSON-style events via on_event. + Returns final result dict with ok, message, tool_events. + """ + cfg = load_llm_config_from_db() + if not llm_is_enabled(cfg): + err = "AI is disabled. Enable AI insights in the AI settings tab and configure a provider." + _emit(on_event, {"type": "error", "message": err}) + return {"ok": False, "error": err} + + try: + client = get_llm_client(cfg) + except ValueError as e: + msg = str(e) + _emit(on_event, {"type": "error", "message": msg}) + return {"ok": False, "error": msg} + + openai_messages = _build_openai_messages(messages) + tools = openai_tools_schema() + tool_events: list[dict[str, Any]] = [] + final_message = "" + + def on_token(text: str) -> None: + _emit(on_event, {"type": "token", "text": text}) + + for _round in range(MAX_TOOL_ROUNDS): + if _supports_native_tools(client): + result = client.chat_with_tools(openai_messages, tools, on_token=on_token) + else: + result = _react_step(client, openai_messages, _tools_description(), on_token) + + if result.tool_calls: + assistant_tool_calls = [] + for tc in result.tool_calls: + assistant_tool_calls.append({ + "id": tc.id, + "type": "function", + "function": {"name": tc.name, "arguments": json.dumps(tc.arguments)}, + }) + + if _supports_native_tools(client): + openai_messages.append({ + "role": "assistant", + "content": result.content or None, + "tool_calls": assistant_tool_calls, + }) + else: + openai_messages.append({ + "role": "assistant", + "content": f"Calling tool {result.tool_calls[0].name}", + }) + + for tc in result.tool_calls: + _emit(on_event, {"type": "tool_start", "name": tc.name, "args": tc.arguments}) + tool_result = dispatch_tool(tc.name, tc.arguments, context=context) + _emit(on_event, {"type": "tool_end", "name": tc.name, "result": tool_result}) + tool_events.append({"name": tc.name, "args": tc.arguments, "result": tool_result}) + + openai_messages.append({ + "role": "tool", + "tool_call_id": tc.id, + "content": json.dumps(tool_result, default=str), + }) + continue + + final_message = result.content.strip() + if final_message: + _emit(on_event, {"type": "done", "message": final_message}) + return {"ok": True, "message": final_message, "tool_events": tool_events} + + break + + err = "Agent stopped after maximum tool rounds without a final answer." + _emit(on_event, {"type": "error", "message": err}) + return {"ok": False, "error": err, "tool_events": tool_events} diff --git a/src/website_profiling/llm/base.py b/src/website_profiling/llm/base.py index b27dd5da..d200af46 100644 --- a/src/website_profiling/llm/base.py +++ b/src/website_profiling/llm/base.py @@ -3,12 +3,38 @@ import json import re -from typing import Any, Protocol +from dataclasses import dataclass, field +from typing import Any, Callable, Protocol + + +@dataclass +class ToolCall: + id: str + name: str + arguments: dict[str, Any] + + +@dataclass +class ChatResult: + content: str = "" + tool_calls: list[ToolCall] = field(default_factory=list) + finish_reason: str = "stop" + + +TokenCallback = Callable[[str], None] class LLMClient(Protocol): def complete_json(self, system: str, user: str) -> dict[str, Any]: ... + def chat_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + *, + on_token: TokenCallback | None = None, + ) -> ChatResult: ... + def parse_json_response(text: str) -> dict[str, Any]: text = (text or "").strip() diff --git a/src/website_profiling/llm/providers/anthropic.py b/src/website_profiling/llm/providers/anthropic.py index ce7e8467..e41b8b24 100644 --- a/src/website_profiling/llm/providers/anthropic.py +++ b/src/website_profiling/llm/providers/anthropic.py @@ -1,9 +1,10 @@ """Anthropic Messages API.""" from __future__ import annotations +import json from typing import Any -from ..base import parse_json_response +from ..base import ChatResult, TokenCallback, ToolCall, parse_json_response class AnthropicClient: @@ -33,3 +34,105 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]: if getattr(block, "type", None) == "text": parts.append(block.text) return parse_json_response("\n".join(parts)) + + def chat_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + *, + on_token: TokenCallback | None = None, + ) -> ChatResult: + if not self._api_key: + raise RuntimeError("Anthropic API key missing. Set it in the AI tab or ANTHROPIC_API_KEY.") + try: + import anthropic + except ImportError as e: + raise ImportError("pip install anthropic (or requirements-llm.txt)") from e + + system_parts: list[str] = [] + anthropic_messages: list[dict[str, Any]] = [] + for msg in messages: + role = msg.get("role") + if role == "system": + system_parts.append(str(msg.get("content") or "")) + elif role == "tool": + anthropic_messages.append({ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": str(msg.get("tool_call_id") or ""), + "content": str(msg.get("content") or ""), + }], + }) + else: + anthropic_messages.append({ + "role": role if role in ("user", "assistant") else "user", + "content": str(msg.get("content") or ""), + }) + + anthropic_tools = [] + for tool in tools: + fn = tool.get("function") or tool + anthropic_tools.append({ + "name": fn.get("name"), + "description": fn.get("description") or "", + "input_schema": fn.get("parameters") or {"type": "object", "properties": {}}, + }) + + client = anthropic.Anthropic(api_key=self._api_key, timeout=self._timeout) + kwargs: dict[str, Any] = { + "model": self._model, + "max_tokens": 4096, + "system": "\n".join(system_parts), + "messages": anthropic_messages, + "tools": anthropic_tools, + } + + if on_token: + content_parts: list[str] = [] + tool_calls: list[ToolCall] = [] + with client.messages.stream(**kwargs) as stream: + for event in stream: + if event.type == "content_block_delta" and hasattr(event.delta, "text"): + text = event.delta.text + content_parts.append(text) + on_token(text) + if event.type == "content_block_start" and getattr(event.content_block, "type", None) == "tool_use": + block = event.content_block + tool_calls.append( + ToolCall(id=block.id, name=block.name, arguments={}), + ) + if event.type == "content_block_delta" and getattr(event.delta, "type", None) == "input_json_delta": + if tool_calls: + partial = getattr(event.delta, "partial_json", "") or "" + prev = tool_calls[-1].arguments.get("_partial", "") + tool_calls[-1].arguments["_partial"] = prev + partial + final = stream.get_final_message() + for tc in tool_calls: + partial = tc.arguments.pop("_partial", "") + if partial: + try: + tc.arguments = json.loads(partial) + except json.JSONDecodeError: + tc.arguments = {} + text_parts = [] + for block in final.content: + if getattr(block, "type", None) == "text": + text_parts.append(block.text) + return ChatResult(content="".join(content_parts) or "".join(text_parts), tool_calls=tool_calls) + + msg = client.messages.create(**kwargs) + content_parts: list[str] = [] + tool_calls = [] + for block in msg.content: + if getattr(block, "type", None) == "text": + content_parts.append(block.text) + if getattr(block, "type", None) == "tool_use": + tool_calls.append( + ToolCall( + id=block.id, + name=block.name, + arguments=dict(block.input) if isinstance(block.input, dict) else {}, + ), + ) + return ChatResult(content="".join(content_parts), tool_calls=tool_calls) diff --git a/src/website_profiling/llm/providers/openai.py b/src/website_profiling/llm/providers/openai.py index 273b7031..6619de2b 100644 --- a/src/website_profiling/llm/providers/openai.py +++ b/src/website_profiling/llm/providers/openai.py @@ -4,7 +4,7 @@ import json from typing import Any -from ..base import parse_json_response +from ..base import ChatResult, TokenCallback, ToolCall, parse_json_response class OpenAIClient: @@ -40,3 +40,122 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]: data = r.json() content = data["choices"][0]["message"]["content"] return parse_json_response(content if isinstance(content, str) else json.dumps(content)) + + def chat_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + *, + on_token: TokenCallback | None = None, + ) -> ChatResult: + if not self._api_key: + raise RuntimeError("OpenAI API key missing. Set it in the AI tab or OPENAI_API_KEY.") + try: + import httpx + except ImportError as e: + raise ImportError("pip install httpx (or requirements-llm.txt)") from e + + payload: dict[str, Any] = { + "model": self._model, + "messages": messages, + "tools": tools, + "tool_choice": "auto", + "temperature": 0.2, + "stream": bool(on_token), + } + headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"} + url = f"{self._base}/chat/completions" + + if on_token: + return self._stream_chat(url, headers, payload, on_token) + + with httpx.Client(timeout=self._timeout) as client: + r = client.post(url, headers=headers, json={**payload, "stream": False}) + r.raise_for_status() + data = r.json() + return self._parse_chat_response(data) + + def _parse_chat_response(self, data: dict[str, Any]) -> ChatResult: + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + content = str(msg.get("content") or "") + tool_calls: list[ToolCall] = [] + for tc in msg.get("tool_calls") or []: + fn = tc.get("function") or {} + raw_args = fn.get("arguments") or "{}" + try: + args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args) + except json.JSONDecodeError: + args = {} + tool_calls.append( + ToolCall( + id=str(tc.get("id") or ""), + name=str(fn.get("name") or ""), + arguments=args if isinstance(args, dict) else {}, + ), + ) + return ChatResult( + content=content, + tool_calls=tool_calls, + finish_reason=str(choice.get("finish_reason") or "stop"), + ) + + def _stream_chat( + self, + url: str, + headers: dict[str, str], + payload: dict[str, Any], + on_token: TokenCallback, + ) -> ChatResult: + import httpx + + content_parts: list[str] = [] + tool_calls_acc: dict[int, dict[str, Any]] = {} + + with httpx.Client(timeout=self._timeout) as client: + with client.stream("POST", url, headers=headers, json=payload) as resp: + resp.raise_for_status() + for line in resp.iter_lines(): + if not line or not line.startswith("data: "): + continue + chunk_raw = line[6:].strip() + if chunk_raw == "[DONE]": + break + try: + chunk = json.loads(chunk_raw) + except json.JSONDecodeError: + continue + delta = ((chunk.get("choices") or [{}])[0]).get("delta") or {} + if delta.get("content"): + text = str(delta["content"]) + content_parts.append(text) + on_token(text) + for tc in delta.get("tool_calls") or []: + idx = int(tc.get("index") or 0) + acc = tool_calls_acc.setdefault( + idx, + {"id": "", "name": "", "arguments": ""}, + ) + if tc.get("id"): + acc["id"] = tc["id"] + fn = tc.get("function") or {} + if fn.get("name"): + acc["name"] = fn["name"] + if fn.get("arguments"): + acc["arguments"] += fn["arguments"] + + tool_calls: list[ToolCall] = [] + for acc in tool_calls_acc.values(): + raw_args = acc.get("arguments") or "{}" + try: + args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args) + except json.JSONDecodeError: + args = {} + tool_calls.append( + ToolCall( + id=str(acc.get("id") or ""), + name=str(acc.get("name") or ""), + arguments=args if isinstance(args, dict) else {}, + ), + ) + return ChatResult(content="".join(content_parts), tool_calls=tool_calls, finish_reason="stop") diff --git a/src/website_profiling/mcp/__init__.py b/src/website_profiling/mcp/__init__.py new file mode 100644 index 00000000..e1ce3837 --- /dev/null +++ b/src/website_profiling/mcp/__init__.py @@ -0,0 +1 @@ +"""Model Context Protocol server for Site Audit.""" diff --git a/src/website_profiling/mcp/__main__.py b/src/website_profiling/mcp/__main__.py new file mode 100644 index 00000000..f5f6e402 --- /dev/null +++ b/src/website_profiling/mcp/__main__.py @@ -0,0 +1,4 @@ +from .server import main + +if __name__ == "__main__": + main() diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py new file mode 100644 index 00000000..0aee9bf5 --- /dev/null +++ b/src/website_profiling/mcp/server.py @@ -0,0 +1,81 @@ +"""stdio MCP server exposing read-only Site Audit tools.""" +from __future__ import annotations + +import json +import os +from typing import Any + +from ..tools.audit_tools import AuditToolContext +from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool + + +def _default_property_id() -> int | None: + raw = os.environ.get("WP_PROPERTY_ID", "").strip() + if not raw: + return None + try: + pid = int(raw) + return pid if pid > 0 else None + except ValueError: + return None + + +def _merge_context(args: dict[str, Any]) -> AuditToolContext: + pid = args.get("property_id") + rid = args.get("report_id") + default_pid = _default_property_id() + try: + property_id = int(pid) if pid is not None else default_pid + except (TypeError, ValueError): + property_id = default_pid + try: + report_id = int(rid) if rid is not None else None + except (TypeError, ValueError): + report_id = None + return AuditToolContext(property_id=property_id, report_id=report_id) + + +def main() -> None: + try: + from mcp.server import Server + from mcp.server.stdio import stdio_server + from mcp.types import TextContent, Tool + except ImportError as e: + raise SystemExit( + "MCP SDK not installed. Run: pip install -r requirements-mcp.txt", + ) from e + + server = Server("site-audit") + + @server.list_tools() + async def list_tools() -> list[Tool]: + out: list[Tool] = [] + for spec in TOOL_DEFINITIONS: + out.append( + Tool( + name=spec["name"], + description=spec.get("description", ""), + inputSchema=spec.get("inputSchema", {"type": "object", "properties": {}}), + ), + ) + return out + + @server.call_tool() + async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]: + args = dict(arguments or {}) + ctx = _merge_context(args) + result = dispatch_tool(name, args, context=ctx) + text = json.dumps(result, indent=2, default=str) + return [TextContent(type="text", text=text)] + + async def run() -> None: + async with stdio_server() as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) + + import asyncio + + asyncio.run(run()) + + +if __name__ == "__main__": + main() diff --git a/src/website_profiling/tools/audit_tools/__init__.py b/src/website_profiling/tools/audit_tools/__init__.py new file mode 100644 index 00000000..47d05972 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/__init__.py @@ -0,0 +1,5 @@ +"""Read-only audit query tools for MCP and in-app chat.""" +from .context import AuditToolContext +from .registry import TOOL_DEFINITIONS, dispatch_tool + +__all__ = ["AuditToolContext", "TOOL_DEFINITIONS", "dispatch_tool"] diff --git a/src/website_profiling/tools/audit_tools/context.py b/src/website_profiling/tools/audit_tools/context.py new file mode 100644 index 00000000..47d23fde --- /dev/null +++ b/src/website_profiling/tools/audit_tools/context.py @@ -0,0 +1,67 @@ +"""Execution context for audit tools (property + report scope).""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +from psycopg import Connection + +from ...db.crawl_store import get_latest_crawl_run_id, read_crawl +from ...db.report_store import read_report_payload +from ...integrations.google.keyword_store import read_latest_keyword_data +from ...integrations.google.store import read_latest_google_data + + +@dataclass +class AuditToolContext: + property_id: Optional[int] = None + report_id: Optional[int] = None + + def load_payload(self, conn: Connection) -> dict[str, Any]: + payload = read_report_payload(conn, self.report_id) + return payload if isinstance(payload, dict) else {} + + def load_crawl_df(self, conn: Connection): + payload = self.load_payload(conn) + run_id = payload.get("crawl_run_id") + try: + rid = int(run_id) if run_id is not None else None + except (TypeError, ValueError): + rid = None + if rid is None: + rid = get_latest_crawl_run_id(conn) + return read_crawl(conn, rid) + + def load_google(self, conn: Connection) -> Optional[dict[str, Any]]: + google = read_latest_google_data(conn, self.property_id) + if google: + return google + payload = self.load_payload(conn) + embedded = payload.get("google") + return embedded if isinstance(embedded, dict) else None + + def load_keywords(self, conn: Connection) -> Optional[dict[str, Any]]: + kw = read_latest_keyword_data(conn, self.property_id) + if kw: + return kw + payload = self.load_payload(conn) + embedded = payload.get("keywords") + return embedded if isinstance(embedded, dict) else None + + def with_args(self, args: dict[str, Any]) -> AuditToolContext: + """Merge tool args property_id/report_id when provided.""" + pid = args.get("property_id") + rid = args.get("report_id") + new_pid = self.property_id + new_rid = self.report_id + if pid is not None: + try: + new_pid = int(pid) + except (TypeError, ValueError): + pass + if rid is not None: + try: + new_rid = int(rid) + except (TypeError, ValueError): + pass + return AuditToolContext(property_id=new_pid, report_id=new_rid) diff --git a/src/website_profiling/tools/audit_tools/crawl.py b/src/website_profiling/tools/audit_tools/crawl.py new file mode 100644 index 00000000..a7da9cba --- /dev/null +++ b/src/website_profiling/tools/audit_tools/crawl.py @@ -0,0 +1,122 @@ +"""Crawl page query tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...integrations.google.page_lookup import slice_from_google_row +from .context import AuditToolContext + +_PAGE_LIMIT_MAX = 30 + + +def _page_row(rec: dict[str, Any]) -> dict[str, Any]: + return { + "url": str(rec.get("url") or ""), + "status": str(rec.get("status") or ""), + "title": str(rec.get("title") or ""), + "inlinks": rec.get("inlinks"), + } + + +def search_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False} + + status_filter = str(args.get("status") or "").strip() + url_contains = str(args.get("url_contains") or "").strip().lower() + limit = args.get("limit", _PAGE_LIMIT_MAX) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = _PAGE_LIMIT_MAX + limit = max(1, min(limit, _PAGE_LIMIT_MAX)) + + records = df.to_dict(orient="records") + if status_filter: + records = [r for r in records if str(r.get("status") or "") == status_filter] + if url_contains: + records = [r for r in records if url_contains in str(r.get("url") or "").lower()] + + total = len(records) + truncated = total > limit + pages = [_page_row(r) for r in records[:limit]] + return {"pages": pages, "total": total, "truncated": truncated} + + +def get_page_details(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip().rstrip("/") + if not url: + return {"error": "url is required"} + + df = scoped.load_crawl_df(conn) + crawl_row: dict[str, Any] | None = None + if df is not None and not df.empty and "url" in df.columns: + norm = url.rstrip("/") + for _, row in df.iterrows(): + row_url = str(row.get("url") or "").rstrip("/") + if row_url == norm or row_url == url: + crawl_row = { + "url": row_url, + "status": str(row.get("status") or ""), + "title": str(row.get("title") or ""), + "meta_description": str(row.get("meta_description") or ""), + "h1": row.get("h1"), + "word_count": row.get("word_count"), + "inlinks": row.get("inlinks"), + "outlinks": row.get("outlinks"), + "content_type": str(row.get("content_type") or ""), + } + break + + payload = scoped.load_payload(conn) + lighthouse_by_url = payload.get("lighthouse_by_url") or {} + lh = lighthouse_by_url.get(url) or lighthouse_by_url.get(url + "/") + if not lh and crawl_row: + lh = lighthouse_by_url.get(crawl_row.get("url", "")) + + google_raw = scoped.load_google(conn) + gsc_ga4 = None + if google_raw: + gsc_ga4 = slice_from_google_row(google_raw, url) + + return { + "url": url, + "crawl": crawl_row, + "lighthouse": lh if isinstance(lh, dict) else None, + "gsc_ga4": gsc_ga4, + "found_in_crawl": crawl_row is not None, + } + + +def get_internal_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip().rstrip("/") + if not url: + return {"error": "url is required"} + + from ...db.crawl_store import read_edges + + payload = scoped.load_payload(conn) + run_id = payload.get("crawl_run_id") + try: + rid = int(run_id) if run_id is not None else None + except (TypeError, ValueError): + rid = None + + edges = read_edges(conn, rid) + outlinks = [b for a, b in edges if a.rstrip("/") == url] + inlinks = [a for a, b in edges if b.rstrip("/") == url] + limit = 50 + return { + "url": url, + "outlinks": outlinks[:limit], + "inlinks": inlinks[:limit], + "outlink_count": len(outlinks), + "inlink_count": len(inlinks), + "truncated": len(outlinks) > limit or len(inlinks) > limit, + } diff --git a/src/website_profiling/tools/audit_tools/google.py b/src/website_profiling/tools/audit_tools/google.py new file mode 100644 index 00000000..55d3e513 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/google.py @@ -0,0 +1,45 @@ +"""Google Search Console / GA4 summary tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from .context import AuditToolContext + + +def get_google_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google(conn) + if not data: + return {"error": "no google data found", "property_id": scoped.property_id} + + gsc = data.get("gsc") if isinstance(data.get("gsc"), dict) else {} + ga4 = data.get("ga4") if isinstance(data.get("ga4"), dict) else {} + gsc_summary = gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {} + ga4_summary = ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {} + + top_queries = gsc.get("top_queries") or [] + top_pages = gsc.get("top_pages") or [] + if isinstance(top_queries, list): + top_queries = top_queries[:10] + if isinstance(top_pages, list): + top_pages = top_pages[:10] + + return { + "fetched_at": data.get("fetched_at"), + "date_range": data.get("date_range"), + "gsc": { + "site_url": gsc.get("site_url"), + "summary": gsc_summary, + "top_queries": top_queries, + "top_pages": top_pages, + }, + "ga4": { + "property_id": ga4.get("property_id"), + "summary": ga4_summary, + "top_pages": (ga4.get("top_pages") or [])[:10] if isinstance(ga4.get("top_pages"), list) else [], + }, + "errors": data.get("errors") or [], + "property_id": scoped.property_id, + } diff --git a/src/website_profiling/tools/audit_tools/health.py b/src/website_profiling/tools/audit_tools/health.py new file mode 100644 index 00000000..0f0efeb6 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/health.py @@ -0,0 +1,61 @@ +"""Health history query tools.""" +from __future__ import annotations + +import json +from typing import Any + +from psycopg import Connection + +from ...db._common import _row_field +from .context import AuditToolContext + + +def get_health_history(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + property_id = scoped.property_id + if property_id is None: + return {"error": "property_id is required"} + + limit = args.get("limit", 10) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 10 + limit = max(1, min(limit, 30)) + + cur = conn.execute( + """SELECT health_score, category_scores, issue_counts, generated_at, report_id + FROM audit_health_snapshots + WHERE property_id = %s + ORDER BY generated_at DESC, id DESC + LIMIT %s""", + (property_id, limit), + ) + snapshots = [] + for row in cur.fetchall() or []: + cat_scores = _row_field(row, "category_scores", index=1) + issue_counts = _row_field(row, "issue_counts", index=2) + if isinstance(cat_scores, str): + try: + cat_scores = json.loads(cat_scores) + except json.JSONDecodeError: + cat_scores = {} + if isinstance(issue_counts, str): + try: + issue_counts = json.loads(issue_counts) + except json.JSONDecodeError: + issue_counts = {} + gen = _row_field(row, "generated_at", index=3) + snapshots.append({ + "health_score": _row_field(row, "health_score", index=0), + "category_scores": cat_scores, + "issue_counts": issue_counts, + "generated_at": gen.isoformat() if hasattr(gen, "isoformat") else str(gen or ""), + "report_id": _row_field(row, "report_id", index=4), + }) + + return { + "property_id": property_id, + "snapshots": snapshots, + "count": len(snapshots), + } diff --git a/src/website_profiling/tools/audit_tools/keywords.py b/src/website_profiling/tools/audit_tools/keywords.py new file mode 100644 index 00000000..4dcd03d2 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/keywords.py @@ -0,0 +1,93 @@ +"""Keyword query tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from .context import AuditToolContext + +_KEYWORD_LIMIT_DEFAULT = 20 +_KEYWORD_LIMIT_MAX = 50 + + +def get_keyword_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required for keyword data"} + + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "property_id": scoped.property_id} + + rows = data.get("rows") or [] + if not isinstance(rows, list): + rows = [] + + striking = data.get("striking_distance") or [] + striking_count = len(striking) if isinstance(striking, list) else 0 + + top_n = args.get("limit", _KEYWORD_LIMIT_DEFAULT) + try: + top_n = int(top_n) + except (TypeError, ValueError): + top_n = _KEYWORD_LIMIT_DEFAULT + top_n = max(1, min(top_n, _KEYWORD_LIMIT_MAX)) + + top_rows = [] + for row in rows[:top_n]: + if not isinstance(row, dict): + continue + top_rows.append({ + "keyword": row.get("keyword"), + "score": row.get("score"), + "gsc_position": row.get("gsc_position"), + "gsc_clicks": row.get("gsc_clicks"), + "gsc_impressions": row.get("gsc_impressions"), + "recommended_action": row.get("recommended_action"), + }) + + return { + "fetched_at": data.get("fetched_at"), + "total_keywords": data.get("total_keywords") or len(rows), + "striking_distance_count": striking_count, + "top_keywords": top_rows, + "property_id": scoped.property_id, + } + + +def search_keywords(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required for keyword data"} + + query = str(args.get("query") or "").strip().lower() + if not query: + return {"error": "query is required"} + + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "keywords": [], "total": 0} + + rows = data.get("rows") or [] + matches = [] + for row in rows: + if not isinstance(row, dict): + continue + kw = str(row.get("keyword") or "").lower() + if query in kw: + matches.append({ + "keyword": row.get("keyword"), + "gsc_position": row.get("gsc_position"), + "gsc_clicks": row.get("gsc_clicks"), + "gsc_impressions": row.get("gsc_impressions"), + "recommended_action": row.get("recommended_action"), + }) + + limit = 30 + total = len(matches) + return { + "keywords": matches[:limit], + "total": total, + "truncated": total > limit, + } diff --git a/src/website_profiling/tools/audit_tools/lighthouse.py b/src/website_profiling/tools/audit_tools/lighthouse.py new file mode 100644 index 00000000..5f4c4037 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/lighthouse.py @@ -0,0 +1,58 @@ +"""Lighthouse query tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...db.lighthouse_store import read_lighthouse_page_summaries, read_lighthouse_summary +from .context import AuditToolContext + + +def get_lighthouse_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + + summary = payload.get("lighthouse_summary") + if not isinstance(summary, dict): + db_summary = read_lighthouse_summary(conn) + summary = db_summary if isinstance(db_summary, dict) else {} + + human = payload.get("lighthouse_human_summary") + diagnostics = payload.get("lighthouse_diagnostics") + page_summaries = payload.get("lighthouse_by_url") + if not isinstance(page_summaries, dict): + page_summaries = read_lighthouse_page_summaries(conn) or {} + + poor_pages = [] + for url, data in list(page_summaries.items())[:20]: + if not isinstance(data, dict): + continue + perf = data.get("performance") or data.get("scores", {}).get("performance") + if perf is not None and float(perf) < 50: + poor_pages.append({"url": url, "performance": perf}) + + return { + "summary": summary, + "human_summary": human if isinstance(human, str) else None, + "diagnostics_count": len(diagnostics) if isinstance(diagnostics, list) else 0, + "pages_audited": len(page_summaries) if isinstance(page_summaries, dict) else 0, + "poor_performance_pages": poor_pages[:10], + } + + +def get_lighthouse_for_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip().rstrip("/") + if not url: + return {"error": "url is required"} + + payload = scoped.load_payload(conn) + by_url = payload.get("lighthouse_by_url") or {} + if not isinstance(by_url, dict): + by_url = read_lighthouse_page_summaries(conn) or {} + + data = by_url.get(url) or by_url.get(url + "/") + if not data: + return {"error": "no lighthouse data for url", "url": url} + return {"url": url, "lighthouse": data} diff --git a/src/website_profiling/tools/audit_tools/properties.py b/src/website_profiling/tools/audit_tools/properties.py new file mode 100644 index 00000000..9a3a126e --- /dev/null +++ b/src/website_profiling/tools/audit_tools/properties.py @@ -0,0 +1,45 @@ +"""Property query tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...db.property_store import get_property_by_id, list_properties_public +from .context import AuditToolContext + + +def _public_property_row(prop: dict[str, Any]) -> dict[str, Any]: + return { + "id": prop.get("id"), + "name": prop.get("name"), + "canonical_domain": prop.get("canonical_domain"), + "site_url": prop.get("site_url"), + "gsc_site_url": prop.get("gsc_site_url"), + "ga4_property_id": prop.get("ga4_property_id"), + "google_auth_mode": prop.get("google_auth_mode"), + "google_connected": prop.get("google_connected") or bool(prop.get("google_connected_at")), + "google_connected_at": prop.get("google_connected_at"), + "google_connected_email": prop.get("google_connected_email"), + "google_date_range_days": prop.get("google_date_range_days"), + "crawl_authorized_at": prop.get("crawl_authorized_at"), + } + + +def list_properties(conn: Connection, _ctx: AuditToolContext, _args: dict[str, Any]) -> dict[str, Any]: + rows = list_properties_public(conn) + return {"properties": rows, "count": len(rows)} + + +def get_property(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + property_id = args.get("property_id") or ctx.property_id + if property_id is None: + return {"error": "property_id is required"} + try: + pid = int(property_id) + except (TypeError, ValueError): + return {"error": "invalid property_id"} + prop = get_property_by_id(conn, pid) + if not prop: + return {"error": f"property {pid} not found"} + return {"property": _public_property_row(prop)} diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py new file mode 100644 index 00000000..a2067933 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/registry.py @@ -0,0 +1,192 @@ +"""Tool registry and dispatch for MCP and chat agent.""" +from __future__ import annotations + +from typing import Any, Callable + +from psycopg import Connection + +from ...db.storage import db_session +from .context import AuditToolContext +from .crawl import get_internal_links, get_page_details, search_pages +from .google import get_google_summary +from .health import get_health_history +from .keywords import get_keyword_summary, search_keywords +from .lighthouse import get_lighthouse_for_url, get_lighthouse_summary +from .properties import get_property, list_properties +from .report import get_category_scores, get_report_summary, list_issues + +ToolHandler = Callable[[Connection, AuditToolContext, dict[str, Any]], dict[str, Any]] + +_TOOL_HANDLERS: dict[str, ToolHandler] = { + "list_properties": list_properties, + "get_property": get_property, + "get_report_summary": get_report_summary, + "list_issues": list_issues, + "get_category_scores": get_category_scores, + "search_pages": search_pages, + "get_page_details": get_page_details, + "get_internal_links": get_internal_links, + "get_lighthouse_summary": get_lighthouse_summary, + "get_lighthouse_for_url": get_lighthouse_for_url, + "get_keyword_summary": get_keyword_summary, + "search_keywords": search_keywords, + "get_google_summary": get_google_summary, + "get_health_history": get_health_history, +} + +# v1 public tool list (10 primary + helpers used by agent) +TOOL_DEFINITIONS: list[dict[str, Any]] = [ + { + "name": "list_properties", + "description": "List all configured site properties (domains) in Site Audit.", + "inputSchema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "get_property", + "description": "Get details for one property by property_id.", + "inputSchema": { + "type": "object", + "properties": {"property_id": {"type": "integer", "description": "Property ID"}}, + "required": ["property_id"], + }, + }, + { + "name": "get_report_summary", + "description": "Health score, issue counts by priority, crawl stats, and category scores for the latest or specified report.", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "report_id": {"type": "integer"}, + }, + "required": [], + }, + }, + { + "name": "list_issues", + "description": "List audit issues with optional filters. Returns paginated results (max 50).", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "report_id": {"type": "integer"}, + "priority": {"type": "string", "enum": ["Critical", "High", "Medium", "Low"]}, + "category_id": {"type": "string", "description": "e.g. technical_seo, link_health"}, + "url_contains": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 50}, + }, + "required": [], + }, + }, + { + "name": "search_pages", + "description": "Search crawled pages by status code or URL substring. Max 30 results.", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "report_id": {"type": "integer"}, + "status": {"type": "string", "description": "HTTP status e.g. 404"}, + "url_contains": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 30}, + }, + "required": [], + }, + }, + { + "name": "get_page_details", + "description": "Crawl row, Lighthouse snippet, and GSC/GA4 slice for one URL.", + "inputSchema": { + "type": "object", + "properties": { + "url": {"type": "string"}, + "property_id": {"type": "integer"}, + "report_id": {"type": "integer"}, + }, + "required": ["url"], + }, + }, + { + "name": "get_lighthouse_summary", + "description": "Site-wide Lighthouse summary and pages with poor performance scores.", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "report_id": {"type": "integer"}, + }, + "required": [], + }, + }, + { + "name": "get_keyword_summary", + "description": "Top keywords, striking-distance count, and GSC metrics for a property.", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 50}, + }, + "required": ["property_id"], + }, + }, + { + "name": "get_google_summary", + "description": "GSC and GA4 headline metrics, top queries and pages for a property.", + "inputSchema": { + "type": "object", + "properties": {"property_id": {"type": "integer"}}, + "required": [], + }, + }, + { + "name": "get_health_history", + "description": "Historical health score snapshots for a property (for trend analysis).", + "inputSchema": { + "type": "object", + "properties": { + "property_id": {"type": "integer"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 30}, + }, + "required": ["property_id"], + }, + }, +] + + +def dispatch_tool( + name: str, + args: dict[str, Any] | None, + *, + context: AuditToolContext | None = None, + conn: Connection | None = None, +) -> dict[str, Any]: + """Run a tool by name. Uses db_session when conn is not provided.""" + handler = _TOOL_HANDLERS.get(name) + if handler is None: + return {"error": f"unknown tool: {name}"} + + ctx = context or AuditToolContext() + payload_args = dict(args or {}) + merged_ctx = ctx.with_args(payload_args) + + if conn is not None: + return handler(conn, merged_ctx, payload_args) + + with db_session() as session: + return handler(session, merged_ctx, payload_args) + + +def openai_tools_schema() -> list[dict[str, Any]]: + """Convert TOOL_DEFINITIONS to OpenAI function-calling format.""" + out: list[dict[str, Any]] = [] + for tool in TOOL_DEFINITIONS: + out.append({ + "type": "function", + "function": { + "name": tool["name"], + "description": tool.get("description", ""), + "parameters": tool.get("inputSchema", {"type": "object", "properties": {}}), + }, + }) + return out diff --git a/src/website_profiling/tools/audit_tools/report.py b/src/website_profiling/tools/audit_tools/report.py new file mode 100644 index 00000000..f49da130 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/report.py @@ -0,0 +1,152 @@ +"""Report summary and issue query tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...reporting.terminology import category_display_name +from .context import AuditToolContext + +_PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3} +_ISSUE_LIMIT_DEFAULT = 20 +_ISSUE_LIMIT_MAX = 50 + + +def _normalize_priority(p: str) -> str: + raw = (p or "").strip() + if not raw: + return "" + cap = raw[0].upper() + raw[1:].lower() + if cap in _PRIORITY_ORDER: + return cap + return raw + + +def _iter_category_issues(payload: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + cat_id = str(cat.get("id") or "") + cat_name = category_display_name(str(cat.get("name") or cat_id)) + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + rec = str(issue.get("llm_recommendation") or issue.get("recommendation") or "") + rows.append({ + "category_id": cat_id, + "category": cat_name, + "priority": str(issue.get("priority") or "Medium"), + "message": str(issue.get("message") or ""), + "url": str(issue.get("url") or ""), + "recommendation": rec, + }) + rows.sort(key=lambda x: _PRIORITY_ORDER.get(x.get("priority", "Low"), 99)) + return rows + + +def _health_score(payload: dict[str, Any]) -> int | None: + scores = [ + float(c.get("score")) + for c in (payload.get("categories") or []) + if isinstance(c, dict) and isinstance(c.get("score"), (int, float)) + ] + return round(sum(scores) / len(scores)) if scores else None + + +def _issue_counts(issues: list[dict[str, Any]]) -> dict[str, int]: + counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0} + for issue in issues: + p = str(issue.get("priority") or "Medium") + counts[p] = counts.get(p, 0) + 1 + return counts + + +def get_report_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + all_issues = _iter_category_issues(payload) + summary = payload.get("summary") or {} + categories = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + categories.append({ + "id": cat.get("id"), + "name": category_display_name(str(cat.get("name") or "")), + "score": cat.get("score"), + "issue_count": len(cat.get("issues") or []), + }) + return { + "site_name": payload.get("site_name"), + "report_generated_at": payload.get("report_generated_at"), + "health_score": _health_score(payload), + "issue_counts": _issue_counts(all_issues), + "total_issues": len(all_issues), + "crawl_summary": { + "total_urls": summary.get("total_urls"), + "count_2xx": summary.get("count_2xx"), + "count_3xx": summary.get("count_3xx"), + "count_4xx": summary.get("count_4xx"), + "count_5xx": summary.get("count_5xx"), + "success_rate": summary.get("success_rate"), + }, + "categories": categories, + "property_id": scoped.property_id, + "report_id": scoped.report_id, + } + + +def list_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "issues": [], "total": 0, "truncated": False} + + limit = args.get("limit", _ISSUE_LIMIT_DEFAULT) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = _ISSUE_LIMIT_DEFAULT + limit = max(1, min(limit, _ISSUE_LIMIT_MAX)) + + priority_filter = _normalize_priority(str(args.get("priority") or "")) + category_id = str(args.get("category_id") or "").strip() + url_contains = str(args.get("url_contains") or "").strip().lower() + + issues = _iter_category_issues(payload) + if priority_filter: + issues = [i for i in issues if i.get("priority") == priority_filter] + if category_id: + issues = [i for i in issues if i.get("category_id") == category_id] + if url_contains: + issues = [i for i in issues if url_contains in str(i.get("url") or "").lower()] + + total = len(issues) + truncated = total > limit + return { + "issues": issues[:limit], + "total": total, + "truncated": truncated, + } + + +def get_category_scores(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "categories": []} + categories = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + categories.append({ + "id": cat.get("id"), + "name": category_display_name(str(cat.get("name") or "")), + "score": cat.get("score"), + "issue_count": len(cat.get("issues") or []), + }) + return {"categories": categories, "health_score": _health_score(payload)} diff --git a/tests/test_audit_tools.py b/tests/test_audit_tools.py new file mode 100644 index 00000000..b203e677 --- /dev/null +++ b/tests/test_audit_tools.py @@ -0,0 +1,356 @@ +"""Tests for audit_tools registry and issue/page queries.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.tools.audit_tools import AuditToolContext, dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, openai_tools_schema +from website_profiling.tools.audit_tools.report import ( + get_category_scores, + get_report_summary, + list_issues, +) + + +def _sample_payload() -> dict: + return { + "site_name": "Example", + "report_generated_at": "2026-06-07T12:00:00Z", + "crawl_run_id": 9, + "summary": {"total_urls": 10, "count_2xx": 8, "count_4xx": 2}, + "lighthouse_summary": {"performance": 72}, + "lighthouse_human_summary": "OK", + "lighthouse_diagnostics": [{"id": "x"}], + "lighthouse_by_url": { + "https://ex.com/slow": {"performance": 40}, + "https://ex.com/ok": {"performance": 90}, + }, + "google": {"fetched_at": "2026-06-07", "gsc": {"summary": {"clicks": 1}}}, + "keywords": { + "fetched_at": "2026-06-07", + "total_keywords": 2, + "rows": [{"keyword": "widgets", "score": 0.5, "gsc_clicks": 3}], + "striking_distance": [{"keyword": "repair"}], + }, + "categories": [ + { + "id": "technical_seo", + "name": "Technical SEO", + "score": 80, + "issues": [ + {"priority": "Critical", "message": "Missing title", "url": "https://ex.com/a", "recommendation": "Add title"}, + {"priority": "High", "message": "Slow page", "url": "https://ex.com/blog/slow", "recommendation": "Optimize"}, + {"priority": "Low", "message": "Minor", "url": "https://ex.com/z", "recommendation": "Fix"}, + ], + }, + { + "id": "link_health", + "name": "Links", + "score": 70, + "issues": [ + {"priority": "High", "message": "Broken link", "url": "https://ex.com/404", "recommendation": "Fix link"}, + ], + }, + "not-a-dict", + ], + } + + +def test_tool_definitions_have_required_fields() -> None: + for tool in TOOL_DEFINITIONS: + assert tool.get("name") + assert tool.get("description") + assert "inputSchema" in tool + + +def test_openai_tools_schema() -> None: + schema = openai_tools_schema() + assert len(schema) == len(TOOL_DEFINITIONS) + assert schema[0]["type"] == "function" + + +def test_dispatch_unknown_tool() -> None: + conn = MagicMock() + result = dispatch_tool("nonexistent", {}, conn=conn) + assert result["error"] == "unknown tool: nonexistent" + + +def test_dispatch_via_db_session() -> None: + conn = MagicMock() + with patch("website_profiling.tools.audit_tools.registry.db_session") as mock_sess: + mock_sess.return_value.__enter__.return_value = conn + with patch( + "website_profiling.tools.audit_tools.properties.list_properties_public", + return_value=[], + ): + result = dispatch_tool("list_properties", {}) + assert result["count"] == 0 + + +def test_context_with_args_and_loaders() -> None: + conn = MagicMock() + ctx = Ctx(property_id=1, report_id=2) + merged = ctx.with_args({"property_id": 3, "report_id": 4}) + assert merged.property_id == 3 + assert merged.report_id == 4 + + with patch( + "website_profiling.tools.audit_tools.context.read_report_payload", + return_value=_sample_payload(), + ): + payload = ctx.load_payload(conn) + assert payload["site_name"] == "Example" + + df = pd.DataFrame([{"url": "https://ex.com", "status": "200"}]) + with patch( + "website_profiling.tools.audit_tools.context.read_report_payload", + return_value=_sample_payload(), + ), patch( + "website_profiling.tools.audit_tools.context.read_crawl", + return_value=df, + ): + assert not ctx.load_crawl_df(conn).empty + + with patch( + "website_profiling.tools.audit_tools.context.read_latest_google_data", + return_value={"gsc": {}}, + ): + assert ctx.load_google(conn) is not None + + with patch( + "website_profiling.tools.audit_tools.context.read_latest_google_data", + return_value=None, + ), patch( + "website_profiling.tools.audit_tools.context.read_report_payload", + return_value=_sample_payload(), + ): + assert ctx.load_google(conn)["gsc"]["summary"]["clicks"] == 1 + + with patch( + "website_profiling.tools.audit_tools.context.read_latest_keyword_data", + return_value={"rows": []}, + ): + assert ctx.load_keywords(conn) is not None + + +def test_list_issues_filtering_and_limit() -> None: + conn = MagicMock() + ctx = AuditToolContext() + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + all_issues = list_issues(conn, ctx, {}) + assert all_issues["total"] == 4 + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + critical = list_issues(conn, ctx, {"priority": "Critical"}) + assert critical["total"] == 1 + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + blog = list_issues(conn, ctx, {"url_contains": "/blog"}) + assert blog["total"] == 1 + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + capped = list_issues(conn, ctx, {"limit": 2}) + assert capped["truncated"] is True + + +def test_list_issues_empty_report() -> None: + conn = MagicMock() + with patch.object(Ctx, "load_payload", return_value={}): + result = list_issues(conn, AuditToolContext(), {}) + assert result["error"] == "no report found" + + +def test_get_report_summary_and_categories() -> None: + conn = MagicMock() + ctx = AuditToolContext() + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + summary = get_report_summary(conn, ctx, {}) + assert summary["health_score"] == 75 + assert summary["total_issues"] == 4 + + with patch.object(Ctx, "load_payload", return_value={}): + empty = get_category_scores(conn, ctx, {}) + assert empty["error"] == "no report found" + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + cats = get_category_scores(conn, ctx, {}) + assert len(cats["categories"]) == 2 + + +def test_properties_tools() -> None: + conn = MagicMock() + with patch( + "website_profiling.tools.audit_tools.properties.list_properties_public", + return_value=[{"id": 1}], + ): + assert dispatch_tool("list_properties", {}, conn=conn)["count"] == 1 + + with patch( + "website_profiling.tools.audit_tools.properties.get_property_by_id", + return_value=None, + ): + missing = dispatch_tool("get_property", {"property_id": 9}, conn=conn) + assert "not found" in missing["error"] + + with patch( + "website_profiling.tools.audit_tools.properties.get_property_by_id", + return_value={"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}, + ): + ok = dispatch_tool("get_property", {"property_id": 1}, conn=conn) + assert ok["property"]["id"] == 1 + + assert dispatch_tool("get_property", {}, context=AuditToolContext(), conn=conn)["error"] + + +def test_crawl_tools() -> None: + conn = MagicMock() + df = pd.DataFrame([ + {"url": "https://ex.com/ok", "status": "200", "title": "OK", "inlinks": 3}, + {"url": "https://ex.com/missing", "status": "404", "title": "", "inlinks": 0}, + ]) + ctx = AuditToolContext(property_id=1) + with patch.object(Ctx, "load_crawl_df", return_value=df): + pages = dispatch_tool("search_pages", {"status": "404"}, context=ctx, conn=conn) + assert pages["total"] == 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()), patch.object( + Ctx, "load_payload", return_value=_sample_payload(), + ), patch( + "website_profiling.tools.audit_tools.crawl.slice_from_google_row", + return_value={"gsc": {"clicks": 1}}, + ): + detail = dispatch_tool( + "get_page_details", + {"url": "https://ex.com/ok"}, + context=ctx, + conn=conn, + ) + assert detail["found_in_crawl"] is False + + with patch( + "website_profiling.db.crawl_store.read_edges", + return_value=[("https://ex.com", "https://ex.com/child")], + ), patch.object(Ctx, "load_payload", return_value=_sample_payload()): + links = dispatch_tool( + "get_internal_links", + {"url": "https://ex.com"}, + context=ctx, + conn=conn, + ) + assert links["outlink_count"] == 1 + + assert dispatch_tool("get_page_details", {}, context=ctx, conn=conn)["error"] + + +def test_lighthouse_keywords_google_health() -> None: + conn = MagicMock() + ctx = AuditToolContext(property_id=1, report_id=1) + df = pd.DataFrame([ + {"url": "https://ex.com/ok", "status": "200", "title": "OK", "inlinks": 3}, + {"url": "https://ex.com/missing", "status": "404", "title": "", "inlinks": 0}, + ]) + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + lh = dispatch_tool("get_lighthouse_summary", {}, context=ctx, conn=conn) + assert lh["pages_audited"] == 2 + assert lh["poor_performance_pages"] + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + url_lh = dispatch_tool( + "get_lighthouse_for_url", + {"url": "https://ex.com/slow"}, + context=ctx, + conn=conn, + ) + assert url_lh["lighthouse"]["performance"] == 40 + + with patch.object(Ctx, "load_keywords", return_value=_sample_payload()["keywords"]): + kw = dispatch_tool("get_keyword_summary", {"property_id": 1}, context=ctx, conn=conn) + assert kw["total_keywords"] == 2 + + with patch.object(Ctx, "load_keywords", return_value=_sample_payload()["keywords"]): + search = dispatch_tool( + "search_keywords", + {"property_id": 1, "query": "widget"}, + context=ctx, + conn=conn, + ) + assert search["total"] == 1 + + with patch.object(Ctx, "load_google", return_value={ + "fetched_at": "x", + "gsc": {"summary": {"clicks": 5}, "top_queries": [], "top_pages": []}, + "ga4": {"summary": {"sessions": 10}, "top_pages": []}, + }): + g = dispatch_tool("get_google_summary", {}, context=ctx, conn=conn) + assert g["gsc"]["summary"]["clicks"] == 5 + + from datetime import datetime, timezone + + now = datetime.now(timezone.utc) + conn.set_next_cursor = lambda cur: None # type: ignore + from tests.db_test_fakes import FakeConn, FakeCursor + + fake = FakeConn() + fake.set_next_cursor( + FakeCursor( + fetchall_value=[{ + "health_score": 80, + "category_scores": "{}", + "issue_counts": "{}", + "generated_at": now, + "report_id": 1, + }], + ), + ) + hist = dispatch_tool("get_health_history", {"property_id": 1}, conn=fake) + assert hist["count"] == 1 + + assert dispatch_tool("get_keyword_summary", {}, context=AuditToolContext(), conn=conn)["error"] + assert dispatch_tool("search_keywords", {"property_id": 1}, context=ctx, conn=conn)["error"] + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + miss_lh = dispatch_tool("get_lighthouse_for_url", {"url": "https://ex.com/none"}, context=ctx, conn=conn) + assert "error" in miss_lh + + with patch.object(Ctx, "load_google", return_value=None): + no_g = dispatch_tool("get_google_summary", {}, context=ctx, conn=conn) + assert no_g["error"] + + assert dispatch_tool("get_health_history", {}, context=AuditToolContext(), conn=conn)["error"] + + with patch.object(Ctx, "load_crawl_df", return_value=df): + url_search = dispatch_tool("search_pages", {"url_contains": "missing"}, context=ctx, conn=conn) + assert url_search["total"] == 1 + + with patch.object(Ctx, "load_crawl_df", return_value=df), patch.object( + Ctx, "load_payload", return_value=_sample_payload(), + ), patch( + "website_profiling.tools.audit_tools.crawl.slice_from_google_row", + return_value={}, + ): + found = dispatch_tool( + "get_page_details", + {"url": "https://ex.com/ok"}, + context=ctx, + conn=conn, + ) + assert found["found_in_crawl"] is True + + assert dispatch_tool("get_internal_links", {}, context=ctx, conn=conn)["error"] + + bad_prop = dispatch_tool("get_property", {"property_id": "bad"}, conn=conn) + assert bad_prop["error"] == "invalid property_id" + + with patch.object(Ctx, "load_payload", return_value=_sample_payload()): + cat_filter = dispatch_tool( + "list_issues", + {"category_id": "technical_seo"}, + context=ctx, + conn=conn, + ) + assert cat_filter["total"] == 3 diff --git a/tests/test_chat_agent.py b/tests/test_chat_agent.py new file mode 100644 index 00000000..a15c15e4 --- /dev/null +++ b/tests/test_chat_agent.py @@ -0,0 +1,90 @@ +"""Tests for chat agent loop.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from website_profiling.llm.agent import MAX_TOOL_ROUNDS, run_agent_turn +from website_profiling.llm.base import ChatResult, ToolCall +from website_profiling.tools.audit_tools import AuditToolContext + + +class FakeToolClient: + def __init__(self, steps: list[ChatResult]) -> None: + self._steps = list(steps) + self._calls = 0 + + def chat_with_tools(self, messages, tools, *, on_token=None): + result = self._steps[min(self._calls, len(self._steps) - 1)] + self._calls += 1 + if on_token and result.content: + on_token(result.content) + return result + + +def test_agent_tool_then_answer() -> None: + client = FakeToolClient([ + ChatResult(tool_calls=[ToolCall(id="tc1", name="list_issues", arguments={"limit": 5})]), + ChatResult(content="Found 3 critical issues."), + ]) + events: list[dict] = [] + ctx = AuditToolContext(property_id=1) + + with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={ + "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test", + }): + with patch("website_profiling.llm.agent.get_llm_client", return_value=client): + with patch( + "website_profiling.llm.agent.dispatch_tool", + return_value={"issues": [], "total": 0}, + ) as mock_dispatch: + result = run_agent_turn( + [{"role": "user", "content": "What are the top issues?"}], + ctx, + on_event=events.append, + ) + + assert result["ok"] is True + assert "critical" in result["message"].lower() + mock_dispatch.assert_called_once() + types = [e["type"] for e in events] + assert "tool_start" in types + assert "tool_end" in types + assert "done" in types + + +def test_agent_disabled_llm() -> None: + events: list[dict] = [] + with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={ + "llm_enabled": False, "llm_provider": "none", + }): + result = run_agent_turn( + [{"role": "user", "content": "Hi"}], + AuditToolContext(), + on_event=events.append, + ) + assert result["ok"] is False + assert events[-1]["type"] == "error" + + +def test_max_tool_rounds() -> None: + always_tool = ChatResult( + tool_calls=[ToolCall(id="x", name="list_properties", arguments={})], + ) + client = FakeToolClient([always_tool] * (MAX_TOOL_ROUNDS + 1)) + ctx = AuditToolContext() + + with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={ + "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test", + }): + with patch("website_profiling.llm.agent.get_llm_client", return_value=client): + with patch( + "website_profiling.llm.agent.dispatch_tool", + return_value={"properties": []}, + ): + result = run_agent_turn( + [{"role": "user", "content": "List properties"}], + ctx, + ) + + assert result["ok"] is False + assert "maximum tool rounds" in result["error"].lower() diff --git a/tests/test_chat_store.py b/tests/test_chat_store.py new file mode 100644 index 00000000..f6f54625 --- /dev/null +++ b/tests/test_chat_store.py @@ -0,0 +1,93 @@ +"""Unit tests for chat_store.""" +from __future__ import annotations + +from datetime import datetime, timezone + +from tests.db_test_fakes import FakeConn, FakeCursor + +from website_profiling.db.chat_store import ( + append_message, + create_session, + delete_session, + get_messages, + get_session, + list_sessions, + update_session_title, +) + + +def test_create_session_returns_id() -> None: + conn = FakeConn() + conn.set_next_cursor(FakeCursor(fetchone_value={"id": 42})) + sid = create_session(conn, 7, "Test chat") + assert sid == 42 + assert conn.commits == 1 + + +def test_list_sessions() -> None: + conn = FakeConn() + now = datetime.now(timezone.utc) + conn.set_next_cursor( + FakeCursor( + fetchall_value=[ + {"id": 1, "property_id": 7, "title": "Chat A", "created_at": now, "updated_at": now}, + ], + ), + ) + rows = list_sessions(conn, 7) + assert len(rows) == 1 + assert rows[0]["title"] == "Chat A" + assert rows[0]["property_id"] == 7 + + +def test_get_session_missing() -> None: + conn = FakeConn() + conn.set_next_cursor(FakeCursor(fetchone_value=None)) + assert get_session(conn, 99) is None + + +def test_get_messages() -> None: + conn = FakeConn() + now = datetime.now(timezone.utc) + conn.set_next_cursor( + FakeCursor( + fetchall_value=[ + { + "id": 1, + "role": "user", + "content": "Hello", + "tool_name": None, + "tool_args": None, + "tool_result": None, + "created_at": now, + }, + ], + ), + ) + msgs = get_messages(conn, 5) + assert msgs[0]["content"] == "Hello" + assert msgs[0]["role"] == "user" + + +def test_append_message_and_touch() -> None: + conn = FakeConn() + conn.set_next_cursor(FakeCursor(fetchone_value={"id": 10})) + mid = append_message(conn, 3, "user", "Hi there") + assert mid == 10 + assert conn.commits == 1 + touch_sql = [sql for sql, _ in conn.executed if "UPDATE chat_sessions" in sql] + assert touch_sql + + +def test_update_session_title() -> None: + conn = FakeConn() + update_session_title(conn, 3, "Renamed") + assert conn.commits == 1 + + +def test_delete_session() -> None: + conn = FakeConn() + conn.set_next_cursor(FakeCursor(fetchone_value={"id": 3})) + assert delete_session(conn, 3) is True + conn.set_next_cursor(FakeCursor(fetchone_value=None)) + assert delete_session(conn, 99) is False diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py new file mode 100644 index 00000000..f25b742c --- /dev/null +++ b/tests/test_mcp_registry.py @@ -0,0 +1,26 @@ +"""MCP tool registry validation (no live MCP process).""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool + + +def test_tool_definitions_schema() -> None: + for tool in TOOL_DEFINITIONS: + assert tool.get("name") + assert tool.get("description") + schema = tool.get("inputSchema") + assert isinstance(schema, dict) + assert schema.get("type") == "object" + + +def test_dispatch_list_properties_roundtrip() -> None: + conn = MagicMock() + props = [{"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}] + with patch( + "website_profiling.tools.audit_tools.properties.list_properties_public", + return_value=props, + ): + result = dispatch_tool("list_properties", {}, conn=conn) + assert result["count"] == 1 diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.ts new file mode 100644 index 00000000..9861fa6d --- /dev/null +++ b/web/app/api/chat/route.ts @@ -0,0 +1,177 @@ +import { type NextRequest } from 'next/server'; +import { spawn } from 'child_process'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuthForChat } from '@/server/auth'; +import { getPipelineSpawnEnv, getRepoRoot } from '@/server/pipelineSpawnEnv'; +import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; +import { + appendChatMessage, + getChatMessages, + getChatSession, + messagesForAgentContext, + updateChatSessionTitle, +} from '@/server/chatDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +const CHAT_TIMEOUT_MS = 120_000; + +interface ChatBody { + sessionId?: number; + message?: string; + propertyId?: number; + reportId?: number; +} + +function sseLine(event: string, data: Record): string { + return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; +} + +/** POST /api/chat — stream agent response via SSE. */ +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + let body: ChatBody; + try { + body = await request.json(); + } catch { + return new Response(JSON.stringify({ error: 'Invalid JSON' }), { status: 400 }); + } + + const sessionId = Number(body.sessionId || 0); + const propertyId = Number(body.propertyId || 0); + const message = String(body.message || '').trim(); + const reportId = body.reportId != null ? Number(body.reportId) : undefined; + + if (!sessionId || !propertyId || !message) { + return new Response( + JSON.stringify({ error: 'sessionId, propertyId, and message are required' }), + { status: 400 }, + ); + } + + const session = await getChatSession(sessionId); + if (!session || session.property_id !== propertyId) { + return new Response(JSON.stringify({ error: 'session not found' }), { status: 404 }); + } + + await appendChatMessage(sessionId, 'user', message); + + const history = await getChatMessages(sessionId); + const agentMessages = messagesForAgentContext(history, 20); + + const repoRoot = getRepoRoot(); + const pythonExe = resolvePythonExecutable(null, repoRoot); + const stdinPayload = JSON.stringify({ + messages: agentMessages, + property_id: propertyId, + report_id: Number.isFinite(reportId) ? reportId : undefined, + }); + + const stream = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + let assistantText = ''; + let buffer = ''; + let timedOut = false; + + const push = (event: string, data: Record) => { + controller.enqueue(encoder.encode(sseLine(event, data))); + }; + + const proc = spawn( + pythonExe, + ['-m', 'src', 'chat', '--stdin-json'], + { + cwd: repoRoot, + env: getPipelineSpawnEnv(repoRoot, propertyId), + shell: false, + }, + ); + + const timer = setTimeout(() => { + timedOut = true; + try { + proc.kill(); + } catch { + /* ignore */ + } + push('error', { message: 'Chat timed out after 120s' }); + controller.close(); + }, CHAT_TIMEOUT_MS); + + proc.stdin?.write(stdinPayload); + proc.stdin?.end(); + + proc.stdout?.on('data', (chunk: Buffer) => { + buffer += chunk.toString(); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const evt = JSON.parse(trimmed) as { type?: string; text?: string; message?: string }; + if (evt.type === 'token' && evt.text) { + assistantText += evt.text; + push('token', { text: evt.text }); + } else if (evt.type === 'tool_start') { + push('tool_start', evt as Record); + } else if (evt.type === 'tool_end') { + push('tool_end', evt as Record); + } else if (evt.type === 'done' && evt.message) { + assistantText = evt.message; + push('done', { message: evt.message }); + } else if (evt.type === 'error') { + push('error', { message: evt.message || 'Agent error' }); + } + } catch { + /* ignore non-JSON log lines */ + } + } + }); + + proc.stderr?.on('data', () => { + /* stderr logged by python; not forwarded to client */ + }); + + proc.on('error', (err: Error) => { + clearTimeout(timer); + push('error', { message: formatPythonSpawnError(err, pythonExe, repoRoot) }); + controller.close(); + }); + + proc.on('close', async () => { + clearTimeout(timer); + if (timedOut) return; + + if (assistantText.trim()) { + try { + await appendChatMessage(sessionId, 'assistant', assistantText.trim()); + if (session.title === 'New chat') { + const title = message.slice(0, 60) + (message.length > 60 ? '…' : ''); + await updateChatSessionTitle(sessionId, title); + } + } catch { + /* persistence failure should not break stream */ + } + } + + controller.close(); + }); + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); +}; diff --git a/web/app/api/chat/sessions/[id]/messages/route.ts b/web/app/api/chat/sessions/[id]/messages/route.ts new file mode 100644 index 00000000..e4f17d4c --- /dev/null +++ b/web/app/api/chat/sessions/[id]/messages/route.ts @@ -0,0 +1,33 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuthForChat } from '@/server/auth'; +import { getChatMessages } from '@/server/chatDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** GET /api/chat/sessions/[id]/messages */ +export const GET: ApiRouteHandler = async ( + request: NextRequest, + context?: { params?: Promise<{ id: string }> }, +): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + const params = context?.params ? await context.params : { id: '' }; + const sessionId = Number(params.id || '0'); + if (!sessionId) { + return NextResponse.json({ error: 'invalid session id' }, { status: 400 }); + } + + try { + const messages = await getChatMessages(sessionId); + return NextResponse.json({ messages }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/chat/sessions/[id]/route.ts b/web/app/api/chat/sessions/[id]/route.ts new file mode 100644 index 00000000..9b9f0c7d --- /dev/null +++ b/web/app/api/chat/sessions/[id]/route.ts @@ -0,0 +1,36 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuthForChat } from '@/server/auth'; +import { deleteChatSession } from '@/server/chatDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** DELETE /api/chat/sessions/[id] */ +export const DELETE: ApiRouteHandler = async ( + request: NextRequest, + context?: { params?: Promise<{ id: string }> }, +): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + const params = context?.params ? await context.params : { id: '' }; + const sessionId = Number(params.id || '0'); + if (!sessionId) { + return NextResponse.json({ error: 'invalid session id' }, { status: 400 }); + } + + try { + const deleted = await deleteChatSession(sessionId); + if (!deleted) { + return NextResponse.json({ error: 'session not found' }, { status: 404 }); + } + return NextResponse.json({ ok: true }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/api/chat/sessions/route.ts b/web/app/api/chat/sessions/route.ts new file mode 100644 index 00000000..01cb9195 --- /dev/null +++ b/web/app/api/chat/sessions/route.ts @@ -0,0 +1,57 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuthForChat } from '@/server/auth'; +import { createChatSession, listChatSessions } from '@/server/chatDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** GET /api/chat/sessions?propertyId= — list chat sessions for a property. */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + const propertyId = Number(request.nextUrl.searchParams.get('propertyId') || '0'); + if (!propertyId) { + return NextResponse.json({ error: 'propertyId required' }, { status: 400 }); + } + + try { + const sessions = await listChatSessions(propertyId); + return NextResponse.json({ sessions }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; + +/** POST /api/chat/sessions — create session { propertyId, title? }. */ +export const POST: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + let body: { propertyId?: number; title?: string }; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 }); + } + + const propertyId = Number(body.propertyId || 0); + if (!propertyId) { + return NextResponse.json({ error: 'propertyId required' }, { status: 400 }); + } + + try { + const id = await createChatSession(propertyId, body.title); + return NextResponse.json({ id, propertyId, title: body.title?.trim() || 'New chat' }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/app/chat/page.tsx b/web/app/chat/page.tsx new file mode 100644 index 00000000..c1490b3a --- /dev/null +++ b/web/app/chat/page.tsx @@ -0,0 +1,7 @@ +import ChatPage from '@/views/Chat'; + +export const dynamic = 'force-dynamic'; + +export default function ChatRoutePage() { + return ; +} diff --git a/web/src/components/chat/ChatComposer.tsx b/web/src/components/chat/ChatComposer.tsx new file mode 100644 index 00000000..b70cd474 --- /dev/null +++ b/web/src/components/chat/ChatComposer.tsx @@ -0,0 +1,54 @@ +'use client'; + +import { useState, type FormEvent } from 'react'; +import { Loader2, Send } from 'lucide-react'; +import { strings } from '@/lib/strings'; + +const c = strings.components.chat; + +export interface ChatComposerProps { + disabled?: boolean; + busy?: boolean; + onSend: (message: string) => void; +} + +export default function ChatComposer({ disabled, busy, onSend }: ChatComposerProps) { + const [text, setText] = useState(''); + + const submitDisabled = disabled || busy || !text.trim(); + + const handleSubmit = (e: FormEvent) => { + e.preventDefault(); + const msg = text.trim(); + if (!msg || submitDisabled) return; + onSend(msg); + setText(''); + }; + + return ( +
+