diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..c37edd1c
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,17 @@
+.git
+.mypy_cache
+.pytest_cache
+.qdrant_db
+.ruff_cache
+**/__pycache__/
+**/*.pyc
+.DS_Store
+
+data/ml/
+eval/results/
+htmlcov/
+.coverage
+
+.env
+.env.*
+!.env.example
diff --git a/Dockerfile b/Dockerfile
index 70edaca9..45bb5e3e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,38 +1,22 @@
-# ClickAdvisor MCP Server
-# Supports:
-# Local: docker run -p 8000:8000 clickadvisor-mcp
-# Railway: auto-detects PORT env var
-
FROM python:3.11-slim
WORKDIR /app
-# Install system deps
-RUN apt-get update && apt-get install -y --no-install-recommends \
- curl \
- && rm -rf /var/lib/apt/lists/*
-
-# Install poetry
-RUN pip install --no-cache-dir poetry==1.8.2
-
-# Copy only dependency files first (layer cache)
-COPY pyproject.toml poetry.lock* ./
+ENV PYTHONUNBUFFERED=1 \
+ PYTHONDONTWRITEBYTECODE=1 \
+ PORT=8000 \
+ MCP_PATH=/mcp \
+ PYTHONPATH=/app
-# Install dependencies without dev extras
-RUN poetry config virtualenvs.create false \
- && poetry install --no-interaction --no-ansi --only main
+COPY requirements-mcp.txt ./
+RUN pip install --no-cache-dir -r requirements-mcp.txt
-# Copy source
COPY clickadvisor/ ./clickadvisor/
-COPY README.md ./
+COPY docs/rules/cards/ ./docs/rules/cards/
-# Health check
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
- CMD curl -f http://localhost:${PORT:-8000}/health || exit 1
-
-# Railway / Render inject PORT automatically; fall back to 8000 locally
-ENV PORT=8000
+EXPOSE 8000
-EXPOSE ${PORT}
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+ CMD python -c "import os, socket; s=socket.create_connection(('127.0.0.1', int(os.environ.get('PORT', '8000'))), 5); s.close()"
-CMD ["sh", "-c", "python -m clickadvisor.mcp_server --transport sse --host 0.0.0.0 --port ${PORT}"]
+CMD ["sh", "-c", "python -c 'import os; from clickadvisor.mcp_server.server import run_http; run_http(host=\"0.0.0.0\", port=int(os.environ.get(\"PORT\", \"8000\")), path=os.environ.get(\"MCP_PATH\", \"/mcp\"))'"]
diff --git a/README.en.md b/README.en.md
index 91fec28f..62ce8c47 100644
--- a/README.en.md
+++ b/README.en.md
@@ -1,91 +1,76 @@
-# ClickAdvisor
-
-
-
-
-
-
-
-[](README.md)
-[](README.en.md)
-
-> Local-first CLI and MCP server for ClickHouse SQL analysis.
-> It detects anti-patterns, suggests safe rewrites, and explains why a
-> recommendation applies to ClickHouse.
-
-ClickAdvisor helps DBAs, data engineers, and backend developers investigate slow
-or risky ClickHouse queries faster. It accepts SQL, optionally uses schema DDL,
-EXPLAIN, and environment context, applies deterministic rules, and returns a
-console, JSON, or Markdown report.
-
-[Project website](https://clickadvisor.lovable.app)
-
-## Why ClickAdvisor If ChatGPT Already Exists?
-
-ChatGPT, Claude, and other assistants are excellent for thinking, coding, and
-exploring hypotheses. ClickAdvisor solves a different layer of the problem:
-repeatable ClickHouse SQL checks where determinism, versioned rules, audit
-trails, and data control matter.
-
-- Every finding has a `rule_id`, tier, ClickHouse version, and applicability conditions.
-- Version-aware filtering hides rules that do not match the selected ClickHouse version.
-- The tier model separates provable rewrites from approximate and advisory recommendations.
-- `--mode explain` explains ClickHouse behavior in practical language.
-- Local retrieval can add documentation links, but it does not replace the rule engine.
-
-This is especially important for enterprise environments. In banks, telecom,
-government, and other regulated sectors, SQL, DDL, EXPLAIN output, and
-environment details often fall under compliance requirements.
-
-By default, ClickAdvisor runs locally. When used through CLI, Docker, or CI
-inside your own environment, user SQL is not sent to external servers and is not
-sent to a generative LLM unless you explicitly do so. The MCP server is also
-local, but if you connect it to an external AI client, data sharing depends on
-what you send to that client and on your organization's security policies.
+
+ 
+
+
+ClickAdvisor
+
+
+ Local-first ClickHouse Performance Advisor for SQL, workload review, and AI-agent workflows.
+
+
+
+ 
+ 
+
+
+
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+
+ClickAdvisor helps DBAs, data engineers, backend engineers, and platform teams
+find risky ClickHouse SQL patterns before production incidents, CPU/RAM waste,
+and expensive cloud bills.
+
+It is not a generic SQL chatbot. The trusted runtime is built around a
+deterministic ClickHouse rule engine: every finding has a `rule_id`, severity,
+tier, confidence, explanation, and rewrite example where the rewrite is safe.
+AI is used as an interface through MCP and as a research workflow, not as the
+source of production recommendations.
+
+## Contents
+
+- [Capabilities](#capabilities)
+- [Quick start](#quick-start)
+- [Single-query advisor](#single-query-advisor)
+- [Workload analyzer](#workload-analyzer)
+- [MCP deployment](#mcp-deployment)
+- [Data Science and ML](#data-science-and-ml)
+- [Evaluation](#evaluation)
+- [Security](#security)
+- [Project criteria](#project-criteria)
+- [Documentation](#documentation)
## Capabilities
-- Analyze ClickHouse SQL without connecting to a database.
-- Use ClickHouse version through `--ch-version` or HTTP `SELECT version()`.
-- Read optional inputs: `--schema`, `--explain`, `--environment`.
-- Produce `console`, `json`, and `markdown` reports.
-- Expose ClickAdvisor to AI agents as a local MCP server.
-- Add local links to ClickHouse docs / Altinity KB through embedded Qdrant.
-- Compare rewrite candidates with `EXPLAIN ESTIMATE` when `--connect` is explicitly provided.
-- Build prototype workload reports from sanitized `system.query_log` CSV exports.
+| Surface | Capability |
+|---|---|
+| SQL advisor | Analyze one ClickHouse SQL query through CLI, JSON, Markdown, or MCP |
+| Rule engine | 119 ClickHouse-specific rules, detectors, and environment checks |
+| Workload analyzer | `system.query_log` CSV/live analysis, normalized fingerprints, top-N risks |
+| EXPLAIN ESTIMATE | Planner rows/marks comparison without executing user queries |
+| MCP server | Local stdio MCP and Streamable HTTP MCP for remote-compatible demos |
+| Local retrieval | Embedded Qdrant KB over ClickHouse docs, Altinity KB, blog/release notes |
+| DS/ML lab | Expert dataset, EDA, features, group split, baselines, error analysis |
-## Project Numbers
-
-| Area | Value |
-|---|---:|
-| SQL/config/workload analysis patterns | 119 |
-| Rewrite/advisory rules `R-*` | 74 |
-| Anti-pattern detectors `D-*` | 25 |
-| Environment rules `E-*` | 20 |
-| Total benchmark YAML cases | 327 |
-| `synthetic_expanded` benchmark cases | 222 |
-| Unit / validation / benchmark tests excluding integration | 325 |
-
-## Architecture
-
-```mermaid
-flowchart LR
- SQL[SQL query] --> Parser[sqlglot parser]
- Schema[Schema DDL] --> Context[QueryContext]
- Env[environment.json] --> Context
- Explain[EXPLAIN] --> Context
- Parser --> Context
- Context --> Rules[119 rules]
- Rules --> Report[Console / JSON / Markdown / MCP]
- KB[Local knowledge base] --> Retrieval[retrieval advisory]
- Retrieval --> Report
-```
+## Why this matters for ClickHouse
-More details: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
+ClickHouse can execute analytical queries extremely fast, but performance
+depends on engine details: MergeTree, sparse primary keys, marks/parts, `FINAL`,
+skip indexes, `PREWHERE`, materialized views, distributed execution,
+memory/thread settings, and the actual workload.
-## Quick Start
+ClickAdvisor solves a practical problem: it does not “guess an optimization”.
+It produces a verifiable ClickHouse-specific signal that can be reviewed by a
+DBA, used in CI, passed to an AI agent through MCP, or turned into a workload
+review queue.
-### From Source
+## Quick start
```bash
git clone https://github.com/olyannaa/clickadvisor.git
@@ -94,16 +79,28 @@ poetry install
poetry run chadvisor analyze --sql query.sql
```
-### Docker
+Docker:
```bash
docker build -t clickadvisor .
-docker run --rm -v "$(pwd)":/queries clickadvisor analyze --sql /queries/query.sql
+docker run --rm -p 8000:8000 clickadvisor
```
-## Real Example
+By default, the Docker image starts a lightweight Streamable HTTP MCP endpoint
+at `/mcp`. The image is meant for demo/deploy use and does not install heavy
+ML/retrieval dependencies.
-`query.sql`:
+## Single-query advisor
+
+```bash
+poetry run chadvisor analyze \
+ --sql query.sql \
+ --ch-version 25.3 \
+ --output-format markdown \
+ --no-retrieval
+```
+
+Example risky query:
```sql
SELECT
@@ -124,107 +121,119 @@ HAVING e.country = 'RU'
ORDER BY paid_revenue DESC;
```
-Run:
-
-```bash
-poetry run chadvisor analyze \
- --sql query.sql \
- --ch-version 25.3 \
- --output-format markdown \
- --no-retrieval
-```
-
-On this query, ClickAdvisor reports 10 findings, including:
-
-- `R-001`: replace `COUNT(DISTINCT user_id)` with `uniqExact(user_id)`;
-- `R-002`: consider `uniq(user_id)` when approximate distinct is acceptable;
-- `D-005` and `R-102`: `LIKE '%...'` may need a skip index or another search strategy;
-- `D-007`: `FINAL` can be expensive on large MergeTree tables;
-- `D-011`, `R-008`, `R-020`: check type conversion around the JOIN key;
-- `R-011`: move a non-aggregate `HAVING` predicate to `WHERE`.
-- `R-014`: `GROUP BY` over a string column can be expensive and needs type/cardinality review.
+For this query, ClickAdvisor returns 10 findings, including:
-Actual CLI output excerpt:
+- `R-001`: `COUNT(DISTINCT user_id)` -> `uniqExact(user_id)`;
+- `R-002`: approximate distinct with `uniq` when acceptable;
+- `D-005` / `R-102`: leading wildcard search and skip-index/search strategy;
+- `D-007`: expensive `FINAL` on MergeTree;
+- `D-011`, `R-008`, `R-020`: type casts around JOIN/filter keys;
+- `R-011`: non-aggregate `HAVING` can move to `WHERE`;
+- `R-014`: expensive `GROUP BY` over a string column.
-
+
+ 
+
-## CLI Usage
+## Workload analyzer
-### Basic Analysis
+CSV export from `system.query_log`:
```bash
-poetry run chadvisor analyze --sql query.sql
+poetry run chadvisor workload \
+ --query-log examples/query_log_sample.csv \
+ --output-format markdown \
+ --top-n 3
```
-### ClickHouse Version
+Live read-only mode through the ClickHouse HTTP API:
```bash
-poetry run chadvisor analyze --sql query.sql --ch-version 25.3
+poetry run chadvisor workload \
+ --connect http://localhost:8123 \
+ --user default \
+ --password secret \
+ --since 24h \
+ --output-format markdown \
+ --top-n 10
```
-The version is used to filter rules by `ch_version_introduced`.
+`workload` groups similar queries by normalized fingerprint, computes
+executions, total/avg/p95 latency, read rows/bytes, and memory usage, then runs
+representative SQL through the rule engine and builds a top-N DBA review queue.
-### Version Detection Through HTTP API
+Example top risk from the sample:
-```bash
-poetry run chadvisor analyze --sql query.sql \
- --connect http://localhost:8123 \
- --ch-user default \
- --ch-password secret
+```text
+Priority: high
+Executions: 2
+Total duration: 2180 ms
+Read bytes: 350000000
+Rule IDs: D-003, D-004, D-005, D-007, R-102
+Normalized SQL: select * from events final where message like ?
```
-ClickAdvisor only runs `SELECT version()` and normalizes responses such as
-`25.3.2.39` -> `25.3`.
+More details: [docs/workload.md](docs/workload.md).
+
+## MCP deployment
-### Explain Mode
+Local stdio MCP for Claude Desktop, Cursor, Zed, and other MCP clients:
```bash
-poetry run chadvisor analyze --sql query.sql --mode explain
+poetry run chadvisor mcp-server
```
-Explain mode describes why the recommendation matters for ClickHouse: sparse
-primary key indexes, granules, `WHERE` / `HAVING` order, `FINAL`, `UNION` vs
-`UNION ALL`, and related concepts.
-
-### Output Formats
+Streamable HTTP MCP endpoint for remote-compatible demos:
```bash
-poetry run chadvisor analyze --sql query.sql --output-format console
-poetry run chadvisor analyze --sql query.sql --output-format json
-poetry run chadvisor analyze --sql query.sql --output-format markdown
+poetry run chadvisor mcp-http-server \
+ --host 127.0.0.1 \
+ --port 8765 \
+ --path /mcp
```
-`console` is useful locally, `json` fits CI/CD, and `markdown` works well for PR
-comments and MCP responses.
+Endpoint:
-### EXPLAIN ESTIMATE
+```text
+http://127.0.0.1:8765/mcp
+```
-```bash
-poetry run chadvisor analyze --sql query.sql \
- --connect http://localhost:8123 \
- --ch-user default \
- --ch-password secret \
- --explain-estimate
+For a defense/demo, the server can run as a remote endpoint behind HTTPS/auth:
+VPS + reverse proxy, Railway/Render/Fly.io, Cloudflare Tunnel, or Tailscale.
+A public unauthenticated MCP endpoint is not recommended because SQL and query
+context are sensitive.
+
+Fastest path to a public link:
+
+1. push the repository to GitHub;
+2. create a Railway project from the GitHub repo or a Render Blueprint;
+3. let the platform build the root `Dockerfile`;
+4. generate/open the public domain;
+5. share an endpoint like:
+
+```text
+https://.up.railway.app/mcp
+https://.onrender.com/mcp
```
-ClickAdvisor compares the original SQL and rewrite candidate through `EXPLAIN
-ESTIMATE`. It does not execute the query, run `ANALYZE`, or read user table
-data.
+More details:
-### Schema, EXPLAIN, And Environment
+- [docs/MCP.md](docs/MCP.md)
+- [docs/mcp-deployment.md](docs/mcp-deployment.md)
+- [docs/ai-mcp-workflow.md](docs/ai-mcp-workflow.md)
+
+## Schema, EXPLAIN, and environment
```bash
-poetry run chadvisor analyze --sql query.sql \
+poetry run chadvisor analyze \
+ --sql query.sql \
--schema schema.sql \
- --environment environment.json \
- --explain explain.json
+ --explain explain.json \
+ --environment environment.json
```
-`environment.json` carries settings, hardware, cluster/workload facts, and
-system metrics for `E-*` rules and some Tier 2 advisory rules. If no
-environment file is provided, environment rules do not fire.
-
-Minimal example:
+Environment context includes settings, hardware, cluster, and workload facts
+for `E-*` rules and selected Tier 2 advisory rules:
```json
{
@@ -241,171 +250,150 @@ Minimal example:
"workload": {
"interactive_queries": true,
"large_join": true,
- "bulk_inserts": true,
- "insert_format": "JSONEachRow"
+ "bulk_inserts": true
},
"cluster": {
"shards": 4,
- "replicas": 2,
- "has_local_replica": true
+ "replicas": 2
}
}
```
-## Knowledge Base And Retrieval Advisory
-
-The knowledge base is built in `/data/kb/` from ClickHouse docs, Altinity KB,
-ClickHouse blog posts, and release notes. To enable local semantic search,
-index Markdown chunks:
-
-```bash
-poetry run chadvisor index-kb
-```
-
-Reindex:
+EXPLAIN ESTIMATE:
```bash
-poetry run chadvisor index-kb --reindex
-```
-
-Embedding model selection:
-
-```bash
-poetry run chadvisor index-kb --embedding-model multilingual-e5-small
-poetry run chadvisor index-kb --embedding-model minilm-l6
+poetry run chadvisor analyze \
+ --sql query.sql \
+ --connect http://localhost:8123 \
+ --ch-user default \
+ --ch-password secret \
+ --explain-estimate
```
-After indexing, `.qdrant_db` is created locally. If it exists, `analyze` adds a
-section with relevant documentation snippets.
+ClickAdvisor runs only `EXPLAIN ESTIMATE`; it does not execute the user query
+and does not read result data.
-## MCP Server
+## Data Science and ML
-ClickAdvisor can be exposed to AI agents as a local MCP server:
+The DS layer is not meant to replace the rule engine. Its job is to formalize
+quality, compare approaches, expose limitations, and prepare a
+triage/prioritization layer.
-```bash
-poetry run chadvisor mcp-server
-```
+Expert dataset:
-Example `claude_desktop_config.json`:
+| Metric | Value |
+|---|---:|
+| SQL records | 20 235 |
+| Real / synthetic | 19 090 / 1 145 |
+| Successful local replay records | 9 837 |
+| Final labels | low 4 253 / medium 14 285 / high 1 697 |
+| Numeric feature count | 115 |
+| Rule vocabulary | 54 |
-```json
-{
- "mcpServers": {
- "clickadvisor": {
- "command": "poetry",
- "args": ["run", "chadvisor", "mcp-server"],
- "cwd": "/path/to/clickadvisor"
- }
- }
-}
-```
+Key label-source fact:
-Available MCP tools:
+| Label source | Records | Interpretation |
+|---|---:|---|
+| `rule_only` | 14 693 | model mostly learns a compressed rule-engine signal |
+| `measured_only` | 4 635 | independent signal from latency/read/memory metrics |
+| `both` | 907 | strongest core where static and measured signals agree |
-- `analyze_query` — Markdown SQL report;
-- `analyze_query_json` — structured JSON report;
-- `list_rules` — registered rules;
-- `detect_ch_version` — ClickHouse version detection through HTTP API.
+Baseline ladder:
-More details: [docs/MCP.md](docs/MCP.md).
+| Model | CV macro-F1 | Holdout macro-F1 |
+|---|---:|---:|
+| Dummy most frequent | 0.275 +/- 0.000 | 0.278 |
+| Dummy stratified | 0.328 +/- 0.009 | 0.335 |
+| TF-IDF + Logistic Regression | 0.864 +/- 0.011 | 0.882 |
+| Structural/rule LR | 0.827 +/- 0.004 | 0.837 |
+| Random Forest all features | 0.938 +/- 0.006 | 0.949 |
+| CatBoost tabular | 0.873 +/- 0.008 | 0.871 |
-## Workload Analyzer Prototype
+Random Forest holdout error analysis:
-To move from single-query review toward a DBA review queue, ClickAdvisor includes
-a prototype for sanitized `system.query_log` CSV exports:
+| Slice | Records | Macro-F1 | High recall |
+|---|---:|---:|---:|
+| all_holdout | 3 039 | 0.949 | 0.887 |
+| rule_only | 2 235 | 0.970 | 0.990 |
+| measured_only | 672 | 0.595 | 0.785 |
+| both | 132 | 0.975 | 1.000 |
-```bash
-poetry run chadvisor workload \
- --query-log examples/query_log_sample.csv \
- --output-format markdown \
- --top-n 3
-```
+Conclusion: ML is useful for triage, confidence grouping, and review queue
+ordering, but production recommendations remain rule-first.
-It groups similar queries by normalized fingerprint, computes executions,
-total/avg/p95 latency, read rows/bytes, and memory, then runs the representative
-query through the rule engine and ranks top opportunities.
+More details:
-More details: [docs/workload.md](docs/workload.md).
+- [docs/evaluation.md](docs/evaluation.md)
+- [docs/experiments/risk_labeling_ds_summary.md](docs/experiments/risk_labeling_ds_summary.md)
+- [data/ml/expert_dataset/eda/ds_report.md](data/ml/expert_dataset/eda/ds_report.md)
-## Quality Metrics
+## Evaluation
-Current reproducible metrics as of 2026-06-30:
-
-| Evaluation surface | Data | Metric |
+| Evaluation surface | Data | Result |
|---|---|---:|
-| Rule detection | `222` synthetic/schema/env benchmark cases | precision `1.000`, recall `1.000`, F1 `1.000` |
-| ML classifier | `synthetic_expanded_v1` train/test split | best test macro F1 `0.691`, best test micro F1 `0.988` |
-| Retrieval | `20` explicit query -> relevant docs pairs | MRR@3 `0.517` with MiniLM-L6 |
-| Risk-label DS pipeline | `20 235` SQL records, group split + holdout | RF holdout macro F1 `0.949`, measured-only macro F1 `0.595` |
-| Workload prototype | sample sanitized `query_log` CSV | normalized groups + top-N risk report |
-
-What was evaluated:
-
-- Rule detection: strict check that the analyzer returns exactly the expected `rule_id` set.
-- Classifier ablation: multi-label classification over AST/SQL features.
-- Retrieval ablation: `MRR@3` over explicit gold query -> relevant URL/keyword mappings.
+| Rule detection | 222 synthetic/schema/env cases | precision 1.000 / recall 1.000 / F1 1.000 |
+| Retrieval | 20 query -> docs pairs | best MRR@3 0.517 |
+| Risk-label DS | 20 235 SQL records | RF holdout macro-F1 0.949 |
+| Workload prototype | sample query_log CSV | normalized groups + top-N risk report |
-Run the expanded synthetic benchmark:
+Reproducible checks:
```bash
-poetry run python scripts/eval/run_benchmark.py \
- --cases-dir benchmark/cases/synthetic_expanded \
- --mode strict
+poetry run ruff check clickadvisor tests scripts
+poetry run mypy clickadvisor
+poetry run pytest --ignore=tests/integration -q
+poetry run python scripts/rules/validate_catalog.py
+poetry run python scripts/benchmark/validate_cases.py
+poetry run python scripts/eval/run_benchmark.py --cases-dir benchmark/cases/synthetic_expanded --mode strict
```
-Methodology: [docs/evaluation.md](docs/evaluation.md),
-[docs/experiments/classifier_ablation.md](docs/experiments/classifier_ablation.md),
-[docs/experiments/retrieval_ablation.md](docs/experiments/retrieval_ablation.md),
-[docs/experiments/risk_labeling_ds_summary.md](docs/experiments/risk_labeling_ds_summary.md).
+## Security
-## Security And Data Handling
+ClickAdvisor can run inside a company network, CI/CD, or an engineer's local
+environment without sending SQL, DDL, EXPLAIN, environment context, or
+`query_log` data to external LLM/API services.
-ClickAdvisor does not execute user SQL to measure speedup. When connected to
-ClickHouse, it only uses:
+What ClickAdvisor may read:
+- SQL text;
+- optional schema DDL;
+- optional EXPLAIN output;
+- optional environment JSON;
+- sanitized `system.query_log` CSV or read-only live metadata;
- `SELECT version()` for version detection;
-- `EXPLAIN ESTIMATE ...` when `--explain-estimate` is explicitly enabled.
+- `EXPLAIN ESTIMATE` only when explicitly requested.
-For basic analysis, a SQL file is enough. Schema, EXPLAIN, environment context,
-and ClickHouse connection are optional.
+What it does not do by default:
-For enterprise adoption, the key point is that ClickAdvisor can run inside the
-company perimeter, CI/CD, or an engineer's local environment without sending
-SQL, DDL, EXPLAIN, or environment context to external LLM/API services. This
-reduces risks around compliance, banking secrecy, personal data, trade secrets,
-and internal naming conventions.
+- does not execute user SQL for speedup measurement;
+- does not read result data;
+- does not run `ANALYZE`;
+- does not apply DDL/mutations;
+- does not make hidden remote LLM calls.
More details: [docs/security-local-first.md](docs/security-local-first.md).
-## Development
-
-```bash
-poetry install
-poetry run ruff check clickadvisor tests scripts
-poetry run mypy clickadvisor
-poetry run pytest --ignore=tests/integration
-poetry run python scripts/eval/run_benchmark.py
-```
-
-The version detection integration test expects a ClickHouse HTTP endpoint at
-`localhost:8123`. GitHub Actions starts it as a service container.
+## Project criteria
-## AI-Assisted Development
+| Criterion | How ClickAdvisor addresses it |
+|---|---|
+| Development | Poetry, Docker, GitHub Actions, typed core, linting, tests, validators, ClickHouse integration |
+| Data Science | EDA, feature extraction, group split, CV/holdout, baseline ladder, error analysis |
+| AI usage | MCP tools, local and remote-compatible agent workflow, DS/research automation narrative |
+| Product thinking | target users, pain, competitors/SOTA framing, MVP surfaces, impact path through workload analysis |
-Codex and Claude were used systematically during development for architecture
-review, test variants, documentation, and consistency checks. They are not part
-of ClickAdvisor's trusted runtime path: CLI/MCP recommendations are produced by
-the rule engine, ML evaluation surface, and local retrieval.
+Detailed self-assessment: [docs/project-readiness.en.md](docs/project-readiness.en.md).
-## Not Claimed As Ready
+## Documentation
-- Product generative LLM in the trusted runtime path.
-- Live `query_log` analysis through `--connect --since`; a CSV prototype exists.
-- Automatic DDL changes.
-- Running `ANALYZE` or replaying queries on user data.
-- Automatically applying Tier 2 design/storage recommendations without DBA review.
+- [Architecture](docs/ARCHITECTURE.md)
+- [Evaluation](docs/evaluation.md)
+- [Workload Analyzer](docs/workload.md)
+- [MCP](docs/MCP.md)
+- [MCP Deployment](docs/mcp-deployment.md)
+- [Security / Local-First](docs/security-local-first.md)
+- [Rule Catalog](docs/rules/README.md)
----
+## License
-[](README.md)
-[](README.en.md)
+MIT
diff --git a/README.md b/README.md
index 14838855..e6a8c919 100644
--- a/README.md
+++ b/README.md
@@ -1,351 +1,398 @@
-# ClickAdvisor
-
-**Local-first ClickHouse Performance Advisor**
-
-ClickAdvisor finds risky SQL patterns before they cause production incidents.
-It runs entirely on your machine — no SQL, DDL, or query context ever leaves your environment.
-
-```bash
-# Analyze a single query
-chadvisor analyze --sql "SELECT * FROM events FINAL WHERE date > '2024-01-01'"
-
-# Analyze a workload from query_log export
-chadvisor workload --query-log query_log.csv --output-format markdown
-
-# Analyze a live ClickHouse instance
-chadvisor workload --connect http://localhost:8123 --since 24
-```
-
----
-
-## Why ClickAdvisor
-
-Generic SQL assistants see SQL syntax. ClickAdvisor knows ClickHouse:
-
-- **Deterministic rule engine** — every finding has a `rule_id`, tier, severity, and confidence. No guessing.
-- **ClickHouse-specific** — `FINAL`, sparse primary key, marks/parts, `PREWHERE`, `LowCardinality`, distributed joins, MergeTree mutations.
-- **Local-first** — your SQL and query logs never touch an external LLM at runtime.
-- **MCP-ready** — AI agents (Claude, Cursor, Codex) call ClickAdvisor as a tool, not as a source of truth.
-- **Workload-aware** — goes beyond single queries to rank your most expensive query groups by actual resource consumption.
-
----
-
-## Installation
-
-**Requirements:** Python 3.11+, [Poetry](https://python-poetry.org/)
+
+
+
+
+ClickAdvisor
+
+
+ Local-first ClickHouse Performance Advisor для SQL, workload и AI-agent workflow.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ClickAdvisor помогает DBA, data engineers, backend engineers и platform-командам
+находить рискованные ClickHouse SQL-паттерны до production-инцидентов,
+перерасхода CPU/RAM и дорогих cloud-счетов.
+
+Это не generic SQL chatbot. Trusted runtime построен вокруг deterministic
+ClickHouse rule engine: каждое срабатывание имеет `rule_id`, severity, tier,
+confidence, объяснение и rewrite-пример там, где rewrite безопасен. AI
+используется как интерфейс через MCP и как исследовательский workflow, но не как
+источник production-рекомендаций.
+
+## Содержание
+
+- [Что умеет](#что-умеет)
+- [Быстрый старт](#быстрый-старт)
+- [Single-query advisor](#single-query-advisor)
+- [Workload analyzer](#workload-analyzer)
+- [MCP deployment](#mcp-deployment)
+- [Data Science и ML](#data-science-и-ml)
+- [Evaluation](#evaluation)
+- [Security](#security)
+- [Критерии проекта](#критерии-проекта)
+- [Документация](#документация)
+
+## Что умеет
+
+| Поверхность | Возможность |
+|---|---|
+| SQL advisor | Анализ одного ClickHouse SQL через CLI, JSON, Markdown или MCP |
+| Rule engine | 119 ClickHouse-specific правил, detectors и environment checks |
+| Workload analyzer | `system.query_log` CSV/live анализ, normalized fingerprints, top-N risks |
+| EXPLAIN ESTIMATE | Planner rows/marks comparison без выполнения пользовательского запроса |
+| MCP server | Local stdio MCP и Streamable HTTP MCP для remote-compatible demo |
+| Local retrieval | Embedded Qdrant KB по ClickHouse docs, Altinity KB, blog/release notes |
+| DS/ML lab | Expert dataset, EDA, features, group split, baselines, error analysis |
+
+## Почему это важно для ClickHouse
+
+ClickHouse быстро выполняет аналитические запросы, но performance зависит от
+деталей движка: MergeTree, sparse primary key, marks/parts, `FINAL`, skip
+indexes, `PREWHERE`, materialized views, distributed execution, memory/thread
+settings и реального workload.
+
+ClickAdvisor закрывает практическую задачу: не “угадать оптимизацию”, а дать
+проверяемый ClickHouse-specific сигнал, который можно показать DBA, положить в
+CI, передать AI-агенту через MCP или использовать для workload review queue.
+
+## Быстрый старт
```bash
git clone https://github.com/olyannaa/clickadvisor.git
cd clickadvisor
poetry install
-poetry run chadvisor --help
+poetry run chadvisor analyze --sql query.sql
```
-For live mode, add `httpx`:
+Docker:
+
```bash
-poetry install --extras live
+docker build -t clickadvisor .
+docker run --rm -p 8000:8000 clickadvisor
```
----
+По умолчанию Docker image поднимает lightweight Streamable HTTP MCP endpoint
+на `/mcp`. Образ предназначен для demo/deploy сценария и не ставит тяжёлые
+ML/retrieval зависимости.
-## Three modes of use
-
-### 1 — Single query analysis
+## Single-query Advisor
```bash
-chadvisor analyze \
- --sql "SELECT countDistinct(user_id) FROM events WHERE toDate(ts) = today()" \
- --schema schema.sql \
- --format markdown
+poetry run chadvisor analyze \
+ --sql query.sql \
+ --ch-version 25.3 \
+ --output-format markdown \
+ --no-retrieval
```
-Example output:
+Пример рискованного запроса:
+```sql
+SELECT
+ e.country,
+ COUNT(DISTINCT e.user_id) AS unique_users,
+ sumIf(e.revenue, e.status = 'paid') AS paid_revenue
+FROM
+(
+ SELECT *
+ FROM events FINAL
+ WHERE message LIKE '%timeout%'
+ AND (country = 'RU' OR country = 'KZ' OR country = 'BY')
+) AS e
+JOIN users AS u
+ ON toUInt64(e.user_id) = u.id
+GROUP BY e.country
+HAVING e.country = 'RU'
+ORDER BY paid_revenue DESC;
```
-## ClickAdvisor findings
-### D-007 · high · Function on filter column
-`toDate(ts)` wraps the primary key column. ClickHouse cannot use the sparse index.
-→ Rewrite: `WHERE ts >= today() AND ts < today() + INTERVAL 1 DAY`
+На этом запросе ClickAdvisor находит 10 срабатываний, включая:
-### R-102 · medium · Exact COUNT DISTINCT on high-cardinality column
-Consider `uniq(user_id)` if an approximate result is acceptable (~2% error).
-Exact `countDistinct` materializes all values in memory.
-```
+- `R-001`: `COUNT(DISTINCT user_id)` -> `uniqExact(user_id)`;
+- `R-002`: approximate distinct через `uniq`, если это допустимо;
+- `D-005` / `R-102`: leading wildcard search и skip-index/search strategy;
+- `D-007`: дорогой `FINAL` на MergeTree;
+- `D-011`, `R-008`, `R-020`: приведение типов вокруг JOIN/filter keys;
+- `R-011`: non-aggregate `HAVING` можно перенести в `WHERE`;
+- `R-014`: дорогой `GROUP BY` по строковой колонке.
-### 2 — Workload analysis (CSV mode)
+
+
+
-Export a sanitized snapshot from ClickHouse:
+## Workload Analyzer
-```sql
-SELECT
- query, query_duration_ms, read_rows, read_bytes, memory_usage,
- event_time, user, query_kind
-FROM system.query_log
-WHERE type = 'QueryFinish'
- AND event_time >= now() - INTERVAL 24 HOUR
-FORMAT CSVWithNames
-```
+CSV export из `system.query_log`:
```bash
-chadvisor workload \
- --query-log query_log.csv \
- --top-n 5 \
+poetry run chadvisor workload \
+ --query-log examples/query_log_sample.csv \
--output-format markdown \
- --output report.md
+ --top-n 3
```
-Example output:
+Live read-only режим через ClickHouse HTTP API:
+```bash
+poetry run chadvisor workload \
+ --connect http://localhost:8123 \
+ --user default \
+ --password secret \
+ --since 24h \
+ --output-format markdown \
+ --top-n 10
```
-# ClickHouse workload risk report
-## #1 · Dashboard fingerprint 8f3a… · RISK: HIGH
-Executions/day: 1 240 | Avg latency: 3.4 s | P95: 18.2 s
-Total read/day: 2.8 TB | Total CPU time: 4.7 h
+`workload` группирует похожие запросы по normalized fingerprint, считает
+executions, total/avg/p95 latency, read rows/bytes и memory usage, затем
+прогоняет representative SQL через rule engine и формирует top-N DBA review
+queue.
-Findings:
- D-007 · high · Function on filter column
- D-003 · high · FINAL on large table (est. 2× scan overhead)
- R-102 · medium · Exact COUNT DISTINCT
+Пример top risk из sample:
-→ DBA actions: verify FINAL necessity, inspect selected marks, test range rewrite
+```text
+Priority: high
+Executions: 2
+Total duration: 2180 ms
+Read bytes: 350000000
+Rule IDs: D-003, D-004, D-005, D-007, R-102
+Normalized SQL: select * from events final where message like ?
```
-### 3 — Live mode
+Подробнее: [docs/workload.md](docs/workload.md).
-Connect directly to a running ClickHouse instance:
+## MCP Deployment
+
+Локальный stdio MCP для Claude Desktop, Cursor, Zed и других MCP-клиентов:
```bash
-chadvisor workload \
- --connect http://localhost:8123 \
- --user default \
- --since 24 \
- --top-n 10 \
- --output-format markdown
+poetry run chadvisor mcp-server
```
-ClickAdvisor reads `system.query_log` for the last N hours, groups queries by normalized fingerprint, ranks by total resource cost, and runs the full rule engine on each group.
-
-> **Security note:** Only `SELECT` queries from `system.query_log` are read. No data is written. Credentials stay local.
+Streamable HTTP MCP endpoint для remote-compatible demo:
----
-
-## MCP integration
+```bash
+poetry run chadvisor mcp-http-server \
+ --host 127.0.0.1 \
+ --port 8765 \
+ --path /mcp
+```
-ClickAdvisor exposes a [Model Context Protocol](https://modelcontextprotocol.io/) server so AI agents can call it as a trusted tool — not generate advice themselves.
+Endpoint:
-```
-AI agent (Claude / Cursor / Codex)
- │
- │ MCP tool call: analyze_sql(sql, schema)
- ▼
-ClickAdvisor rule engine ←── deterministic, local, no LLM
- │
- │ structured findings: rule_id, severity, rewrite example
- ▼
-AI agent presents findings to the user
+```text
+http://127.0.0.1:8765/mcp
```
-The agent gets structured, explainable findings. It never invents ClickHouse advice.
+Для защиты можно поднять удалённый demo endpoint behind HTTPS/auth proxy:
+VPS + reverse proxy, Railway/Render/Fly.io, Cloudflare Tunnel или Tailscale.
+Публичный unauthenticated MCP endpoint не рекомендуется: SQL и query context
+считаются чувствительными.
-### Option A — Remote MCP server (for demos and team use)
+Самый быстрый путь получить публичную ссылку:
-Connect Claude Desktop or any MCP client to the public instance:
+1. push репозитория в GitHub;
+2. создать Railway project from GitHub repo или Render Blueprint;
+3. платформа соберёт root `Dockerfile`;
+4. открыть public domain;
+5. передать экспертам URL вида:
-```json
-{
- "mcpServers": {
- "clickadvisor": {
- "transport": "sse",
- "url": "https://clickadvisor-mcp.up.railway.app/sse"
- }
- }
-}
+```text
+https://.up.railway.app/mcp
+https://.onrender.com/mcp
```
-No installation needed. The remote server runs the rule engine only — your SQL is processed in memory and never stored.
+Подробнее:
-### Option B — Local MCP server (for enterprise / NDA environments)
+- [docs/MCP.md](docs/MCP.md)
+- [docs/mcp-deployment.md](docs/mcp-deployment.md)
+- [docs/ai-mcp-workflow.md](docs/ai-mcp-workflow.md)
-Run the server on your own machine in one command:
+## Schema, EXPLAIN и Environment
```bash
-docker run -p 8000:8000 ghcr.io/olyannaa/clickadvisor-mcp:latest
-```
-
-Then add to Claude Desktop config:
-
-```json
-{
- "mcpServers": {
- "clickadvisor": {
- "transport": "sse",
- "url": "http://localhost:8000/sse"
- }
- }
-}
+poetry run chadvisor analyze \
+ --sql query.sql \
+ --schema schema.sql \
+ --explain explain.json \
+ --environment environment.json
```
-Or via stdio (no Docker required):
+Environment context включает настройки, hardware, cluster и workload facts для
+`E-*` и части Tier 2 advisory rules:
```json
{
- "mcpServers": {
- "clickadvisor": {
- "command": "poetry",
- "args": ["run", "python", "-m", "clickadvisor.mcp_server"],
- "cwd": "/path/to/clickadvisor"
- }
+ "settings": {
+ "max_threads": 64,
+ "max_memory_usage": 90000000000,
+ "join_use_nulls": true
+ },
+ "hardware": {
+ "cpu_cores": 16,
+ "ram_bytes": 128000000000,
+ "disk_type": "hdd"
+ },
+ "workload": {
+ "interactive_queries": true,
+ "large_join": true,
+ "bulk_inserts": true
+ },
+ "cluster": {
+ "shards": 4,
+ "replicas": 2
}
}
```
-### Available MCP tools
-
-| Tool | Description |
-|------|-------------|
-| `analyze_sql` | Analyze a single SQL query |
-| `analyze_with_schema` | Analyze SQL with DDL context |
-| `analyze_workload_csv` | Analyze query_log CSV export |
-| `list_rules` | List all available rules with descriptions |
-| `explain_finding` | Get detailed explanation for a rule_id |
-
----
-
-## Architecture
+EXPLAIN ESTIMATE:
+```bash
+poetry run chadvisor analyze \
+ --sql query.sql \
+ --connect http://localhost:8123 \
+ --ch-user default \
+ --ch-password secret \
+ --explain-estimate
```
-SQL / DDL / EXPLAIN / query_log
- │
- ▼
- ┌─────────────────────────┐
- │ Deterministic rule │ ← trusted core, no LLM
- │ engine │
- │ · 40+ ClickHouse rules │
- │ · rule_id / tier / │
- │ severity / confidence│
- └────────────┬────────────┘
- │
- ┌──────────┼──────────┐
- ▼ ▼ ▼
- CLI/JSON Markdown MCP server
- reports reports (SSE / stdio)
- │
- ▼
- Workload analyzer
- · fingerprint grouping
- · resource aggregation
- · top-N DBA review queue
- │
- ▼
- DS experiment layer
- · risk classification
- · triage / prioritization
- · method comparison
-```
-
-LLMs are used as **interface and automation** (MCP, research pipelines) — never as the source of performance findings.
-
----
-## Data Science layer
+ClickAdvisor выполняет только `EXPLAIN ESTIMATE`, не запускает пользовательский
+запрос и не читает result data.
-ClickAdvisor includes a research layer that formalizes evaluation quality and prepares impact-based prioritization.
+## Data Science и ML
-**Dataset:** 20 235 ClickHouse SQL queries from ClickBench, TPC-H, TPC-DS, SSB, ClickHouse functional tests, and open-source projects. Each query carries:
-- `rule_findings` — deterministic rule engine output
-- `measured_metrics` — `read_rows`, `query_duration_ms`, `memory_usage` from local replay
-- `final_risk_label` — reconciled `low / medium / high` from both signals
-- `label_source` — `rule_only / measured_only / both`
+DS-часть нужна не для замены rule engine. Её задача — формализовать качество,
+сравнить подходы, найти ограничения и подготовить triage/prioritization layer.
-**Baseline results (holdout):**
+Expert dataset:
-| Model | Macro-F1 |
-|-------|----------|
-| Dummy (majority) | 0.278 |
-| TF-IDF + Logistic Regression | 0.882 |
-| Structured + Rule features + LR | 0.837 |
-| **Random Forest (all features)** | **0.949** |
-| CatBoost | 0.871 |
+| Показатель | Значение |
+|---|---:|
+| SQL records | 20 235 |
+| Real / synthetic | 19 090 / 1 145 |
+| Successful local replay records | 9 837 |
+| Final labels | low 4 253 / medium 14 285 / high 1 697 |
+| Numeric feature count | 115 |
+| Rule vocabulary | 54 |
-**Key finding:** RF F1=0.949 overall, but on `measured_only` subset (no rule signal) it drops to F1=0.595. This confirms that the rule engine is the production runtime — ML adds value as a triage and prioritization layer, not as a replacement.
+Ключевой факт по источнику меток:
-Full experiment report: [`docs/experiments/risk_labeling_ds_summary.md`](docs/experiments/risk_labeling_ds_summary.md)
+| Label source | Records | Интерпретация |
+|---|---:|---|
+| `rule_only` | 14 693 | модель в основном учит compressed rule-engine signal |
+| `measured_only` | 4 635 | независимый сигнал из latency/read/memory |
+| `both` | 907 | самый надёжный core, где static и measured signals согласны |
----
+Baseline ladder:
-## Rule catalog
+| Model | CV macro-F1 | Holdout macro-F1 |
+|---|---:|---:|
+| Dummy most frequent | 0.275 +/- 0.000 | 0.278 |
+| Dummy stratified | 0.328 +/- 0.009 | 0.335 |
+| TF-IDF + Logistic Regression | 0.864 +/- 0.011 | 0.882 |
+| Structural/rule LR | 0.827 +/- 0.004 | 0.837 |
+| Random Forest all features | 0.938 +/- 0.006 | 0.949 |
+| CatBoost tabular | 0.873 +/- 0.008 | 0.871 |
-ClickAdvisor covers six risk families:
+Holdout error analysis для Random Forest:
-| Family | Examples |
-|--------|---------|
-| MergeTree & pruning | `FINAL`, functions on PK columns, bad partition pruning, missing `PREWHERE` |
-| Aggregation & cardinality | `COUNT(DISTINCT)`, high-cardinality `GROUP BY`, memory-heavy aggregate states |
-| Joins & distributed | Casts around join keys, large right-side joins, distributed join overhead |
-| Schema design | `LowCardinality` candidates, `Nullable` overuse, oversized integer types |
-| Settings & environment | `max_threads` vs CPU, `max_memory_usage`, async insert settings |
-| Safety & operations | Mutations on large tables, `DELETE` without `WHERE`, `OPTIMIZE FINAL` misuse |
+| Slice | Records | Macro-F1 | High recall |
+|---|---:|---:|---:|
+| all_holdout | 3 039 | 0.949 | 0.887 |
+| rule_only | 2 235 | 0.970 | 0.990 |
+| measured_only | 672 | 0.595 | 0.785 |
+| both | 132 | 0.975 | 1.000 |
-Each rule has: rule card · positive/negative tests · benchmark case · tier · confidence.
+Вывод: ML полезен для triage, confidence grouping и review queue ordering, но
+production-рекомендации остаются rule-first.
-Browse the catalog: [`clickadvisor/rules/`](clickadvisor/rules/)
+Подробнее:
----
+- [docs/evaluation.md](docs/evaluation.md)
+- [docs/experiments/risk_labeling_ds_summary.md](docs/experiments/risk_labeling_ds_summary.md)
+- [data/ml/expert_dataset/eda/ds_report.md](data/ml/expert_dataset/eda/ds_report.md)
-## Security & local-first posture
+## Evaluation
-ClickAdvisor is designed for environments where SQL, DDL, and query logs are sensitive:
+| Evaluation surface | Data | Result |
+|---|---|---:|
+| Rule detection | 222 synthetic/schema/env cases | precision 1.000 / recall 1.000 / F1 1.000 |
+| Retrieval | 20 query -> docs pairs | best MRR@3 0.517 |
+| Risk-label DS | 20 235 SQL records | RF holdout macro-F1 0.949 |
+| Workload prototype | sample query_log CSV | normalized groups + top-N risk report |
-- **No external calls at runtime** — the rule engine is pure Python, offline by default.
-- **MCP server processes in memory** — no query storage, no logging of SQL content.
-- **Live mode** — reads `system.query_log` with a read-only user, no writes.
-- **query_log export** — literals can be redacted before export; the rule engine works on normalized fingerprints.
-- **LLM is optional** — agents use MCP to call ClickAdvisor; the LLM never generates trusted findings.
+Воспроизводимые проверки:
-For strict environments: run the Docker image on-premise with no outbound network access.
+```bash
+poetry run ruff check clickadvisor tests scripts
+poetry run mypy clickadvisor
+poetry run pytest --ignore=tests/integration -q
+poetry run python scripts/rules/validate_catalog.py
+poetry run python scripts/benchmark/validate_cases.py
+poetry run python scripts/eval/run_benchmark.py --cases-dir benchmark/cases/synthetic_expanded --mode strict
+```
-Full security doc: [`docs/security-local-first.md`](docs/security-local-first.md)
+## Security
----
+ClickAdvisor можно запускать внутри компании, CI/CD или локальной среды
+инженера без отправки SQL, DDL, EXPLAIN, environment context и query_log во
+внешние LLM/API.
-## Roadmap
+Что может читать ClickAdvisor:
-- [x] Single-query advisor (CLI + MCP)
-- [x] Workload advisor — CSV mode
-- [x] Live mode (`--connect`)
-- [x] DS risk classification layer
-- [ ] `EXPLAIN`-based before/after comparator
-- [ ] ClickHouse Cloud / Altinity integration
-- [ ] Impact ranking by `read_bytes × executions_per_day`
-- [ ] Workload diff between time windows (before/after deploy)
+- SQL text;
+- optional schema DDL;
+- optional EXPLAIN output;
+- optional environment JSON;
+- sanitized `system.query_log` CSV или read-only live metadata;
+- `SELECT version()` при version detection;
+- `EXPLAIN ESTIMATE` только при явном флаге.
----
+Что он не делает по умолчанию:
-## Development
+- не выполняет пользовательский SQL для speedup measurement;
+- не читает result data;
+- не выполняет `ANALYZE`;
+- не применяет DDL/mutations;
+- не делает hidden remote LLM calls.
-```bash
-# Run tests
-poetry run pytest --ignore=tests/integration -q
+Подробнее: [docs/security-local-first.md](docs/security-local-first.md).
-# Type check
-poetry run mypy clickadvisor
+## Критерии проекта
-# Lint
-poetry run ruff check clickadvisor tests scripts
+| Критерий | Как закрыт |
+|---|---|
+| Development | Poetry, Docker, GitHub Actions, типизированное ядро, linting, tests, validators, ClickHouse integration |
+| Data Science | EDA, feature extraction, group split, CV/holdout, baseline ladder, error analysis |
+| AI usage | MCP tools, локальный и remote-compatible agent workflow, DS/research automation narrative |
+| Product thinking | целевые пользователи, боль, competitors/SOTA framing, MVP surfaces, impact path через workload analysis |
-# Validate rule catalog
-poetry run python scripts/rules/validate_catalog.py
+Подробная самооценка: [docs/project-readiness.md](docs/project-readiness.md).
-# Run benchmark suite
-poetry run python scripts/eval/run_benchmark.py \
- --cases-dir benchmark/cases/synthetic_expanded \
- --mode strict
-```
+## Документация
----
+- [Architecture](docs/ARCHITECTURE.md)
+- [Evaluation](docs/evaluation.md)
+- [Workload Analyzer](docs/workload.md)
+- [MCP](docs/MCP.md)
+- [MCP Deployment](docs/mcp-deployment.md)
+- [Security / Local-First](docs/security-local-first.md)
+- [Rule Catalog](docs/rules/README.md)
## License
-MIT © 2024 olyannaa
+MIT
diff --git a/clickadvisor/cli/main.py b/clickadvisor/cli/main.py
index 4339d7f3..0438df89 100644
--- a/clickadvisor/cli/main.py
+++ b/clickadvisor/cli/main.py
@@ -58,6 +58,23 @@ def mcp_server() -> None:
run()
+@app.command()
+def mcp_http_server(
+ host: Annotated[str, typer.Option(help="Bind host")] = "127.0.0.1",
+ port: Annotated[int, typer.Option(help="Bind port")] = 8765,
+ path: Annotated[str, typer.Option(help="MCP endpoint path")] = "/mcp",
+) -> None:
+ """Запустить Streamable HTTP MCP сервер для remote-compatible demo."""
+ from clickadvisor.mcp_server.server import run_http
+
+ if host not in {"127.0.0.1", "localhost"}:
+ console.print(
+ "[yellow]Внимание: remote MCP endpoint должен быть защищён HTTPS/auth proxy. "
+ "Для локального demo безопаснее использовать 127.0.0.1.[/yellow]"
+ )
+ run_http(host=host, port=port, path=path)
+
+
@app.command()
def analyze(
sql: Annotated[Path, typer.Option(help="SQL файл")],
@@ -166,9 +183,9 @@ def workload(
typer.Option(help="ClickHouse HTTP URL for live mode, e.g. http://localhost:8123"),
] = None,
since: Annotated[
- int,
- typer.Option(help="Hours back to read from system.query_log (live mode)"),
- ] = 24,
+ str,
+ typer.Option(help="Live query_log window: 15m, 24h, 7d, 2w"),
+ ] = "24h",
ch_user: Annotated[str, typer.Option("--user", help="ClickHouse user (live mode)")] = "default",
ch_password: Annotated[
str, typer.Option("--password", help="ClickHouse password (live mode)")
@@ -184,7 +201,7 @@ def workload(
CSV: chadvisor workload --query-log query_log.csv
- Live: chadvisor workload --connect http://localhost:8123 --since 24
+ Live: chadvisor workload --connect http://localhost:8123 --since 24h
"""
from clickadvisor.workload.analyzer import (
analyze_query_log_csv,
@@ -212,7 +229,7 @@ def workload(
url=connect,
user=ch_user,
password=ch_password,
- since_hours=since,
+ since=since,
)
reader = ClickHouseLiveReader(reader_cfg)
@@ -220,7 +237,7 @@ def workload(
typer.echo(f"Error: cannot reach ClickHouse at {connect}", err=True)
raise typer.Exit(1)
- typer.echo(f"Connected to {connect}. Reading last {since}h from system.query_log...")
+ typer.echo(f"Connected to {connect}. Reading {since} from system.query_log...")
rows = reader.fetch()
if not rows:
diff --git a/clickadvisor/mcp_server/server.py b/clickadvisor/mcp_server/server.py
index 856bf715..ffeab862 100644
--- a/clickadvisor/mcp_server/server.py
+++ b/clickadvisor/mcp_server/server.py
@@ -6,6 +6,7 @@
from typing import Any, cast
from mcp.server import Server
+from mcp.server.fastmcp import FastMCP
from mcp.server.stdio import stdio_server
from mcp.types import GetPromptResult, Prompt, PromptArgument, PromptMessage, TextContent, Tool
@@ -343,6 +344,99 @@ def _get_applicable_rules(ch_version: str | None) -> list[Rule]:
return cast(list[Rule], get_applicable_rules(ch_version))
+def build_fastmcp_server(
+ *,
+ host: str = "127.0.0.1",
+ port: int = 8765,
+ path: str = "/mcp",
+) -> FastMCP:
+ mcp = FastMCP(
+ "clickadvisor",
+ host=host,
+ port=port,
+ streamable_http_path=path,
+ )
+
+ @mcp.tool(name="analyze_query")
+ async def analyze_query_http(
+ sql: str,
+ ch_version: str | None = None,
+ schema_ddl: str | None = None,
+ mode: str = "diagnose",
+ ) -> str:
+ result = await _analyze_query(
+ {
+ "sql": sql,
+ "ch_version": ch_version,
+ "schema_ddl": schema_ddl,
+ "mode": mode,
+ }
+ )
+ return result[0].text
+
+ @mcp.tool(name="analyze_query_json")
+ async def analyze_query_json_http(
+ sql: str,
+ ch_version: str | None = None,
+ schema_ddl: str | None = None,
+ ) -> str:
+ result = await _analyze_query_json(
+ {
+ "sql": sql,
+ "ch_version": ch_version,
+ "schema_ddl": schema_ddl,
+ }
+ )
+ return result[0].text
+
+ @mcp.tool(name="list_rules")
+ async def list_rules_http(tier: str = "all") -> str:
+ result = await _list_rules({"tier": tier})
+ return result[0].text
+
+ @mcp.tool(name="detect_ch_version")
+ async def detect_ch_version_http(
+ connect_url: str,
+ user: str = "default",
+ password: str = "",
+ ) -> str:
+ result = await _detect_ch_version(
+ {
+ "connect_url": connect_url,
+ "user": user,
+ "password": password,
+ }
+ )
+ return result[0].text
+
+ @mcp.prompt(name="analyze")
+ def analyze_prompt(sql: str, ch_version: str = "", mode: str = "diagnose") -> str:
+ version_hint = f" (ClickHouse {ch_version})" if ch_version else ""
+ text = (
+ f"Проанализируй этот ClickHouse SQL запрос{version_hint} "
+ f"через локальный ClickAdvisor MCP tool analyze_query:\n\n"
+ f"```sql\n{sql}\n```"
+ )
+ if mode == "explain":
+ text += "\n\nИспользуй mode='explain'."
+ return text
+
+ @mcp.prompt(name="explain")
+ def explain_prompt(sql: str, ch_version: str = "") -> str:
+ version_hint = f" (ClickHouse {ch_version})" if ch_version else ""
+ return (
+ f"Объясни почему этот ClickHouse запрос{version_hint} может быть медленным. "
+ f"Используй analyze_query с mode='explain':\n\n"
+ f"```sql\n{sql}\n```"
+ )
+
+ return mcp
+
+
+def run_http(*, host: str = "127.0.0.1", port: int = 8765, path: str = "/mcp") -> None:
+ build_fastmcp_server(host=host, port=port, path=path).run(transport="streamable-http")
+
+
async def main() -> None:
async with stdio_server() as (read_stream, write_stream):
await server.run(read_stream, write_stream, server.create_initialization_options())
diff --git a/clickadvisor/workload/analyzer.py b/clickadvisor/workload/analyzer.py
index 3586fc61..18698737 100644
--- a/clickadvisor/workload/analyzer.py
+++ b/clickadvisor/workload/analyzer.py
@@ -23,6 +23,8 @@
READ_BYTES_COLUMNS = ("read_bytes", "bytes_read", "bytes")
MEMORY_COLUMNS = ("memory_usage", "peak_memory_usage", "memory_usage_bytes")
SEVERITY_WEIGHT = {"high": 100.0, "medium": 35.0, "low": 10.0}
+SINCE_RE = re.compile(r"^(?P[1-9]\d*)(?Pm|h|d|w)$", re.IGNORECASE)
+SINCE_UNITS = {"m": "MINUTE", "h": "HOUR", "d": "DAY", "w": "WEEK"}
@dataclass(slots=True)
@@ -145,6 +147,31 @@ def analyze_query_log_csv(
)
+def clickhouse_query_log_sql(since: str) -> str:
+ interval_value, interval_unit = parse_since(since)
+ return f"""
+SELECT
+ event_time,
+ query,
+ query_duration_ms,
+ read_rows,
+ read_bytes,
+ memory_usage
+FROM system.query_log
+WHERE type = 'QueryFinish'
+ AND event_time >= now() - INTERVAL {interval_value} {interval_unit}
+ AND query != ''
+FORMAT CSVWithNames
+""".strip()
+
+
+def parse_since(since: str) -> tuple[int, str]:
+ match = SINCE_RE.match(since.strip())
+ if not match:
+ raise ValueError("since must look like 15m, 24h, 7d, or 2w")
+ return int(match.group("value")), SINCE_UNITS[match.group("unit").lower()]
+
+
def analyze_query_log_rows(
rows: list[dict[str, str]],
*,
diff --git a/clickadvisor/workload/live_reader.py b/clickadvisor/workload/live_reader.py
index 49fd1d24..e748eb78 100644
--- a/clickadvisor/workload/live_reader.py
+++ b/clickadvisor/workload/live_reader.py
@@ -1,23 +1,16 @@
-"""
-clickadvisor/workload/live_reader.py
-
-Reads query_log directly from a running ClickHouse instance via HTTP API.
-Produces the same row format as CSV import so analyzer.py stays unchanged.
-"""
-
from __future__ import annotations
import csv
import io
import logging
from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
import httpx
+from clickadvisor.workload.analyzer import parse_since
+
logger = logging.getLogger(__name__)
-# Columns we need — matches query_log_sample.csv schema
_QUERY = """
SELECT
query,
@@ -33,7 +26,7 @@
FROM system.query_log
WHERE
type = 'QueryFinish'
- AND event_time >= '{since}'
+ AND event_time >= now() - INTERVAL {since_value} {since_unit}
AND query_kind IN ('Select', 'AsyncInsertFlush')
AND query NOT ILIKE '%system.query_log%'
AND query NOT ILIKE '%system.processes%'
@@ -43,12 +36,12 @@
""".strip()
-@dataclass
+@dataclass(slots=True)
class LiveReaderConfig:
- url: str # e.g. http://localhost:8123
+ url: str
user: str = "default"
password: str = ""
- since_hours: int = 24
+ since: str = "24h"
limit: int = 10_000
timeout: float = 30.0
@@ -61,19 +54,18 @@ def __init__(self, config: LiveReaderConfig) -> None:
self._base_url = config.url.rstrip("/")
def fetch(self) -> list[dict[str, str]]:
- """
- Returns list of row-dicts with the same keys as query_log_sample.csv.
- Raises httpx.HTTPStatusError on ClickHouse errors.
- """
- since_dt = datetime.now(tz=UTC) - timedelta(hours=self.config.since_hours)
- since_str = since_dt.strftime("%Y-%m-%d %H:%M:%S")
-
- sql = _QUERY.format(since=since_str, limit=self.config.limit)
+ """Return row dicts with the same keys as query_log_sample.csv."""
+ since_value, since_unit = parse_since(self.config.since)
+ sql = _QUERY.format(
+ since_value=since_value,
+ since_unit=since_unit,
+ limit=self.config.limit,
+ )
logger.info(
"Fetching query_log from %s (since %s, limit %d)",
self._base_url,
- since_str,
+ self.config.since,
self.config.limit,
)
@@ -92,12 +84,15 @@ def fetch(self) -> list[dict[str, str]]:
logger.error("ClickHouse returned error:\n%s", response.text[:2000])
raise exc
- rows = list(csv.DictReader(io.StringIO(response.text)))
+ rows: list[dict[str, str]] = [
+ {str(key): str(value or "") for key, value in row.items()}
+ for row in csv.DictReader(io.StringIO(response.text))
+ ]
logger.info("Fetched %d rows from query_log", len(rows))
return rows
def check_connection(self) -> bool:
- """Quick ping — returns True if ClickHouse responds."""
+ """Return True if ClickHouse responds to /ping."""
try:
r = httpx.get(f"{self._base_url}/ping", timeout=5.0)
return r.status_code == 200 and r.text.strip() == "Ok."
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 3f2be268..803d4218 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -143,11 +143,13 @@ chadvisor analyze
`chadvisor index-kb` builds the Qdrant retrieval index from `data/kb/chunks/`.
It refuses to overwrite an existing index unless `--reindex` is supplied.
-`chadvisor workload` reads a sanitized `system.query_log` CSV export, groups
-queries by normalized fingerprint, runs representative SQL through the same rule
-engine, and renders a top-N workload risk report.
+`chadvisor workload` reads a sanitized `system.query_log` CSV export or recent
+query-log metadata through the ClickHouse HTTP API, groups queries by normalized
+fingerprint, runs representative SQL through the same rule engine, and renders a
+top-N workload risk report.
-`chadvisor mcp-server` starts the stdio MCP server.
+`chadvisor mcp-server` starts the stdio MCP server. `chadvisor mcp-http-server`
+starts a Streamable HTTP MCP endpoint for remote-compatible demos.
## MCP flow
diff --git a/docs/MCP.md b/docs/MCP.md
index b62431c6..08b2062b 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -15,6 +15,26 @@ from the deterministic ClickAdvisor rule engine.
poetry run chadvisor mcp-server
```
+This starts the local stdio MCP server.
+
+For Streamable HTTP MCP:
+
+```bash
+poetry run chadvisor mcp-http-server \
+ --host 127.0.0.1 \
+ --port 8765 \
+ --path /mcp
+```
+
+Endpoint:
+
+```text
+http://127.0.0.1:8765/mcp
+```
+
+For a remote demo, expose this endpoint only through HTTPS and authentication.
+See [MCP Deployment Options](mcp-deployment.md).
+
## Connect to Claude Desktop
Add this to `~/Library/Application Support/Claude/claude_desktop_config.json`:
@@ -118,6 +138,7 @@ that client. Treat SQL and query context as sensitive.
More details:
+- [MCP Deployment Options](mcp-deployment.md)
- [AI And MCP Workflow](ai-mcp-workflow.md)
- [Security And Local-First Runtime](security-local-first.md)
@@ -131,5 +152,7 @@ The server exposes MCP prompts for slash-command style workflows:
## Compatibility
-The server is tested with the Python `mcp` package over stdio transport and is
-compatible with MCP clients that support tools and prompts over stdio.
+The server is tested with the Python `mcp` package over stdio and Streamable
+HTTP transports. It is compatible with MCP clients that support tools and
+prompts over stdio, and with clients that can connect to a Streamable HTTP MCP
+endpoint such as `/mcp`.
diff --git a/docs/mcp-deployment.md b/docs/mcp-deployment.md
new file mode 100644
index 00000000..0a4ee2b8
--- /dev/null
+++ b/docs/mcp-deployment.md
@@ -0,0 +1,231 @@
+# MCP Deployment Options
+
+ClickAdvisor exposes MCP in two deployment profiles:
+
+- local stdio MCP for developer machines and IDE/desktop clients;
+- Streamable HTTP MCP for remote-compatible demos and shared environments.
+
+The trusted analysis path is the same in both profiles: deterministic
+ClickAdvisor tools produce the findings.
+
+## Option 1: Local MCP Server
+
+Best for daily use and enterprise-safe workflows.
+
+```bash
+poetry run chadvisor mcp-server
+```
+
+Example Claude Desktop config:
+
+```json
+{
+ "mcpServers": {
+ "clickadvisor": {
+ "command": "poetry",
+ "args": ["run", "chadvisor", "mcp-server"],
+ "cwd": "/path/to/clickadvisor"
+ }
+ }
+}
+```
+
+Use this when the SQL should stay on the user's laptop or inside the company
+network.
+
+## Option 2: Local HTTP Endpoint For Demo
+
+Best for testing Streamable HTTP clients on the same machine.
+
+```bash
+poetry run chadvisor mcp-http-server \
+ --host 127.0.0.1 \
+ --port 8765 \
+ --path /mcp
+```
+
+Endpoint:
+
+```text
+http://127.0.0.1:8765/mcp
+```
+
+If you need to expose the local endpoint through a temporary tunnel, bind the
+server to `0.0.0.0` so the tunneled Host header is accepted:
+
+```bash
+poetry run chadvisor mcp-http-server \
+ --host 0.0.0.0 \
+ --port 8765 \
+ --path /mcp
+```
+
+Then, in another terminal:
+
+```bash
+cloudflared tunnel --url http://127.0.0.1:8765 --no-autoupdate
+```
+
+Cloudflare prints a temporary URL:
+
+```text
+https://.trycloudflare.com/mcp
+```
+
+This is useful for quick checks, but it is not a stable production/demo URL.
+The link exists only while the local server and tunnel process are running.
+
+## Option 3: Remote Demo Endpoint
+
+Best for a defense/demo where experts should be able to connect without
+installing the repository.
+
+Recommended setup:
+
+```text
+MCP client
+ |
+ v
+HTTPS/auth proxy or tunnel
+ |
+ v
+chadvisor mcp-http-server on a controlled host
+```
+
+Reasonable places to host the demo endpoint:
+
+- a small VPS with HTTPS reverse proxy and an access token;
+- Railway/Render/Fly.io style Python service deployment;
+- Cloudflare Tunnel or similar tunnel in front of a local machine;
+- Tailscale/Funnel for a controlled private demo network.
+
+Do not expose a writable or unauthenticated MCP endpoint publicly. The official
+MCP transport guidance requires special care for remote HTTP servers: validate
+origins, bind local servers to localhost by default, and use proper
+authentication for remote connections.
+
+### Fast path: Railway
+
+Railway is the fastest path when the goal is "give experts a public URL during
+the defense".
+
+1. Push this repository to GitHub.
+2. Open Railway and create a new project from the GitHub repository.
+3. Railway reads `railway.json` and builds the root `Dockerfile`.
+4. In service variables, keep:
+
+```text
+PORT=8000
+MCP_PATH=/mcp
+```
+
+5. Open the service settings and generate a public domain.
+
+Result:
+
+```text
+https://.up.railway.app/mcp
+```
+
+This endpoint is suitable for demo SQL and read-only testing. If experts will
+send real company SQL, use local MCP instead.
+
+### Alternative: Render
+
+Render is a good option when you want a stable paid web service with a clear
+dashboard and HTTPS domain.
+
+1. Push this repository to GitHub.
+2. In Render, create a Blueprint from the repo, or create a Web Service
+ manually.
+3. If using the Blueprint, Render reads `render.yaml`.
+4. If creating manually, set:
+
+```text
+Environment: Docker
+Dockerfile Path: ./Dockerfile
+Docker Context: .
+PORT=8000
+MCP_PATH=/mcp
+```
+
+Result:
+
+```text
+https://.onrender.com/mcp
+```
+
+For a short defense demo, use a paid instance or warm the service before the
+presentation so the first connection is not delayed by cold start.
+
+### Alternative: Fly.io
+
+Fly.io is useful if you prefer CLI-based deploys and regional placement.
+
+Example `fly.toml` build section:
+
+```toml
+[build]
+ dockerfile = "Dockerfile"
+
+[env]
+ PORT = "8000"
+ MCP_PATH = "/mcp"
+```
+
+Then:
+
+```bash
+fly launch --no-deploy
+fly deploy
+```
+
+Result:
+
+```text
+https://.fly.dev/mcp
+```
+
+### What the Docker image contains
+
+The root `Dockerfile` is intentionally MCP-only and lightweight. It installs
+only the packages needed for deterministic SQL analysis and Streamable HTTP
+MCP:
+
+- `mcp`
+- `typer`
+- `rich`
+- `sqlglot`
+- `PyYAML`
+- `httpx`
+
+It does not install `sentence-transformers`, Torch, CatBoost, scikit-learn, or
+the local Qdrant retrieval stack. That keeps public demo deploys fast and
+predictable. Full local development and DS experiments still use:
+
+```bash
+poetry install
+```
+
+## What To Tell Experts During A Demo
+
+There are two safe test paths:
+
+1. Local install:
+
+```bash
+git clone https://github.com/olyannaa/clickadvisor.git
+cd clickadvisor
+poetry install
+poetry run chadvisor mcp-server
+```
+
+2. Remote demo endpoint:
+
+```text
+https://your-demo-domain.example/mcp
+```
+
+The remote endpoint should be used only with demo SQL or explicitly approved
+queries, because anything sent to the remote MCP endpoint leaves the expert's
+machine.
diff --git a/docs/project-readiness.en.md b/docs/project-readiness.en.md
new file mode 100644
index 00000000..e1e46152
--- /dev/null
+++ b/docs/project-readiness.en.md
@@ -0,0 +1,100 @@
+# Project Readiness Against Evaluation Criteria
+
+This document maps the current ClickAdvisor repository to the defense
+evaluation criteria. It is intentionally evidence-based: every claim points to
+code, tests, data, documentation, or reproducible artifacts.
+
+## Development
+
+Target level: 3.
+
+Evidence:
+
+- Python package managed with Poetry.
+- Typed core code checked with `mypy`.
+- Linting with `ruff`.
+- Unit tests, integration tests, validators, and benchmark checks.
+- GitHub Actions CI for lint, type-check, tests, and ClickHouse integration.
+- Docker image for Streamable HTTP MCP deployment.
+- Local ClickHouse replay environment through Docker Compose.
+- Dedicated modules for CLI, rules, retrieval, EXPLAIN, MCP, workload, and ML.
+
+Main commands:
+
+```bash
+poetry run ruff check clickadvisor tests scripts
+poetry run mypy clickadvisor
+poetry run pytest --ignore=tests/integration -q
+poetry run python scripts/rules/validate_catalog.py
+poetry run python scripts/benchmark/validate_cases.py
+```
+
+Assessment: strong level 3 engineering culture. The remaining production gap is
+release packaging for public distribution.
+
+## Data Science
+
+Target level: 3.
+
+Evidence:
+
+- Expert SQL dataset: 20 235 records.
+- Measured local replay metrics for 9 837 successful queries.
+- Rule labels, measured labels, final risk labels, and label-source analysis.
+- EDA for measured metrics and label distributions.
+- Feature extraction: normalized SQL, structural features, rule-derived
+ features, TF-IDF inside baselines.
+- Group-aware train/test/holdout split and 5-fold validation.
+- Baseline ladder: Dummy, TF-IDF + Logistic Regression, structured/rule LR,
+ Random Forest, CatBoost.
+- Holdout error analysis by `label_source`, especially `measured_only` and
+ `both`.
+
+Key result:
+
+- Random Forest all-features holdout macro-F1: 0.949.
+- Measured-only holdout macro-F1: 0.595.
+- Both-source holdout macro-F1: 0.975.
+
+Assessment: level 3 DS contour. The important methodological caveat is already
+documented: the model is a triage/prioritization layer, not a replacement for
+the deterministic rule engine.
+
+## AI Usage
+
+Target level: 3.
+
+Evidence:
+
+- MCP server exposes ClickAdvisor as a local tool for AI clients.
+- Local stdio MCP for desktop/IDE workflows.
+- Streamable HTTP MCP endpoint for remote-compatible demos.
+- AI/MCP workflow documentation explains how agents call deterministic tools
+ instead of inventing optimization advice.
+- Agent-assisted DS workflow is documented as part of research: data
+ preparation, validation, modeling, and review.
+
+Assessment: level 3 if presented as systematic AI-agent usage. The key point is
+that AI is an interface and development accelerator, not the trusted runtime
+source of recommendations.
+
+## Product Thinking
+
+Target level: high level 2 to level 3.
+
+Evidence:
+
+- Clear target users: DBA, data engineers, backend engineers, BI/dashboard
+ owners, platform teams.
+- Clear pain: ClickHouse performance incidents, CPU/RAM waste, high latency,
+ cloud cost, DBA review time.
+- Competitive framing: rule-based SQL antipatterns, DB observability,
+ Postgres plan advisors, learned optimizers, LLM/RAG SQL assistants.
+- MVP surfaces: CLI, JSON/Markdown reports, MCP, workload analyzer, local
+ retrieval, EXPLAIN ESTIMATE.
+- Business-impact path: workload-level top-N opportunities and future
+ impact-based ranking.
+
+Assessment: product thinking is strong for an MVP. The remaining gap for a full
+level 3 product case is external user/market feedback: interviews, expert
+evaluation, or real workload feedback from ClickHouse users.
diff --git a/docs/project-readiness.md b/docs/project-readiness.md
new file mode 100644
index 00000000..77a865e8
--- /dev/null
+++ b/docs/project-readiness.md
@@ -0,0 +1,99 @@
+# Готовность проекта по критериям оценки
+
+Этот документ сопоставляет текущее состояние ClickAdvisor с критериями защиты.
+Формулировки намеренно evidence-based: каждый тезис опирается на код, тесты,
+данные, документацию или воспроизводимые артефакты.
+
+## Разработка
+
+Целевой уровень: 3.
+
+Что уже есть:
+
+- Python package с управлением зависимостями через Poetry.
+- Типизированное ядро с проверкой `mypy`.
+- Линтинг через `ruff`.
+- Unit tests, integration tests, validators и benchmark checks.
+- GitHub Actions CI для lint/type-check/tests/ClickHouse integration.
+- Docker image для Streamable HTTP MCP deployment.
+- Локальное ClickHouse replay окружение через Docker Compose.
+- Отдельные модули для CLI, rules, retrieval, EXPLAIN, MCP, workload и ML.
+
+Основные проверки:
+
+```bash
+poetry run ruff check clickadvisor tests scripts
+poetry run mypy clickadvisor
+poetry run pytest --ignore=tests/integration -q
+poetry run python scripts/rules/validate_catalog.py
+poetry run python scripts/benchmark/validate_cases.py
+```
+
+Оценка: сильный уровень 3 по культуре разработки. Оставшийся production gap —
+релизная упаковка для публичного распространения.
+
+## Data Science
+
+Целевой уровень: 3.
+
+Что уже есть:
+
+- Expert SQL dataset: 20 235 записей.
+- Measured local replay metrics для 9 837 successful queries.
+- Rule labels, measured labels, final risk labels и label-source analysis.
+- EDA по measured metrics и распределениям меток.
+- Feature extraction: normalized SQL, structural features, rule-derived
+ features, TF-IDF внутри baselines.
+- Group-aware train/test/holdout split и 5-fold validation.
+- Baseline ladder: Dummy, TF-IDF + Logistic Regression, structured/rule LR,
+ Random Forest, CatBoost.
+- Holdout error analysis по `label_source`, особенно `measured_only` и `both`.
+
+Ключевой результат:
+
+- Random Forest all-features holdout macro-F1: 0.949.
+- Measured-only holdout macro-F1: 0.595.
+- Both-source holdout macro-F1: 0.975.
+
+Оценка: уровень 3 по DS-контуру. Важная методологическая оговорка уже
+зафиксирована: модель является triage/prioritization layer, а не заменой
+детерминированного rule engine.
+
+## Использование AI
+
+Целевой уровень: 3.
+
+Что уже есть:
+
+- MCP server открывает ClickAdvisor как локальный tool для AI-клиентов.
+- Local stdio MCP для desktop/IDE сценариев.
+- Streamable HTTP MCP endpoint для remote-compatible demos.
+- AI/MCP workflow documentation объясняет, как agents вызывают
+ deterministic tools вместо генерации неподтверждённых optimization advice.
+- Agent-assisted DS workflow описан как часть исследования: data preparation,
+ validation, modeling и review.
+
+Оценка: уровень 3, если на защите правильно подчеркнуть системное применение
+AI-агентов. Ключевой тезис: AI является интерфейсом и ускорителем разработки,
+но не trusted runtime source of recommendations.
+
+## Продуктовое мышление
+
+Целевой уровень: высокий 2 или 3.
+
+Что уже есть:
+
+- Понятные target users: DBA, data engineers, backend engineers,
+ BI/dashboard owners, platform teams.
+- Понятная боль: ClickHouse performance incidents, CPU/RAM waste, high latency,
+ cloud cost, DBA review time.
+- Competitive framing: rule-based SQL antipatterns, DB observability,
+ Postgres plan advisors, learned optimizers, LLM/RAG SQL assistants.
+- MVP surfaces: CLI, JSON/Markdown reports, MCP, workload analyzer, local
+ retrieval, EXPLAIN ESTIMATE.
+- Business-impact path: workload-level top-N opportunities и дальнейший
+ impact-based ranking.
+
+Оценка: продуктовая часть сильная для MVP. Оставшийся gap для полного уровня 3
+как продуктового кейса — внешний user/market feedback: интервью, экспертная
+оценка или feedback на реальном ClickHouse workload.
diff --git a/docs/workload.md b/docs/workload.md
index 35ed05b8..fc89bb40 100644
--- a/docs/workload.md
+++ b/docs/workload.md
@@ -35,7 +35,7 @@ event_time,query,query_duration_ms,read_rows,read_bytes,memory_usage
2026-07-01 10:00:00,"SELECT * FROM events FINAL WHERE message LIKE '%timeout%'",1200,900000,180000000,220000000
```
-## Run
+## Run From CSV
```bash
poetry run chadvisor workload \
@@ -61,6 +61,31 @@ poetry run chadvisor workload \
--output-format json
```
+## Run Live Against ClickHouse
+
+If ClickAdvisor can reach the ClickHouse HTTP API, it can read recent
+`system.query_log` metadata directly:
+
+```bash
+poetry run chadvisor workload \
+ --connect http://localhost:8123 \
+ --user default \
+ --password secret \
+ --since 24h \
+ --output-format markdown \
+ --top-n 10
+```
+
+Supported `--since` values:
+
+- `15m`
+- `24h`
+- `7d`
+- `2w`
+
+The live mode runs a read-only query against `system.query_log` and requests
+`FORMAT CSVWithNames`; the rest of the pipeline is the same as the CSV mode.
+
## Example Output
```text
@@ -113,6 +138,6 @@ WHERE type = 'QueryFinish'
FORMAT CSVWithNames
```
-The future production version should add live `--connect`, time windows,
-identifier hashing, stricter redaction modes, and ranking metrics such as
-Precision@K and NDCG@K.
+The future production version should add identifier hashing, stricter redaction
+modes, richer source filters, and ranking metrics such as Precision@K and
+NDCG@K.
diff --git a/railway.json b/railway.json
new file mode 100644
index 00000000..48c7fef7
--- /dev/null
+++ b/railway.json
@@ -0,0 +1,10 @@
+{
+ "$schema": "https://railway.com/railway.schema.json",
+ "build": {
+ "builder": "DOCKERFILE",
+ "dockerfilePath": "Dockerfile"
+ },
+ "deploy": {
+ "restartPolicyType": "ON_FAILURE"
+ }
+}
diff --git a/render.yaml b/render.yaml
new file mode 100644
index 00000000..d4cf4b50
--- /dev/null
+++ b/render.yaml
@@ -0,0 +1,13 @@
+services:
+ - type: web
+ name: clickadvisor-mcp
+ runtime: docker
+ plan: starter
+ dockerfilePath: ./Dockerfile
+ dockerContext: .
+ autoDeploy: false
+ envVars:
+ - key: PORT
+ value: 8000
+ - key: MCP_PATH
+ value: /mcp
diff --git a/requirements-mcp.txt b/requirements-mcp.txt
new file mode 100644
index 00000000..d2013abc
--- /dev/null
+++ b/requirements-mcp.txt
@@ -0,0 +1,6 @@
+httpx==0.27.2
+mcp>=1.0.0
+PyYAML==6.0.3
+rich==14.3.4
+sqlglot==26.20.0
+typer==0.16.1
diff --git a/scripts/ml/validate_expert_dataset.py b/scripts/ml/validate_expert_dataset.py
index 6b54fbb5..7e58e068 100644
--- a/scripts/ml/validate_expert_dataset.py
+++ b/scripts/ml/validate_expert_dataset.py
@@ -31,12 +31,21 @@ def main() -> None:
print(issue)
raise SystemExit(1)
risks = Counter(record["risk"]["label"] for record in records)
+ final_risks = Counter(str(record.get("final_risk_label")) for record in records)
+ label_sources = Counter(str(record.get("label_source")) for record in records)
+ reconciled_suffix = ""
+ if isinstance(manifest.get("risk_reconciliation"), dict):
+ reconciled_suffix = (
+ f", final_risk_distribution={dict(sorted(final_risks.items()))}, "
+ f"label_source_distribution={dict(sorted(label_sources.items()))}"
+ )
print(
"Validated expert dataset: "
f"{len(records)} records, "
f"{sum(1 for record in records if not record['is_synthetic'])} real, "
f"{sum(1 for record in records if record['is_synthetic'])} synthetic, "
f"risk distribution={dict(sorted(risks.items()))}"
+ f"{reconciled_suffix}"
)
diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py
index 37e5115d..17094b50 100644
--- a/tests/test_mcp_server.py
+++ b/tests/test_mcp_server.py
@@ -66,6 +66,13 @@ async def test_list_prompts() -> None:
assert "explain" in names
+def test_build_fastmcp_server() -> None:
+ from clickadvisor.mcp_server.server import build_fastmcp_server
+
+ app = build_fastmcp_server(host="127.0.0.1", port=8765, path="/mcp")
+ assert app is not None
+
+
@pytest.mark.asyncio
async def test_unknown_tool() -> None:
results = await call_tool("unknown_tool", {})
diff --git a/tests/workload/test_analyzer.py b/tests/workload/test_analyzer.py
index 82991936..ef41ae67 100644
--- a/tests/workload/test_analyzer.py
+++ b/tests/workload/test_analyzer.py
@@ -6,7 +6,12 @@
from typer.testing import CliRunner
from clickadvisor.cli.main import app
-from clickadvisor.workload.analyzer import analyze_query_log_csv, normalize_sql
+from clickadvisor.workload.analyzer import (
+ analyze_query_log_csv,
+ clickhouse_query_log_sql,
+ normalize_sql,
+ parse_since,
+)
def test_normalize_sql_replaces_literals() -> None:
@@ -72,6 +77,17 @@ def test_analyze_query_log_groups_and_prioritizes(tmp_path: Path) -> None:
assert top.priority_label in {"medium", "high"}
+def test_parse_since_and_build_live_query() -> None:
+ assert parse_since("24h") == (24, "HOUR")
+ assert parse_since("7d") == (7, "DAY")
+
+ query = clickhouse_query_log_sql("15m")
+
+ assert "FROM system.query_log" in query
+ assert "INTERVAL 15 MINUTE" in query
+ assert "FORMAT CSVWithNames" in query
+
+
def test_workload_cli_outputs_markdown(tmp_path: Path) -> None:
query_log = tmp_path / "query_log.csv"
query_log.write_text(