BRIC-13: Add classical RAG pathway with multi-query + LLM-as-judge

.env.example

@@ -35,6 +35,14 @@ BM25_ENABLED=true
 BM25_TEXT_CONFIG=english
 BM25_RRF_K=60
 
+# Classical RAG Configuration
+CLASSICAL_CHUNK_SIZE=1000
+CLASSICAL_CHUNK_OVERLAP=200
+CLASSICAL_NUM_QUERY_VARIATIONS=3
+CLASSICAL_RELEVANCE_THRESHOLD=5.0
+CLASSICAL_TABLE_PREFIX=classical_rag_
+CLASSICAL_LLM_TEMPERATURE=0.0
+
 # Server Configuration
 MCP_TRANSPORT=sse
 ALLOWED_ORIGINS=["*"]

Dockerfile.db

@@ -1,33 +1,25 @@
 FROM pgvector/pgvector:pg17
 
-# Install build dependencies
 USER root
 RUN apt-get update && apt-get install -y \
     build-essential \
     git \
     postgresql-server-dev-17 \
     flex \
     bison \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Apache AGE (v1.6.0 for PG17)
-RUN cd /tmp && \
+    && rm -rf /var/lib/apt/lists/* \
+    && cd /tmp && \
     git clone --branch PG17/v1.6.0-rc0 https://github.com/apache/age.git && \
     cd age && \
     make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config install || \
-    (echo "Failed to build AGE" && exit 1)
-
-# Install pg_textsearch extension for BM25 full-text search
-RUN cd /tmp && \
+    (echo "Failed to build AGE" && exit 1) \
+    && cd /tmp && \
     git clone https://github.com/timescale/pg_textsearch.git && \
     cd pg_textsearch && \
     make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config || \
     (echo "Failed to build pg_textsearch" && exit 1) && \
     make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config install || \
-    (echo "Failed to install pg_textsearch" && exit 1)
-
-# Cleanup build artifacts
-RUN rm -rf /tmp/age /tmp/pg_textsearch
+    (echo "Failed to install pg_textsearch" && exit 1) \
+    && rm -rf /tmp/age /tmp/pg_textsearch
 
-# Switch back to non-root user for security
 USER postgres

pyproject.toml

@@ -20,13 +20,16 @@ dependencies = [
     "mcp>=1.24.0",
     "minio>=7.2.18",
     "openai>=2.9.0",
-    "pgvector>=0.4.2",
+    "pgvector>=0.3.6,<0.4.0",
     "pydantic-settings>=2.12.0",
     "python-dotenv>=1.2.1",
     "python-multipart>=0.0.22",
     "raganything[all]>=1.2.10",
     "sqlalchemy[asyncio]>=2.0.0",
     "uvicorn>=0.38.0",
+    "langchain-postgres>=0.0.17",
+    "langchain-core>=1.2.28",
+    "langchain-openai>=1.1.12",
 ]
 
 [dependency-groups]
@@ -59,7 +62,7 @@ select = [
     "ARG",    # flake8-unused-arguments
     "SIM",    # flake8-simplify
 ]
-ignore = ["E501", "B008"]  # E501: line too long (handled by formatter), B008: FastAPI Depends/File in defaults
+ignore = ["E501", "B008", "ARG001", "ARG002"]
 
 [tool.ruff.lint.isort]
 known-first-party = ["src"]

src/application/api/classical_indexing_routes.py

@@ -0,0 +1,81 @@
+import asyncio
+import logging
+
+from fastapi import APIRouter, Depends, status
+
+from application.requests.classical_indexing_request import (
+    ClassicalIndexFileRequest,
+    ClassicalIndexFolderRequest,
+)
+from application.use_cases.classical_index_file_use_case import (
+    ClassicalIndexFileUseCase,
+)
+from application.use_cases.classical_index_folder_use_case import (
+    ClassicalIndexFolderUseCase,
+)
+from dependencies import (
+    get_classical_index_file_use_case,
+    get_classical_index_folder_use_case,
+)
+
+logger = logging.getLogger(__name__)
+
+classical_indexing_router = APIRouter(tags=["Classical Indexing"])
+
+_background_tasks: set[asyncio.Task] = set()
+
+
+async def _run_in_background(coro, label: str) -> None:
+    try:
+        await coro
+    except Exception:
+        logger.exception("Background %s failed", label)
+
+
+@classical_indexing_router.post(
+    "/classical/file/index", response_model=dict, status_code=status.HTTP_202_ACCEPTED
+)
+async def classical_index_file(
+    request: ClassicalIndexFileRequest,
+    use_case: ClassicalIndexFileUseCase = Depends(get_classical_index_file_use_case),
+):
+    task = asyncio.create_task(
+        _run_in_background(
+            use_case.execute(
+                file_name=request.file_name,
+                working_dir=request.working_dir,
+                chunk_size=request.chunk_size,
+                chunk_overlap=request.chunk_overlap,
+            ),
+            label=f"classical file indexing {request.file_name}",
+        )
+    )
+    _background_tasks.add(task)
+    task.add_done_callback(_background_tasks.discard)
+    return {"status": "accepted", "message": "File indexing started in background"}
+
+
+@classical_indexing_router.post(
+    "/classical/folder/index", response_model=dict, status_code=status.HTTP_202_ACCEPTED
+)
+async def classical_index_folder(
+    request: ClassicalIndexFolderRequest,
+    use_case: ClassicalIndexFolderUseCase = Depends(
+        get_classical_index_folder_use_case
+    ),
+):
+    task = asyncio.create_task(
+        _run_in_background(
+            use_case.execute(
+                working_dir=request.working_dir,
+                recursive=request.recursive,
+                file_extensions=request.file_extensions,
+                chunk_size=request.chunk_size,
+                chunk_overlap=request.chunk_overlap,
+            ),
+            label=f"classical folder indexing {request.working_dir}",
+        )
+    )
+    _background_tasks.add(task)
+    task.add_done_callback(_background_tasks.discard)
+    return {"status": "accepted", "message": "Folder indexing started in background"}

src/application/api/classical_query_routes.py

@@ -0,0 +1,25 @@
+from fastapi import APIRouter, Depends, status
+
+from application.requests.classical_query_request import ClassicalQueryRequest
+from application.responses.classical_query_response import ClassicalQueryResponse
+from application.use_cases.classical_query_use_case import ClassicalQueryUseCase
+from dependencies import get_classical_query_use_case
+
+classical_query_router = APIRouter(tags=["Classical Query"])
+
+
+@classical_query_router.post(
+    "/classical/query",
+    status_code=status.HTTP_200_OK,
+)
+async def classical_query(
+    request: ClassicalQueryRequest,
+    use_case: ClassicalQueryUseCase = Depends(get_classical_query_use_case),
+) -> ClassicalQueryResponse:
+    return await use_case.execute(
+        working_dir=request.working_dir,
+        query=request.query,
+        top_k=request.top_k,
+        num_variations=request.num_variations,
+        relevance_threshold=request.relevance_threshold,
+    )

src/application/api/file_routes.py

@@ -21,9 +21,26 @@
 MAX_UPLOAD_SIZE = 50 * 1024 * 1024
 
 ALLOWED_EXTENSIONS = {
-    ".pdf", ".txt", ".docx", ".xlsx", ".pptx", ".md", ".csv",
-    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp",
-    ".html", ".xml", ".json", ".rtf", ".odt", ".ods",
+    ".pdf",
+    ".txt",
+    ".docx",
+    ".xlsx",
+    ".pptx",
+    ".md",
+    ".csv",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".svg",
+    ".bmp",
+    ".html",
+    ".xml",
+    ".json",
+    ".rtf",
+    ".odt",
+    ".ods",
 }
 
 ALLOWED_MIME_PREFIXES = (
@@ -50,9 +67,7 @@ def _sanitize_filename(filename: str | None) -> str:
 def _validate_file_type(filename: str, content_type: str) -> None:
     ext = posixpath.splitext(filename)[1].lower()
     if ext not in ALLOWED_EXTENSIONS:
-        raise HTTPException(
-            status_code=422, detail=f"File type '{ext}' is not allowed"
-        )
+        raise HTTPException(status_code=422, detail=f"File type '{ext}' is not allowed")
     if not any(content_type.startswith(p) for p in ALLOWED_MIME_PREFIXES):
         raise HTTPException(
             status_code=422, detail=f"Content type '{content_type}' is not allowed"
@@ -145,9 +160,7 @@ async def upload_file(
     if normalized == ".":
         normalized = ""
     if normalized.startswith("..") or posixpath.isabs(normalized):
-        raise HTTPException(
-            status_code=422, detail="prefix must be a relative path"
-        )
+        raise HTTPException(status_code=422, detail="prefix must be a relative path")
     if prefix.endswith("/") and not normalized.endswith("/"):
         normalized += "/"
 

src/application/api/mcp_classical_tools.py

@@ -0,0 +1,58 @@
+from fastmcp import FastMCP
+
+from dependencies import (
+    get_classical_index_file_use_case,
+    get_classical_index_folder_use_case,
+    get_classical_query_use_case,
+)
+
+mcp_classical = FastMCP("RAGAnythingClassical")
+
+
+@mcp_classical.tool()
+async def classical_index_file(
+    file_name: str, working_dir: str, chunk_size: int = 1000, chunk_overlap: int = 200
+):
+    use_case = get_classical_index_file_use_case()
+    return await use_case.execute(
+        file_name=file_name,
+        working_dir=working_dir,
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )
+
+
+@mcp_classical.tool()
+async def classical_index_folder(
+    working_dir: str,
+    recursive: bool = True,
+    file_extensions: list[str] | None = None,
+    chunk_size: int = 1000,
+    chunk_overlap: int = 200,
+):
+    use_case = get_classical_index_folder_use_case()
+    return await use_case.execute(
+        working_dir=working_dir,
+        recursive=recursive,
+        file_extensions=file_extensions,
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )
+
+
+@mcp_classical.tool()
+async def classical_query(
+    working_dir: str,
+    query: str,
+    top_k: int = 10,
+    num_variations: int = 3,
+    relevance_threshold: float = 5.0,
+):
+    use_case = get_classical_query_use_case()
+    return await use_case.execute(
+        working_dir=working_dir,
+        query=query,
+        top_k=top_k,
+        num_variations=num_variations,
+        relevance_threshold=relevance_threshold,
+    )

src/application/api/mcp_file_tools.py

@@ -27,7 +27,9 @@ def _validate_prefix(prefix: str) -> str:
     if normalized == ".":
         normalized = ""
     if normalized.startswith("..") or posixpath.isabs(normalized):
-        raise ToolError(f"prefix must be a relative path within the bucket, got: {prefix!r}")
+        raise ToolError(
+            f"prefix must be a relative path within the bucket, got: {prefix!r}"
+        )
     if prefix.endswith("/") and not normalized.endswith("/"):
         normalized += "/"
     return normalized

src/application/requests/classical_indexing_request.py

@@ -0,0 +1,44 @@
+"""Request models for classical RAG indexing endpoints."""
+
+from typing import Annotated
+
+from pydantic import BaseModel, BeforeValidator, Field
+
+
+def _coerce_file_extensions(v: str | list[str] | None) -> list[str] | None:
+    if v is None:
+        return None
+    if isinstance(v, str):
+        return [ext.strip() for ext in v.split(",") if ext.strip()]
+    return v
+
+
+class ClassicalIndexFileRequest(BaseModel):
+    file_name: str = Field(..., description="Object path in the MinIO bucket")
+    working_dir: str = Field(
+        ..., description="RAG workspace directory (project isolation)"
+    )
+    chunk_size: int = Field(
+        default=1000, description="Max chars per chunk", ge=100, le=10000
+    )
+    chunk_overlap: int = Field(
+        default=200, description="Overlap between chunks", ge=0, le=2000
+    )
+
+
+class ClassicalIndexFolderRequest(BaseModel):
+    working_dir: str = Field(
+        ..., description="RAG workspace directory, also used as MinIO prefix"
+    )
+    recursive: bool = Field(
+        default=True, description="Process subdirectories recursively"
+    )
+    file_extensions: Annotated[
+        list[str] | None, BeforeValidator(_coerce_file_extensions)
+    ] = Field(default=None, description="Filter by extensions, e.g. ['.pdf', '.docx']")
+    chunk_size: int = Field(
+        default=1000, description="Max chars per chunk", ge=100, le=10000
+    )
+    chunk_overlap: int = Field(
+        default=200, description="Overlap between chunks", ge=0, le=2000
+    )

src/application/requests/classical_query_request.py

@@ -0,0 +1,28 @@
+"""Request model for classical RAG query endpoint."""
+
+from pydantic import BaseModel, Field
+
+
+class ClassicalQueryRequest(BaseModel):
+    working_dir: str = Field(
+        ..., description="RAG workspace directory for this project"
+    )
+    query: str = Field(..., description="The search query")
+    top_k: int = Field(
+        default=10,
+        description="Maximum number of chunks to retrieve per query variation",
+        ge=1,
+        le=100,
+    )
+    num_variations: int = Field(
+        default=3,
+        description="Number of query variations to generate (multi-query)",
+        ge=1,
+        le=10,
+    )
+    relevance_threshold: float = Field(
+        default=5.0,
+        description="Minimum relevance score (0-10) from LLM judge to include a chunk",
+        ge=0.0,
+        le=10.0,
+    )