Refactor: modular project with uv, Typer CLI, Streamlit, and Pydantic

.github/workflows/pr.yml

@@ -5,28 +5,63 @@ on:
     branches: [main]
 
 jobs:
-  lint:
-    name: Lint
+  ruff-lint:
+    name: Ruff Lint
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --dev
+      - name: Ruff check
+        run: uv run ruff check src/ tests/
 
+  ruff-format:
+    name: Ruff Format
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --dev
+      - name: Ruff format check
+        run: uv run ruff format --check src/ tests/
 
-      - name: Lint with ruff
-        run: pip install ruff && ruff check pipeline.py
+  mypy:
+    name: Mypy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --dev
+      - name: Mypy
+        run: uv run mypy src/
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --dev
+      - name: Run tests
+        run: uv run pytest
 
   secret-scanning:
     name: Scan for secrets
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Full history for comprehensive scan
-
+          fetch-depth: 0
       - name: Pull trufflehog Docker image
         run: docker pull ghcr.io/trufflesecurity/trufflehog:latest
-
       - name: Run trufflehog scan
         run: docker run --rm -v "$PWD:/repo" ghcr.io/trufflesecurity/trufflehog:latest git file:///repo --fail --no-update
 
@@ -35,7 +70,6 @@ jobs:
     runs-on: ubuntu-latest
     needs: [secret-scanning]
     if: failure()
-
     steps:
       - name: Send email alert
         uses: dawidd6/action-send-mail@v3
@@ -55,3 +89,21 @@ jobs:
             Link: ${{ github.event.pull_request.html_url }}
 
             Rotate any exposed credentials immediately, then close or fix the PR.
+
+  all-checks-pass:
+    name: All Checks Pass
+    runs-on: ubuntu-latest
+    if: always()
+    needs: [ruff-lint, ruff-format, mypy, test, secret-scanning]
+    steps:
+      - name: Verify all checks passed
+        run: |
+          if [[ "${{ needs.ruff-lint.result }}" != "success" ||
+                "${{ needs.ruff-format.result }}" != "success" ||
+                "${{ needs.mypy.result }}" != "success" ||
+                "${{ needs.test.result }}" != "success" ||
+                "${{ needs.secret-scanning.result }}" != "success" ]]; then
+            echo "One or more checks failed."
+            exit 1
+          fi
+          echo "All checks passed."

.github/workflows/update.yml

@@ -13,10 +13,10 @@ on:
         required: false
         default: ''
 
-  # Also run on pushes to main (for initial setup)
+  # Also run on pushes to main
   push:
     branches: [main]
-    paths: [pipeline.py]
+    paths: ['src/**', 'templates/**', 'pyproject.toml']
 
 permissions:
   contents: write
@@ -30,26 +30,27 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
 
       - name: Run pipeline
         run: |
           if [ -n "${{ github.event.inputs.years }}" ]; then
-            python pipeline.py ${{ github.event.inputs.years }}
+            uv run boston-needle-map run ${{ github.event.inputs.years }}
           else
-            python pipeline.py
+            uv run boston-needle-map run
           fi
 
       - name: Verify output
         run: |
           if [ ! -f docs/index.html ]; then
-            echo "❌ Pipeline did not produce docs/index.html"
+            echo "Pipeline did not produce docs/index.html"
             exit 1
           fi
-          echo "✅ docs/index.html exists ($(wc -c < docs/index.html) bytes)"
+          echo "docs/index.html exists ($(wc -c < docs/index.html) bytes)"
           ls -la docs/
 
       - name: Commit and push updated map

.gitignore

@@ -1 +1,10 @@
 __pycache__/
+*.pyc
+.venv/
+tmp/
+!tmp/.gitkeep
+dist/
+*.egg-info/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/

CLAUDE.md

@@ -0,0 +1,45 @@
+# Boston Needle Map - Project Guide
+
+## Quick Reference
+- **Package manager:** uv (not pip/poetry)
+- **CLI framework:** Typer
+- **Data validation:** Pydantic v2
+- **AI agents:** pydantic-ai (available, not yet actively used)
+- **Linting:** ruff + mypy (strict mode)
+- **Git hooks:** lefthook (runs ruff + mypy on pre-commit)
+- **Python version:** 3.12+
+- **UI exploration:** Streamlit (with folium + plotly)
+
+## Commands
+- `uv run boston-needle-map run` -- run the full pipeline (generates docs/index.html)
+- `uv run boston-needle-map run 2023 2024 2025` -- specific years
+- `uv run boston-needle-map run --no-cache` -- skip cache
+- `uv run boston-needle-map explore` -- launch Streamlit dashboard
+- `uv run boston-needle-map cache-clear` -- clear tmp/ cache
+- `uv run boston-needle-map serve` -- preview at localhost:8000
+- `uv run ruff check src/` -- lint
+- `uv run ruff format src/` -- format
+- `uv run mypy src/` -- type check
+- `uv run pytest` -- run tests
+
+## Architecture
+- `src/boston_needle_map/` -- main package
+  - `cli.py` -- Typer CLI entrypoint
+  - `config.py` -- constants (CKAN URLs, resource IDs, bounding box)
+  - `models.py` -- Pydantic models (CleanedRecord, DashboardStats, etc.)
+  - `fetcher.py` -- CKAN API data fetching
+  - `cleaner.py` -- raw record normalization and validation
+  - `analytics.py` -- stats computation (heatmap bins, neighborhoods, hourly)
+  - `renderer.py` -- HTML dashboard generation (for GitHub Pages)
+  - `cache.py` -- tmp/ directory caching for fetched data
+  - `app.py` -- Streamlit interactive dashboard
+- `templates/dashboard.html` -- HTML template with $PLACEHOLDER tokens
+- `docs/` -- generated output for GitHub Pages (do not edit manually)
+- `tmp/` -- cached API responses (gitignored)
+
+## Conventions
+- All data flows through Pydantic models (CleanedRecord, DashboardStats)
+- Never commit secrets or API keys
+- Generated files go to docs/; never edit docs/ manually
+- Cache files go to tmp/; this directory is gitignored
+- Use `python-dateutil` for datetime parsing