diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a35d56..9567023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,3 +66,26 @@ jobs: run: | ls -R pytest tests/ -v + + - name: Audit dependencies for known vulnerabilities + continue-on-error: true # surface CVEs without blocking merges initially + run: | + pip install pip-audit + pip-audit -r requirements.txt + + - name: Check for migration drift + # Non-blocking for now: the repo has pre-existing model/migration drift + # (user.otp_secret length, webhook_log FK ondelete=CASCADE). Once those are + # reconciled with a migration, drop continue-on-error to make this a gate. + continue-on-error: true + env: + FLASK_APP: hookwise:create_app + DATABASE_URL: postgresql://hookwise:hookwise_pass@localhost:5432/hookwise_test + SECRET_KEY: test-secret-key + GUI_PASSWORD: test-password + ENCRYPTION_KEY: vmJ34RDpkZk7-sUqAwq0lMA2QN0P0SEAEuC874kov5E= + REDIS_HOST: localhost + TESTING: "true" + run: | + flask db upgrade + flask db check diff --git a/.gitignore b/.gitignore index 371a606..59dd785 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ compare_reqs.py generate_final_reqs.py requirements_final.txt requirements_updated.txt +graphify-out/ + +.claudeignore diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md new file mode 100644 index 0000000..d011144 --- /dev/null +++ b/docs/RUNBOOK.md @@ -0,0 +1,109 @@ +# HookWise Operator Runbook + +Practical procedures for running HookWise (Flask web + Celery worker/beat + +Redis + PostgreSQL, bridging webhooks to ConnectWise Manage tickets). + +## Service map + +| Component | Role | Health signal | +|-----------|------|---------------| +| `hookwise-proxy` | Flask web / API / dashboard | `GET /health`, `GET /health/services` | +| `hookwise-worker` | Celery worker — processes webhooks, creates/updates/closes tickets | `celery` field in `/health/services` | +| `hookwise-beat` | Celery beat — cleanup, timeout checks, health verification | scheduled task freshness | +| `redis` | Broker + metrics + maintenance flag | `redis` field in `/health/services` | +| `postgres` | Config, webhook logs, audit trail | `database` field in `/health/services` | +| `hookwise-llm` (optional) | Ollama for RCA / AI routing | `GET /api/health/llm` | + +--- + +## Dead-letter queue (DLQ) + +A webhook lands in the DLQ (`WebhookLog.status == "dlq"`) after Celery retries +are exhausted. DLQ items are **terminal failures** — they will not retry on +their own. + +**Where to see it** +- Dashboard → *Ticket History* → **DLQ** stat card (today's count). +- Webhook History → filter **Status = DLQ** (`/history?status=dlq`). + +**Triage** +1. Open the DLQ-filtered history; use **View** to inspect the payload and the + `error_message`. +2. Common causes: invalid ConnectWise credentials, a board/status/priority that + no longer exists, mapping gaps, or a CW outage during the retry window. +3. Fix the root cause (rotate keys, correct the endpoint's board/status, add the + missing mapping), then **Replay** the item(s). Bulk-select to replay many. + +**Escalate** if the DLQ count keeps climbing after a replay — that means the +underlying cause is unresolved (usually CW auth or a board rename). + +--- + +## ConnectWise outage / elevated error rate + +Symptoms: failures and DLQ growth, `create_ticket` errors in worker logs, +`ConnectWiseError` in `error_message`. + +1. Confirm scope: is it auth (403/401 from CW) or availability (5xx/timeouts)? + Check worker logs and, if present, the last CW error in each endpoint's + Security/Health HUD. +2. If CW is down, enable **Maintenance Mode** (dashboard toggle, or + `POST /admin/maintenance`). Inbound webhooks then get `503` and are not + dropped by half-processing; sources that retry will re-deliver. +3. When CW recovers, disable Maintenance Mode and **replay** any DLQ items + accumulated during the outage. +4. If a stale board/priority cache is suspected, clear it: + `flask clear-cw-cache` (or the container's equivalent CLI entry). + +--- + +## Bearer token / secret rotation + +1. Dashboard → endpoint → **Rotate Bearer Token**. The old token stops working + immediately. +2. Update the token in the source system's webhook configuration. +3. Verify with **Test Webhook**; confirm a `processed` entry in history. +4. For HMAC, update `hmac_secret` on the endpoint and the signing secret in the + source in lockstep — a mismatch yields `401 Invalid HMAC Signature`. +5. Rotating `ENCRYPTION_KEY` (Fernet) re-keys stored secrets and is a planned + maintenance operation — never rotate it without a re-encryption plan, or + stored bearer tokens/HMAC secrets become undecryptable. + +--- + +## Worker / beat not processing + +1. `GET /health/services` — check the `celery` field (`up` / `warning` / `down`) + and `celery_active`. +2. If `down`: confirm the worker container is running and can reach Redis + (`redis` field). Restart the worker. +3. If webhooks are `queued` but never `processed`: the worker is not consuming — + check broker connectivity and worker logs. +4. Long-running tasks are bounded by `CELERY_TASK_SOFT_TIME_LIMIT` (default 120s) + and `CELERY_TASK_TIME_LIMIT` (default 300s); a task exceeding these is + killed and retried/dead-lettered rather than hanging a worker. + +--- + +## Configuration & limits (env) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `SECRET_KEY` | — (required in prod) | Flask session signing | +| `GUI_PASSWORD` | — (required) | admin login password | +| `ENCRYPTION_KEY` | — (required) | Fernet key for stored secrets | +| `SESSION_COOKIE_SECURE` | `true` | send session cookie only over HTTPS | +| `MAX_CONTENT_LENGTH_KB` | `1024` | reject inbound bodies above this size (413) | +| `CELERY_TASK_SOFT_TIME_LIMIT` | `120` | soft task limit (seconds) | +| `CELERY_TASK_TIME_LIMIT` | `300` | hard task limit (seconds) | +| `VIABILITY_TTL` | `300` | ticket-dedup viability window (seconds) | +| `LOG_RETENTION_DAYS` | `30` | history retention for the daily cleanup task | + +--- + +## Auditing + +Security-relevant events are written to `AuditLog` and viewable at `/audit`: +logins (success and **failed** attempts), 2FA enable/disable, logout, +maintenance toggles, and configuration changes. Review it after any suspected +unauthorized access or unexpected config change. diff --git a/hookwise/__init__.py b/hookwise/__init__.py index 4f30198..df7b647 100644 --- a/hookwise/__init__.py +++ b/hookwise/__init__.py @@ -48,6 +48,23 @@ def _configure_app(app: Flask) -> None: # still bound to the session secret, so CSRF protection is preserved. _csrf_ttl = os.environ.get("WTF_CSRF_TIME_LIMIT") app.config["WTF_CSRF_TIME_LIMIT"] = int(_csrf_ttl) if _csrf_ttl else None + + # Session cookie hardening. HttpOnly + SameSite=Lax block JS access and + # cross-site sends; Secure keeps the session cookie off plaintext HTTP. + # Secure defaults on in production but off under TESTING so the http test + # client still round-trips the session cookie. + _secure_default = "false" if os.environ.get("TESTING", "").lower() == "true" else "true" + app.config["SESSION_COOKIE_HTTPONLY"] = True + app.config["SESSION_COOKIE_SAMESITE"] = "Lax" + app.config["SESSION_COOKIE_SECURE"] = ( + os.environ.get("SESSION_COOKIE_SECURE", _secure_default).lower() == "true" + ) + + # Reject oversized bodies before they are buffered into memory (protects the + # ingestion worker from memory-exhaustion payloads). Configurable in KB. + _max_kb = os.environ.get("MAX_CONTENT_LENGTH_KB", "1024") + app.config["MAX_CONTENT_LENGTH"] = (int(_max_kb) if _max_kb.isdigit() else 1024) * 1024 + app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get( "DATABASE_URL", "postgresql://hookwise:hookwise_pass@postgres:5432/hookwise" ) @@ -194,6 +211,12 @@ def bad_request(e: Any) -> Any: def rate_limit_error(e: Any) -> Any: return render_template("429.html"), 429 + @app.errorhandler(413) + def payload_too_large(e: Any) -> Any: + if request.path.startswith("/w/") or request.path.startswith("/api/"): + return jsonify({"status": "error", "message": "Payload too large"}), 413 + return render_template("500.html"), 413 + def _register_commands(app: Flask) -> None: """Register Flask CLI commands.""" diff --git a/hookwise/api.py b/hookwise/api.py index db3f0b8..04fcff5 100644 --- a/hookwise/api.py +++ b/hookwise/api.py @@ -193,6 +193,8 @@ def history() -> Any: date_from = request.args.get("date_from", "") date_to = request.args.get("date_to", "") endpoint_id = request.args.get("endpoint_id", "") + status = request.args.get("status", "") + source_ip = request.args.get("source_ip", "") per_page = 25 query = WebhookLog.query @@ -213,6 +215,12 @@ def history() -> Any: if endpoint_id: query = query.filter(WebhookLog.config_id == endpoint_id) + if status: + query = query.filter(WebhookLog.status == status) + + if source_ip: + query = query.filter(WebhookLog.source_ip.ilike(f"%{source_ip}%")) + if date_from: from datetime import datetime @@ -241,6 +249,8 @@ def history() -> Any: date_from=date_from, date_to=date_to, endpoint_id=endpoint_id, + status=status, + source_ip=source_ip, all_configs=all_configs, debug_mode=debug_mode, cw_url=cw_url, @@ -587,8 +597,33 @@ def get_stats() -> Any: WebhookLog.query.join(WebhookConfig) .filter( WebhookConfig.is_draft.is_(False), - WebhookLog.status.in_(["failed", "dlq"]), + WebhookLog.status == "failed", + WebhookLog.created_at >= today_start, + ) + .count() + ) + + dlq_today = ( + WebhookLog.query.join(WebhookConfig) + .filter( + WebhookConfig.is_draft.is_(False), + WebhookLog.status == "dlq", + WebhookLog.created_at >= today_start, + ) + .count() + ) + + # "No action" = webhooks that were handled but resulted in no ticket + # change (skipped, or processed without a create/update/close action). + non_action_today = ( + WebhookLog.query.join(WebhookConfig) + .filter( + WebhookConfig.is_draft.is_(False), WebhookLog.created_at >= today_start, + db.or_( + WebhookLog.status == "skipped", + db.and_(WebhookLog.status == "processed", WebhookLog.action.is_(None)), + ), ) .count() ) @@ -621,6 +656,8 @@ def get_stats() -> Any: "updated_today": tickets_updated, "closed_today": tickets_closed, "failed_today": failed_attempts, + "dlq_today": dlq_today, + "non_action_today": non_action_today, "success_rate": round(success_rate, 1), "avg_processing_time": round(float(avg_proc), 2), } diff --git a/hookwise/auth.py b/hookwise/auth.py index f9b9f6b..b020865 100644 --- a/hookwise/auth.py +++ b/hookwise/auth.py @@ -47,6 +47,7 @@ def login() -> Any: log_audit("login_2fa", None, f"User {user.username} logged in with 2FA") return redirect(url_for("main.index")) + log_audit("login_2fa_failed", None, f"Failed 2FA attempt for pending user {pending_user_id}") flash("Invalid 2FA code", "danger") return render_template("login.html", step="2fa") @@ -71,6 +72,7 @@ def login() -> Any: log_audit("login", None, f"User {username} logged in") return redirect(url_for("main.index")) + log_audit("login_failed", None, f"Failed login attempt for username '{username}'") flash("Invalid username or password", "danger") # GET request - always reset pending state to ensure clean login flow diff --git a/hookwise/models.py b/hookwise/models.py index f911a14..91e6418 100644 --- a/hookwise/models.py +++ b/hookwise/models.py @@ -140,10 +140,16 @@ class WebhookLog(Base): ticket_id = db.Column(db.Integer) matched_rule = db.Column(db.Text) processing_time = db.Column(db.Float) # in seconds - source_ip = db.Column(db.String(50)) + source_ip = db.Column(db.String(50), index=True) retry_count = db.Column(db.Integer, default=0) created_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc), index=True) + # Composite index for the common history query: filter by endpoint + status, + # ordered by recency. Complements the single-column indexes above. + __table_args__ = ( + db.Index("ix_webhook_log_config_status_created", "config_id", "status", "created_at"), + ) + config = db.relationship( "WebhookConfig", backref=db.backref("logs", lazy=True, cascade="all, delete-orphan", passive_deletes=True) ) diff --git a/hookwise/tasks.py b/hookwise/tasks.py index d82107a..a2c721f 100644 --- a/hookwise/tasks.py +++ b/hookwise/tasks.py @@ -76,6 +76,28 @@ def make_celery(app_name: str) -> Celery: }, } +# Execution guards: a hung ConnectWise/LLM call must not pin a worker forever. +# The soft limit raises SoftTimeLimitExceeded (catchable for cleanup); the hard +# limit force-kills the task. Defaults are generous enough for slow LLM RCA runs +# and are overridable via env. Parsing is defensive (mirrors VIABILITY_TTL above): +# a malformed env value falls back to the default instead of crashing worker +# startup at import time, and the soft limit is kept strictly below the hard limit +# (Celery requires soft < hard for the soft limit to ever fire). +_raw_soft_limit = os.environ.get("CELERY_TASK_SOFT_TIME_LIMIT", "120") +_soft_time_limit = max(1, int(_raw_soft_limit)) if _raw_soft_limit.isdigit() else 120 +_raw_hard_limit = os.environ.get("CELERY_TASK_TIME_LIMIT", "300") +_hard_time_limit = max(1, int(_raw_hard_limit)) if _raw_hard_limit.isdigit() else 300 +if _soft_time_limit >= _hard_time_limit: + logger.warning( + "CELERY_TASK_SOFT_TIME_LIMIT (%s) must be less than CELERY_TASK_TIME_LIMIT (%s); " + "clamping the soft limit below the hard limit.", + _soft_time_limit, + _hard_time_limit, + ) + _soft_time_limit = _hard_time_limit - 1 +celery.conf.task_soft_time_limit = _soft_time_limit +celery.conf.task_time_limit = _hard_time_limit + _app = None diff --git a/migrations/versions/c4e5f6a7b8c9_add_history_filter_indexes.py b/migrations/versions/c4e5f6a7b8c9_add_history_filter_indexes.py new file mode 100644 index 0000000..474bd0d --- /dev/null +++ b/migrations/versions/c4e5f6a7b8c9_add_history_filter_indexes.py @@ -0,0 +1,31 @@ +"""Add history-filter indexes (source_ip + composite config/status/created_at) + +Revision ID: c4e5f6a7b8c9 +Revises: 3f8d9c123abc +Create Date: 2026-07-06 00:00:00.000000 + +Speeds up the Webhook History page's status / source-IP filters and the +common "endpoint + status, newest first" query. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4e5f6a7b8c9" +down_revision = "3f8d9c123abc" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index("ix_webhook_log_source_ip", "webhook_log", ["source_ip"]) + op.create_index( + "ix_webhook_log_config_status_created", + "webhook_log", + ["config_id", "status", "created_at"], + ) + + +def downgrade(): + op.drop_index("ix_webhook_log_config_status_created", table_name="webhook_log") + op.drop_index("ix_webhook_log_source_ip", table_name="webhook_log") diff --git a/templates/history.html b/templates/history.html index d1ed402..bc53377 100644 --- a/templates/history.html +++ b/templates/history.html @@ -35,6 +35,18 @@