From 33de7a19664863fb5bda48b233ba1c0e7fc5e3eb Mon Sep 17 00:00:00 2001 From: JosephMG Date: Mon, 13 Jul 2026 18:40:19 +0800 Subject: [PATCH 1/3] feat: add n8n metrics, alloy log pipeline, re-source DCGM exporter - Enable n8n Prometheus metrics (N8N_METRICS, queue metrics) on main + worker, scrape via new prometheus n8n job, visualize with Grafana dashboard 24474. - Add Grafana Alloy service to ship all container stdout/stderr to Loki. - Re-source nvidia/arm64 DCGM exporter to nvcr.io (multi-arch, avoids Docker Hub rate limits); amd continues using the ROCm exporter. Applied identically across nvidia/amd/arm64-compose-stack. --- deployments/amd-compose-stack/.env.example | 9 +- .../04-automation-n8n/docker-compose.yaml | 4 + .../alloy-config.alloy | 43 + .../docker-compose.yaml | 32 + .../grafana/provisioning/dashboards/n8n.json | 1415 +++++++++++++++++ .../prometheus/prometheus-amd.yml | 7 + deployments/arm64-compose-stack/.env.example | 9 +- .../04-automation-n8n/docker-compose.yaml | 4 + .../alloy-config.alloy | 43 + .../docker-compose.yaml | 34 +- .../grafana/provisioning/dashboards/n8n.json | 1415 +++++++++++++++++ .../prometheus/prometheus.yml | 7 + deployments/nvidia-compose-stack/.env.example | 9 +- .../04-automation-n8n/docker-compose.yaml | 4 + .../alloy-config.alloy | 43 + .../docker-compose.yaml | 34 +- .../grafana/provisioning/dashboards/n8n.json | 1415 +++++++++++++++++ .../prometheus/prometheus.yml | 7 + 18 files changed, 4529 insertions(+), 5 deletions(-) create mode 100644 deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy create mode 100644 deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json create mode 100644 deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy create mode 100644 deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json create mode 100644 deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy create mode 100644 deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json diff --git a/deployments/amd-compose-stack/.env.example b/deployments/amd-compose-stack/.env.example index 08d4597..3bff9cb 100644 --- a/deployments/amd-compose-stack/.env.example +++ b/deployments/amd-compose-stack/.env.example @@ -72,6 +72,10 @@ WORKER_CONCURRENCY=5 N8N_URL= N8N_CORS_ALLOW_ORIGIN=* +# n8n Metrics — free Prometheus endpoint; both main + worker expose /metrics +N8N_METRICS=true +N8N_METRICS_INCLUDE_QUEUE_METRICS=true + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) @@ -93,6 +97,9 @@ GATEWAY_API_URL=http://system-api-bridge:8000 GRAFANA_PORT=3000 GRAFANA_PASS=CHANGE_ME +# Grafana Alloy (collects container logs -> Loki) +ALLOY_IMAGE=grafana/alloy:v1.17.1 + # --- 12) commercial gateway --- SYSTEM_API_GATEWAY_PORT=5055 @@ -120,5 +127,5 @@ PROMETHEUS_IMAGE=prom/prometheus:latest GRAFANA_IMAGE=grafana/grafana:latest LOKI_IMAGE=grafana/loki:latest CADVISOR_IMAGE=gcr.io/cadvisor/cadvisor:latest -DCGM_EXPORTER_IMAGE=nvidia/dcgm-exporter:latest +DCGM_EXPORTER_IMAGE=nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04 ROCM_EXPORTER_IMAGE=rocm/device-metrics-exporter:v1.4.2 diff --git a/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml index cd9388c..a115600 100644 --- a/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml @@ -42,6 +42,8 @@ services: - OFFLOAD_MANUAL_EXECUTIONS_TO_WORKERS=true - N8N_EDITOR_BASE_URL=${N8N_URL:-http://host.docker.internal:5678} - WEBHOOK_URL=${N8N_URL:-http://host.docker.internal:5678} + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" logging: @@ -84,6 +86,8 @@ services: - N8N_SECURE_COOKIE=false - N8N_WORKER_MAX_CONCURRENT_EXECUTIONS=${WORKER_CONCURRENCY:-5} - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy new file mode 100644 index 0000000..38ca9d5 --- /dev/null +++ b/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy @@ -0,0 +1,43 @@ +// Grafana Alloy — collect every Docker container's stdout/stderr and ship to Loki. +// Containers are discovered through the Docker socket, tailed live, and forwarded +// to the Loki service on the shared ai_stack_net network. + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // Strip the leading "/" from the Docker container name -> "container" label. + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + // Promote the compose service name -> "compose_service" label. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + + // Static "job" label so every stream is queryable as {job="docker"}. + rule { + target_label = "job" + replacement = "docker" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.containers.targets + relabel_rules = discovery.relabel.containers.rules + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/deployments/amd-compose-stack/10-observability-grafana/docker-compose.yaml b/deployments/amd-compose-stack/10-observability-grafana/docker-compose.yaml index 813fabc..0eae2f5 100644 --- a/deployments/amd-compose-stack/10-observability-grafana/docker-compose.yaml +++ b/deployments/amd-compose-stack/10-observability-grafana/docker-compose.yaml @@ -166,6 +166,37 @@ services: max-size: "50m" max-file: "3" + alloy: + image: ${ALLOY_IMAGE:-grafana/alloy:v1.17.1} + container_name: alloy + restart: always + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/12345"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy-config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - alloy_data:/var/lib/alloy/data + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - ai_stack_net + depends_on: + - loki + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + networks: ai_stack_net: external: true @@ -173,3 +204,4 @@ networks: volumes: prometheus_data: grafana_data: + alloy_data: diff --git a/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json new file mode 100644 index 0000000..e1a4a85 --- /dev/null +++ b/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -0,0 +1,1415 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.3.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitor your n8n workflow automation platform with this comprehensive Node.js runtime and application health dashboard. Tracks CPU, memory, heap usage, garbage collection, event loop performance, and cluster status using n8n's native Prometheus metrics. \r\n\r\nhttps://github.com/nluecke/grafana-n8n-dashboards/tree/main/dashboards/prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Process & Instance Information", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Current n8n version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "n8n version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Node.js runtime version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_nodejs_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Node.js version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows uptime of the n8n instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "time() - n8n_process_start_time_seconds", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Instance uptime", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Instance role in cluster - 1 = Leader, 0 = Follower", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-blue", + "index": 1, + "text": "Follower" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_instance_role_leader", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Leader status", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_active_workflow_count", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active workflows", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 7, + "panels": [], + "title": "CPU & Event Loop Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows CPU usage breakdown", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_seconds_total[$__rate_interval]) * 100", + "instant": false, + "legendFormat": "Total CPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_user_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "User CPU", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_system_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "System CPU", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Event loop lag", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p99_seconds", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p90_seconds", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p50_seconds", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C" + } + ], + "title": "Event loop latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 11, + "panels": [], + "title": "Memory & Heap Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Resident memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1200000000 + }, + { + "color": "red", + "value": 1800000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_resident_memory_bytes", + "instant": false, + "legendFormat": "RSS", + "range": true, + "refId": "A" + } + ], + "title": "Memory (RSS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "JavaScript heap memory - Node.js automatically manages heap size. High percentage (80-95%) is NORMAL and expected. Watch for heap total growing over time, not the percentage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 8, + "y": 15 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_used_bytes", + "instant": false, + "legendFormat": "Used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_total_bytes", + "hide": false, + "instant": false, + "legendFormat": "Total (dynamic)", + "range": true, + "refId": "B" + } + ], + "title": "Heap memory (dynamic allocation)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Heap usage percentage shows how full the CURRENT heap allocation is. 80-95% is NORMAL for Node.js! This is NOT an indicator of memory pressure. Only worry if heap total keeps growing OR if you see excessive GC activity.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 15 + }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "(n8n_nodejs_heap_size_used_bytes / n8n_nodejs_heap_size_total_bytes) * 100", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Heap % ", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 49, + "panels": [], + "title": "GC & Runtime Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Gargabe Collector duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"minor\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Minor GC", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"major\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Major GC", + "range": true, + "refId": "B" + } + ], + "title": "GC duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "File descriptors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_open_fds", + "instant": false, + "legendFormat": "Open FDs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_max_fds", + "hide": false, + "instant": false, + "legendFormat": "Max FDs", + "range": true, + "refId": "B" + } + ], + "title": "File descriptors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [ + "n8n", + "nodejs", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "n8n System Health Overview", + "uid": "adv6qmb", + "version": 14, + "weekStart": "", + "id": null, + "gnetId": 24474 +} \ No newline at end of file diff --git a/deployments/amd-compose-stack/10-observability-grafana/prometheus/prometheus-amd.yml b/deployments/amd-compose-stack/10-observability-grafana/prometheus/prometheus-amd.yml index 15637ed..c7c2c51 100644 --- a/deployments/amd-compose-stack/10-observability-grafana/prometheus/prometheus-amd.yml +++ b/deployments/amd-compose-stack/10-observability-grafana/prometheus/prometheus-amd.yml @@ -32,3 +32,10 @@ scrape_configs: - job_name: 'docker-containers' static_configs: - targets: [] + + # n8n metrics (main: process + queue metrics) + # n8n-worker is started via --scale with no fixed container_name / stable DNS, so a static scrape + # target is unreliable; it is intentionally excluded (main already carries the queue metrics). + - job_name: 'n8n' + static_configs: + - targets: ['n8n-main:5678'] diff --git a/deployments/arm64-compose-stack/.env.example b/deployments/arm64-compose-stack/.env.example index 163b545..ebea9c1 100644 --- a/deployments/arm64-compose-stack/.env.example +++ b/deployments/arm64-compose-stack/.env.example @@ -58,6 +58,10 @@ WORKER_CONCURRENCY=5 N8N_URL= N8N_CORS_ALLOW_ORIGIN=* +# n8n Metrics — free Prometheus endpoint; both main + worker expose /metrics +N8N_METRICS=true +N8N_METRICS_INCLUDE_QUEUE_METRICS=true + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) @@ -77,6 +81,9 @@ GATEWAY_API_URL=http://system-api-bridge:8000 GRAFANA_PORT=3000 GRAFANA_PASS=CHANGE_ME +# Grafana Alloy (collects container logs -> Loki) +ALLOY_IMAGE=grafana/alloy:v1.17.1 + # --- 12) commercial gateway --- SYSTEM_API_GATEWAY_PORT=5055 @@ -97,5 +104,5 @@ PROMETHEUS_IMAGE=prom/prometheus:latest GRAFANA_IMAGE=grafana/grafana:latest LOKI_IMAGE=grafana/loki:latest CADVISOR_IMAGE=gcr.io/cadvisor/cadvisor:latest -DCGM_EXPORTER_IMAGE=nvidia/dcgm-exporter:latest +DCGM_EXPORTER_IMAGE=nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04 diff --git a/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml index 678f32c..b3bc98c 100644 --- a/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml @@ -42,6 +42,8 @@ services: - OFFLOAD_MANUAL_EXECUTIONS_TO_WORKERS=true - N8N_EDITOR_BASE_URL=${N8N_URL:-http://localhost:5678} - WEBHOOK_URL=${N8N_URL:-http://localhost:5678} + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" logging: @@ -84,6 +86,8 @@ services: - N8N_SECURE_COOKIE=false - N8N_WORKER_MAX_CONCURRENT_EXECUTIONS=${WORKER_CONCURRENCY:-5} - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy new file mode 100644 index 0000000..38ca9d5 --- /dev/null +++ b/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy @@ -0,0 +1,43 @@ +// Grafana Alloy — collect every Docker container's stdout/stderr and ship to Loki. +// Containers are discovered through the Docker socket, tailed live, and forwarded +// to the Loki service on the shared ai_stack_net network. + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // Strip the leading "/" from the Docker container name -> "container" label. + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + // Promote the compose service name -> "compose_service" label. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + + // Static "job" label so every stream is queryable as {job="docker"}. + rule { + target_label = "job" + replacement = "docker" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.containers.targets + relabel_rules = discovery.relabel.containers.rules + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/deployments/arm64-compose-stack/10-observability-grafana/docker-compose.yaml b/deployments/arm64-compose-stack/10-observability-grafana/docker-compose.yaml index 531341e..60f8540 100644 --- a/deployments/arm64-compose-stack/10-observability-grafana/docker-compose.yaml +++ b/deployments/arm64-compose-stack/10-observability-grafana/docker-compose.yaml @@ -28,7 +28,7 @@ services: max-file: "3" gpu-exporter: - image: ${DCGM_EXPORTER_IMAGE:-nvidia/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04} + image: ${DCGM_EXPORTER_IMAGE:-nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04} container_name: gpu-exporter restart: always healthcheck: @@ -168,6 +168,37 @@ services: max-size: "50m" max-file: "3" + alloy: + image: ${ALLOY_IMAGE:-grafana/alloy:v1.17.1} + container_name: alloy + restart: always + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/12345"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy-config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - alloy_data:/var/lib/alloy/data + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - ai_stack_net + depends_on: + - loki + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + networks: ai_stack_net: external: true @@ -175,3 +206,4 @@ networks: volumes: prometheus_data: grafana_data: + alloy_data: diff --git a/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json new file mode 100644 index 0000000..e1a4a85 --- /dev/null +++ b/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -0,0 +1,1415 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.3.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitor your n8n workflow automation platform with this comprehensive Node.js runtime and application health dashboard. Tracks CPU, memory, heap usage, garbage collection, event loop performance, and cluster status using n8n's native Prometheus metrics. \r\n\r\nhttps://github.com/nluecke/grafana-n8n-dashboards/tree/main/dashboards/prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Process & Instance Information", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Current n8n version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "n8n version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Node.js runtime version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_nodejs_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Node.js version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows uptime of the n8n instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "time() - n8n_process_start_time_seconds", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Instance uptime", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Instance role in cluster - 1 = Leader, 0 = Follower", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-blue", + "index": 1, + "text": "Follower" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_instance_role_leader", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Leader status", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_active_workflow_count", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active workflows", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 7, + "panels": [], + "title": "CPU & Event Loop Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows CPU usage breakdown", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_seconds_total[$__rate_interval]) * 100", + "instant": false, + "legendFormat": "Total CPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_user_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "User CPU", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_system_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "System CPU", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Event loop lag", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p99_seconds", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p90_seconds", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p50_seconds", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C" + } + ], + "title": "Event loop latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 11, + "panels": [], + "title": "Memory & Heap Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Resident memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1200000000 + }, + { + "color": "red", + "value": 1800000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_resident_memory_bytes", + "instant": false, + "legendFormat": "RSS", + "range": true, + "refId": "A" + } + ], + "title": "Memory (RSS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "JavaScript heap memory - Node.js automatically manages heap size. High percentage (80-95%) is NORMAL and expected. Watch for heap total growing over time, not the percentage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 8, + "y": 15 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_used_bytes", + "instant": false, + "legendFormat": "Used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_total_bytes", + "hide": false, + "instant": false, + "legendFormat": "Total (dynamic)", + "range": true, + "refId": "B" + } + ], + "title": "Heap memory (dynamic allocation)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Heap usage percentage shows how full the CURRENT heap allocation is. 80-95% is NORMAL for Node.js! This is NOT an indicator of memory pressure. Only worry if heap total keeps growing OR if you see excessive GC activity.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 15 + }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "(n8n_nodejs_heap_size_used_bytes / n8n_nodejs_heap_size_total_bytes) * 100", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Heap % ", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 49, + "panels": [], + "title": "GC & Runtime Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Gargabe Collector duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"minor\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Minor GC", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"major\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Major GC", + "range": true, + "refId": "B" + } + ], + "title": "GC duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "File descriptors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_open_fds", + "instant": false, + "legendFormat": "Open FDs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_max_fds", + "hide": false, + "instant": false, + "legendFormat": "Max FDs", + "range": true, + "refId": "B" + } + ], + "title": "File descriptors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [ + "n8n", + "nodejs", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "n8n System Health Overview", + "uid": "adv6qmb", + "version": 14, + "weekStart": "", + "id": null, + "gnetId": 24474 +} \ No newline at end of file diff --git a/deployments/arm64-compose-stack/10-observability-grafana/prometheus/prometheus.yml b/deployments/arm64-compose-stack/10-observability-grafana/prometheus/prometheus.yml index 63337d1..73483b3 100644 --- a/deployments/arm64-compose-stack/10-observability-grafana/prometheus/prometheus.yml +++ b/deployments/arm64-compose-stack/10-observability-grafana/prometheus/prometheus.yml @@ -27,3 +27,10 @@ scrape_configs: - job_name: 'docker-containers' static_configs: - targets: [] + + # n8n metrics (main: process + queue metrics) + # n8n-worker is started via --scale with no fixed container_name / stable DNS, so a static scrape + # target is unreliable; it is intentionally excluded (main already carries the queue metrics). + - job_name: 'n8n' + static_configs: + - targets: ['n8n-main:5678'] diff --git a/deployments/nvidia-compose-stack/.env.example b/deployments/nvidia-compose-stack/.env.example index a4fa084..89d571c 100644 --- a/deployments/nvidia-compose-stack/.env.example +++ b/deployments/nvidia-compose-stack/.env.example @@ -57,6 +57,10 @@ WORKER_CONCURRENCY=5 N8N_URL= N8N_CORS_ALLOW_ORIGIN=* +# n8n Metrics — free Prometheus endpoint; both main + worker expose /metrics +N8N_METRICS=true +N8N_METRICS_INCLUDE_QUEUE_METRICS=true + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) @@ -76,6 +80,9 @@ GATEWAY_API_URL=http://system-api-bridge:8000 GRAFANA_PORT=3000 GRAFANA_PASS=CHANGE_ME +# Grafana Alloy (collects container logs -> Loki) +ALLOY_IMAGE=grafana/alloy:v1.17.1 + # --- 12) commercial gateway --- SYSTEM_API_GATEWAY_PORT=5055 @@ -96,6 +103,6 @@ PROMETHEUS_IMAGE=prom/prometheus:latest GRAFANA_IMAGE=grafana/grafana:latest LOKI_IMAGE=grafana/loki:latest CADVISOR_IMAGE=gcr.io/cadvisor/cadvisor:latest -DCGM_EXPORTER_IMAGE=nvidia/dcgm-exporter:latest +DCGM_EXPORTER_IMAGE=nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04 diff --git a/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml index 7675248..dcb8151 100644 --- a/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml @@ -42,6 +42,8 @@ services: - N8N_CORS_ALLOW_ORIGIN=${N8N_CORS_ALLOW_ORIGIN:-*} - N8N_DIAGNOSTICS_ENABLED=false - N8N_SECURE_COOKIE=false + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" # Note: Redis dependency is external (from 03 stack) @@ -86,6 +88,8 @@ services: - N8N_SECURE_COOKIE=false - N8N_WORKER_MAX_CONCURRENT_EXECUTIONS=${WORKER_CONCURRENCY:-5} - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_METRICS=${N8N_METRICS:-true} + - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy new file mode 100644 index 0000000..38ca9d5 --- /dev/null +++ b/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy @@ -0,0 +1,43 @@ +// Grafana Alloy — collect every Docker container's stdout/stderr and ship to Loki. +// Containers are discovered through the Docker socket, tailed live, and forwarded +// to the Loki service on the shared ai_stack_net network. + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // Strip the leading "/" from the Docker container name -> "container" label. + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + // Promote the compose service name -> "compose_service" label. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + + // Static "job" label so every stream is queryable as {job="docker"}. + rule { + target_label = "job" + replacement = "docker" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.containers.targets + relabel_rules = discovery.relabel.containers.rules + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/docker-compose.yaml b/deployments/nvidia-compose-stack/10-observability-grafana/docker-compose.yaml index 2fa0799..63a5937 100644 --- a/deployments/nvidia-compose-stack/10-observability-grafana/docker-compose.yaml +++ b/deployments/nvidia-compose-stack/10-observability-grafana/docker-compose.yaml @@ -28,7 +28,7 @@ services: max-file: "3" gpu-exporter: - image: ${DCGM_EXPORTER_IMAGE:-nvidia/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04} + image: ${DCGM_EXPORTER_IMAGE:-nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04} container_name: gpu-exporter restart: always healthcheck: @@ -166,6 +166,37 @@ services: max-size: "50m" max-file: "3" + alloy: + image: ${ALLOY_IMAGE:-grafana/alloy:v1.17.1} + container_name: alloy + restart: always + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/12345"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy-config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - alloy_data:/var/lib/alloy/data + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - ai_stack_net + depends_on: + - loki + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + networks: ai_stack_net: external: true @@ -173,3 +204,4 @@ networks: volumes: prometheus_data: grafana_data: + alloy_data: diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json new file mode 100644 index 0000000..e1a4a85 --- /dev/null +++ b/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -0,0 +1,1415 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.3.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitor your n8n workflow automation platform with this comprehensive Node.js runtime and application health dashboard. Tracks CPU, memory, heap usage, garbage collection, event loop performance, and cluster status using n8n's native Prometheus metrics. \r\n\r\nhttps://github.com/nluecke/grafana-n8n-dashboards/tree/main/dashboards/prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Process & Instance Information", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Current n8n version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "n8n version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Node.js runtime version", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "n8n_nodejs_version_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Node.js version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows uptime of the n8n instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "time() - n8n_process_start_time_seconds", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Instance uptime", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Instance role in cluster - 1 = Leader, 0 = Follower", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-blue", + "index": 1, + "text": "Follower" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_instance_role_leader", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Leader status", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_active_workflow_count", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active workflows", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": false, + "__name__": true, + "instance": true, + "job": true, + "major": true, + "minor": true, + "patch": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": {} + } + } + ], + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 7, + "panels": [], + "title": "CPU & Event Loop Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Shows CPU usage breakdown", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_seconds_total[$__rate_interval]) * 100", + "instant": false, + "legendFormat": "Total CPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_user_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "User CPU", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_process_cpu_system_seconds_total[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "System CPU", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Event loop lag", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p99_seconds", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p90_seconds", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_eventloop_lag_p50_seconds", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C" + } + ], + "title": "Event loop latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 11, + "panels": [], + "title": "Memory & Heap Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Resident memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1200000000 + }, + { + "color": "red", + "value": 1800000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_resident_memory_bytes", + "instant": false, + "legendFormat": "RSS", + "range": true, + "refId": "A" + } + ], + "title": "Memory (RSS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "JavaScript heap memory - Node.js automatically manages heap size. High percentage (80-95%) is NORMAL and expected. Watch for heap total growing over time, not the percentage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 8, + "y": 15 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_used_bytes", + "instant": false, + "legendFormat": "Used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_nodejs_heap_size_total_bytes", + "hide": false, + "instant": false, + "legendFormat": "Total (dynamic)", + "range": true, + "refId": "B" + } + ], + "title": "Heap memory (dynamic allocation)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Heap usage percentage shows how full the CURRENT heap allocation is. 80-95% is NORMAL for Node.js! This is NOT an indicator of memory pressure. Only worry if heap total keeps growing OR if you see excessive GC activity.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 15 + }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "(n8n_nodejs_heap_size_used_bytes / n8n_nodejs_heap_size_total_bytes) * 100", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Heap % ", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 49, + "panels": [], + "title": "GC & Runtime Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "Gargabe Collector duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"minor\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Minor GC", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(n8n_nodejs_gc_duration_seconds_sum{kind=\"major\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Major GC", + "range": true, + "refId": "B" + } + ], + "title": "GC duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "description": "File descriptors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_open_fds", + "instant": false, + "legendFormat": "Open FDs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "n8n_process_max_fds", + "hide": false, + "instant": false, + "legendFormat": "Max FDs", + "range": true, + "refId": "B" + } + ], + "title": "File descriptors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [ + "n8n", + "nodejs", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "n8n System Health Overview", + "uid": "adv6qmb", + "version": 14, + "weekStart": "", + "id": null, + "gnetId": 24474 +} \ No newline at end of file diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/prometheus/prometheus.yml b/deployments/nvidia-compose-stack/10-observability-grafana/prometheus/prometheus.yml index d1c03e6..bd68818 100644 --- a/deployments/nvidia-compose-stack/10-observability-grafana/prometheus/prometheus.yml +++ b/deployments/nvidia-compose-stack/10-observability-grafana/prometheus/prometheus.yml @@ -28,3 +28,10 @@ scrape_configs: static_configs: - targets: [] + # n8n metrics (main: process + queue metrics) + # n8n-worker is started via --scale with no fixed container_name / stable DNS, so a static scrape + # target is unreliable; it is intentionally excluded (main already carries the queue metrics). + - job_name: 'n8n' + static_configs: + - targets: ['n8n-main:5678'] + From dead36aab8cf59c6ad26f7cd2df2f2a6a75c515c Mon Sep 17 00:00:00 2001 From: JosephMG Date: Mon, 13 Jul 2026 19:37:08 +0800 Subject: [PATCH 2/3] feat: surface n8n stdout logs in the n8n Grafana dashboard Add a Loki "n8n stdout logs" panel under a new Logs row at the bottom of n8n.json (all three stacks), so n8n logs are visible next to its metrics without needing Explore. Existing metric panels are unchanged. --- .../grafana/provisioning/dashboards/n8n.json | 42 ++++++++++++++++++- .../grafana/provisioning/dashboards/n8n.json | 42 ++++++++++++++++++- .../grafana/provisioning/dashboards/n8n.json | 42 ++++++++++++++++++- 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json index e1a4a85..fc59073 100644 --- a/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json +++ b/deployments/amd-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -1388,6 +1388,46 @@ ], "title": "File descriptors", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 50, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "id": 51, + "type": "logs", + "title": "n8n stdout logs (Loki)", + "datasource": "Loki", + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 33 + }, + "targets": [ + { + "datasource": "Loki", + "expr": "{job=\"docker\", compose_service=~\"n8n-.*\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "prettifyLogMessage": false + } } ], "preload": false, @@ -1412,4 +1452,4 @@ "weekStart": "", "id": null, "gnetId": 24474 -} \ No newline at end of file +} diff --git a/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json index e1a4a85..fc59073 100644 --- a/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json +++ b/deployments/arm64-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -1388,6 +1388,46 @@ ], "title": "File descriptors", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 50, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "id": 51, + "type": "logs", + "title": "n8n stdout logs (Loki)", + "datasource": "Loki", + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 33 + }, + "targets": [ + { + "datasource": "Loki", + "expr": "{job=\"docker\", compose_service=~\"n8n-.*\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "prettifyLogMessage": false + } } ], "preload": false, @@ -1412,4 +1452,4 @@ "weekStart": "", "id": null, "gnetId": 24474 -} \ No newline at end of file +} diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json b/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json index e1a4a85..fc59073 100644 --- a/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json +++ b/deployments/nvidia-compose-stack/10-observability-grafana/grafana/provisioning/dashboards/n8n.json @@ -1388,6 +1388,46 @@ ], "title": "File descriptors", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 50, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "id": 51, + "type": "logs", + "title": "n8n stdout logs (Loki)", + "datasource": "Loki", + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 33 + }, + "targets": [ + { + "datasource": "Loki", + "expr": "{job=\"docker\", compose_service=~\"n8n-.*\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "prettifyLogMessage": false + } } ], "preload": false, @@ -1412,4 +1452,4 @@ "weekStart": "", "id": null, "gnetId": 24474 -} \ No newline at end of file +} From b307b2b3d3886cbf715b97b89fb383e2abfabae1 Mon Sep 17 00:00:00 2001 From: JosephMG Date: Thu, 16 Jul 2026 11:37:36 +0800 Subject: [PATCH 3/3] feat: emit n8n stdout as JSON and parse it in the Alloy log pipeline Set N8N_LOG_FORMAT=json on n8n-main/n8n-worker so stdout is one JSON object per line, and add a loki.process "n8n_json" stage (scoped to compose_service=~"n8n-.*") that extracts level/timestamp and promotes only the low-cardinality level label before forwarding to Loki write. --- deployments/amd-compose-stack/.env.example | 3 +++ .../04-automation-n8n/docker-compose.yaml | 2 ++ .../alloy-config.alloy | 23 ++++++++++++++++++- deployments/arm64-compose-stack/.env.example | 3 +++ .../04-automation-n8n/docker-compose.yaml | 2 ++ .../alloy-config.alloy | 23 ++++++++++++++++++- deployments/nvidia-compose-stack/.env.example | 3 +++ .../04-automation-n8n/docker-compose.yaml | 2 ++ .../alloy-config.alloy | 23 ++++++++++++++++++- 9 files changed, 81 insertions(+), 3 deletions(-) diff --git a/deployments/amd-compose-stack/.env.example b/deployments/amd-compose-stack/.env.example index 3bff9cb..44c97ff 100644 --- a/deployments/amd-compose-stack/.env.example +++ b/deployments/amd-compose-stack/.env.example @@ -76,6 +76,9 @@ N8N_CORS_ALLOW_ORIGIN=* N8N_METRICS=true N8N_METRICS_INCLUDE_QUEUE_METRICS=true +# n8n stdout as JSON (one JSON object per line) so Alloy/Loki can parse fields +N8N_LOG_FORMAT=json + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) diff --git a/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml index a115600..f9c6149 100644 --- a/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml @@ -44,6 +44,7 @@ services: - WEBHOOK_URL=${N8N_URL:-http://host.docker.internal:5678} - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" logging: @@ -88,6 +89,7 @@ services: - QUEUE_HEALTH_CHECK_ACTIVE=true - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy index 38ca9d5..9b379cc 100644 --- a/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy +++ b/deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy @@ -33,7 +33,28 @@ loki.source.docker "containers" { host = "unix:///var/run/docker.sock" targets = discovery.docker.containers.targets relabel_rules = discovery.relabel.containers.rules - forward_to = [loki.write.default.receiver] + forward_to = [loki.process.n8n_json.receiver] +} + +// Parse ONLY the n8n services' JSON stdout. stage.match action defaults to "keep": +// non-matching lines pass through untouched (other containers' plain-text logs are +// NOT JSON-parsed). Only the low-cardinality `level` becomes a label; high-cardinality +// fields stay in the log body to bound Loki series. +loki.process "n8n_json" { + stage.match { + selector = "{compose_service=~\"n8n-.*\"}" + stage.json { + expressions = { level = "level", ts = "timestamp" } + } + stage.timestamp { + source = "ts" + format = "RFC3339" + } + stage.labels { + values = { level = "" } + } + } + forward_to = [loki.write.default.receiver] } loki.write "default" { diff --git a/deployments/arm64-compose-stack/.env.example b/deployments/arm64-compose-stack/.env.example index ebea9c1..9c46854 100644 --- a/deployments/arm64-compose-stack/.env.example +++ b/deployments/arm64-compose-stack/.env.example @@ -62,6 +62,9 @@ N8N_CORS_ALLOW_ORIGIN=* N8N_METRICS=true N8N_METRICS_INCLUDE_QUEUE_METRICS=true +# n8n stdout as JSON (one JSON object per line) so Alloy/Loki can parse fields +N8N_LOG_FORMAT=json + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) diff --git a/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml index b3bc98c..af93b4b 100644 --- a/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/arm64-compose-stack/04-automation-n8n/docker-compose.yaml @@ -44,6 +44,7 @@ services: - WEBHOOK_URL=${N8N_URL:-http://localhost:5678} - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" logging: @@ -88,6 +89,7 @@ services: - QUEUE_HEALTH_CHECK_ACTIVE=true - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy index 38ca9d5..9b379cc 100644 --- a/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy +++ b/deployments/arm64-compose-stack/10-observability-grafana/alloy-config.alloy @@ -33,7 +33,28 @@ loki.source.docker "containers" { host = "unix:///var/run/docker.sock" targets = discovery.docker.containers.targets relabel_rules = discovery.relabel.containers.rules - forward_to = [loki.write.default.receiver] + forward_to = [loki.process.n8n_json.receiver] +} + +// Parse ONLY the n8n services' JSON stdout. stage.match action defaults to "keep": +// non-matching lines pass through untouched (other containers' plain-text logs are +// NOT JSON-parsed). Only the low-cardinality `level` becomes a label; high-cardinality +// fields stay in the log body to bound Loki series. +loki.process "n8n_json" { + stage.match { + selector = "{compose_service=~\"n8n-.*\"}" + stage.json { + expressions = { level = "level", ts = "timestamp" } + } + stage.timestamp { + source = "ts" + format = "RFC3339" + } + stage.labels { + values = { level = "" } + } + } + forward_to = [loki.write.default.receiver] } loki.write "default" { diff --git a/deployments/nvidia-compose-stack/.env.example b/deployments/nvidia-compose-stack/.env.example index 89d571c..196bba3 100644 --- a/deployments/nvidia-compose-stack/.env.example +++ b/deployments/nvidia-compose-stack/.env.example @@ -61,6 +61,9 @@ N8N_CORS_ALLOW_ORIGIN=* N8N_METRICS=true N8N_METRICS_INCLUDE_QUEUE_METRICS=true +# n8n stdout as JSON (one JSON object per line) so Alloy/Loki can parse fields +N8N_LOG_FORMAT=json + # --- 5) RAG & Memory --- QDRANT_PORT=6333 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS) diff --git a/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml b/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml index dcb8151..228cdc6 100644 --- a/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml +++ b/deployments/nvidia-compose-stack/04-automation-n8n/docker-compose.yaml @@ -44,6 +44,7 @@ services: - N8N_SECURE_COOKIE=false - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" # Note: Redis dependency is external (from 03 stack) @@ -90,6 +91,7 @@ services: - QUEUE_HEALTH_CHECK_ACTIVE=true - N8N_METRICS=${N8N_METRICS:-true} - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true} + - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json} extra_hosts: - "host.docker.internal:host-gateway" logging: diff --git a/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy b/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy index 38ca9d5..9b379cc 100644 --- a/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy +++ b/deployments/nvidia-compose-stack/10-observability-grafana/alloy-config.alloy @@ -33,7 +33,28 @@ loki.source.docker "containers" { host = "unix:///var/run/docker.sock" targets = discovery.docker.containers.targets relabel_rules = discovery.relabel.containers.rules - forward_to = [loki.write.default.receiver] + forward_to = [loki.process.n8n_json.receiver] +} + +// Parse ONLY the n8n services' JSON stdout. stage.match action defaults to "keep": +// non-matching lines pass through untouched (other containers' plain-text logs are +// NOT JSON-parsed). Only the low-cardinality `level` becomes a label; high-cardinality +// fields stay in the log body to bound Loki series. +loki.process "n8n_json" { + stage.match { + selector = "{compose_service=~\"n8n-.*\"}" + stage.json { + expressions = { level = "level", ts = "timestamp" } + } + stage.timestamp { + source = "ts" + format = "RFC3339" + } + stage.labels { + values = { level = "" } + } + } + forward_to = [loki.write.default.receiver] } loki.write "default" {