Feat/n8n stdout log

deployments/amd-compose-stack/.env.example

@@ -72,6 +72,13 @@ WORKER_CONCURRENCY=5
 N8N_URL=
 N8N_CORS_ALLOW_ORIGIN=*
 
+# n8n Metrics — free Prometheus endpoint; both main + worker expose /metrics
+N8N_METRICS=true
+N8N_METRICS_INCLUDE_QUEUE_METRICS=true
+
+# n8n stdout as JSON (one JSON object per line) so Alloy/Loki can parse fields
+N8N_LOG_FORMAT=json
+
 # --- 5) RAG & Memory ---
 QDRANT_PORT=6333
 # Qdrant search thread count (QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS)
@@ -93,6 +100,9 @@ GATEWAY_API_URL=http://system-api-bridge:8000
 GRAFANA_PORT=3000
 GRAFANA_PASS=CHANGE_ME
 
+# Grafana Alloy (collects container logs -> Loki)
+ALLOY_IMAGE=grafana/alloy:v1.17.1
+
 # --- 12) commercial gateway ---
 SYSTEM_API_GATEWAY_PORT=5055
 
@@ -120,5 +130,5 @@ PROMETHEUS_IMAGE=prom/prometheus:latest
 GRAFANA_IMAGE=grafana/grafana:latest
 LOKI_IMAGE=grafana/loki:latest
 CADVISOR_IMAGE=gcr.io/cadvisor/cadvisor:latest
-DCGM_EXPORTER_IMAGE=nvidia/dcgm-exporter:latest
+DCGM_EXPORTER_IMAGE=nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-ubuntu22.04
 ROCM_EXPORTER_IMAGE=rocm/device-metrics-exporter:v1.4.2

deployments/amd-compose-stack/04-automation-n8n/docker-compose.yaml

@@ -42,6 +42,9 @@ services:
       - OFFLOAD_MANUAL_EXECUTIONS_TO_WORKERS=true
       - N8N_EDITOR_BASE_URL=${N8N_URL:-http://host.docker.internal:5678}
       - WEBHOOK_URL=${N8N_URL:-http://host.docker.internal:5678}
+      - N8N_METRICS=${N8N_METRICS:-true}
+      - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true}
+      - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     logging:
@@ -84,6 +87,9 @@ services:
       - N8N_SECURE_COOKIE=false
       - N8N_WORKER_MAX_CONCURRENT_EXECUTIONS=${WORKER_CONCURRENCY:-5}
       - QUEUE_HEALTH_CHECK_ACTIVE=true
+      - N8N_METRICS=${N8N_METRICS:-true}
+      - N8N_METRICS_INCLUDE_QUEUE_METRICS=${N8N_METRICS_INCLUDE_QUEUE_METRICS:-true}
+      - N8N_LOG_FORMAT=${N8N_LOG_FORMAT:-json}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     logging:

deployments/amd-compose-stack/10-observability-grafana/alloy-config.alloy

@@ -0,0 +1,64 @@
+// Grafana Alloy — collect every Docker container's stdout/stderr and ship to Loki.
+// Containers are discovered through the Docker socket, tailed live, and forwarded
+// to the Loki service on the shared ai_stack_net network.
+
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  // Strip the leading "/" from the Docker container name -> "container" label.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+
+  // Promote the compose service name -> "compose_service" label.
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+
+  // Static "job" label so every stream is queryable as {job="docker"}.
+  rule {
+    target_label = "job"
+    replacement  = "docker"
+  }
+}
+
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.docker.containers.targets
+  relabel_rules = discovery.relabel.containers.rules
+  forward_to    = [loki.process.n8n_json.receiver]
+}
+
+// Parse ONLY the n8n services' JSON stdout. stage.match action defaults to "keep":
+// non-matching lines pass through untouched (other containers' plain-text logs are
+// NOT JSON-parsed). Only the low-cardinality `level` becomes a label; high-cardinality
+// fields stay in the log body to bound Loki series.
+loki.process "n8n_json" {
+  stage.match {
+    selector = "{compose_service=~\"n8n-.*\"}"
+    stage.json {
+      expressions = { level = "level", ts = "timestamp" }
+    }
+    stage.timestamp {
+      source = "ts"
+      format = "RFC3339"
+    }
+    stage.labels {
+      values = { level = "" }
+    }
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}

deployments/amd-compose-stack/10-observability-grafana/docker-compose.yaml

@@ -166,10 +166,42 @@ services:
         max-size: "50m"
         max-file: "3"
 
+  alloy:
+    image: ${ALLOY_IMAGE:-grafana/alloy:v1.17.1}
+    container_name: alloy
+    restart: always
+    healthcheck:
+      test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/12345"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - ./alloy-config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - alloy_data:/var/lib/alloy/data
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    networks:
+      - ai_stack_net
+    depends_on:
+      - loki
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "3"
+
 networks:
   ai_stack_net:
     external: true
 
 volumes:
   prometheus_data:
   grafana_data:
+  alloy_data: