From 79b78dbcb04c35d0fe502a4278552a5bae76988b Mon Sep 17 00:00:00 2001 From: n0str Date: Fri, 30 Jan 2026 14:26:46 +0300 Subject: [PATCH 1/2] Prepare otel-collector and grafana with loki --- .env.sample | 14 +- README.md | 34 ++ docker-compose.yml | 43 +++ loki/.gitignore | 2 + loki/config/loki.yml | 42 +++ loki/data/.gitkeep | 1 + loki/grafana/.gitignore | 2 + loki/grafana/data/.gitkeep | 1 + .../dashboards/collector-logs.json | 335 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 12 + .../grafana/provisioning/datasources/loki.yml | 9 + loki/otel-collector/config.yml | 52 +++ 12 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 loki/.gitignore create mode 100644 loki/config/loki.yml create mode 100644 loki/data/.gitkeep create mode 100644 loki/grafana/.gitignore create mode 100644 loki/grafana/data/.gitkeep create mode 100644 loki/grafana/provisioning/dashboards/collector-logs.json create mode 100644 loki/grafana/provisioning/dashboards/dashboards.yml create mode 100644 loki/grafana/provisioning/datasources/loki.yml create mode 100644 loki/otel-collector/config.yml diff --git a/.env.sample b/.env.sample index 5ae5fc8..cdd183b 100644 --- a/.env.sample +++ b/.env.sample @@ -1 +1,13 @@ -s3_basic_auth= \ No newline at end of file +s3_basic_auth= + +# Loki logging (local vs cloud) +# Local (default) +LOKI_ENDPOINT=http://loki:3100/loki/api/v1/push +LOKI_AUTH_HEADER= +GRAFANA_CLOUD_LOKI_USER= +GRAFANA_CLOUD_LOKI_API_KEY= + +# Grafana Cloud Loki example (comment local values above and uncomment below) +# LOKI_ENDPOINT=https://logs-prod-012.grafana.net/loki/api/v1/push +# GRAFANA_CLOUD_LOKI_USER=123456 +# GRAFANA_CLOUD_LOKI_API_KEY=glc_xxx diff --git a/README.md b/README.md index 442eb09..da2dd30 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,40 @@ This repo contains all subrepos of Hawk modules except Client SDKs (Catchers). I 2. Create `.env` file in those repositories where there is `.env.sample` file. 3. Run `docker-compose up` to run all hawk services or list only the necessary services in the command above. +## Logging (Loki + Grafana) + +Grafana runs in Docker Compose and ships with a provisioned Logs dashboard that opens by default. + +### Local Loki (default) + +1. Copy `.env.sample` to `.env` (or update your existing `.env`) and keep the local values: + +``` +LOKI_ENDPOINT=http://loki:3100/loki/api/v1/push +LOKI_AUTH_HEADER= +GRAFANA_CLOUD_LOKI_USER= +GRAFANA_CLOUD_LOKI_API_KEY= +``` + +2. Start services: `docker-compose up` +3. Open Grafana: `http://localhost:3001` + User: `admin` Password: `admin` + +### Grafana Cloud Loki + +1. In `.env`, comment the local `LOKI_ENDPOINT` and set the cloud values: + +``` +# LOKI_ENDPOINT=http://loki:3100/loki/api/v1/push +LOKI_ENDPOINT=https:///loki/api/v1/push +GRAFANA_CLOUD_LOKI_USER= +GRAFANA_CLOUD_LOKI_API_KEY= +``` + +2. Start services: `docker-compose up` +3. Open Grafana: `http://localhost:3001` + User: `admin` Password: `admin` + ## Troubleshooting If something went wrong, check this items. diff --git a/docker-compose.yml b/docker-compose.yml index ee2308d..9786a40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,6 +69,49 @@ services: - 9091:9091 restart: unless-stopped + loki: + image: grafana/loki:2.9.5 + ports: + - 3100:3100 + volumes: + - ./loki/config/loki.yml:/etc/loki/loki.yml:ro + - ./loki/data:/loki + command: + - -config.file=/etc/loki/loki.yml + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.98.0 + ports: + - 4317:4317 + - 4318:4318 + environment: + - LOKI_ENDPOINT=${LOKI_ENDPOINT} + - LOKI_AUTH_HEADER=${LOKI_AUTH_HEADER} + - GRAFANA_CLOUD_LOKI_USER=${GRAFANA_CLOUD_LOKI_USER} + - GRAFANA_CLOUD_LOKI_API_KEY=${GRAFANA_CLOUD_LOKI_API_KEY} + volumes: + - ./loki/otel-collector/config.yml:/etc/otel-collector/config.yml:ro + command: + - --config=/etc/otel-collector/config.yml + restart: unless-stopped + + grafana: + image: grafana/grafana:10.4.3 + ports: + - 3001:3000 + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_DEFAULT_THEME=light + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/collector-logs.json + volumes: + - ./loki/grafana/provisioning:/etc/grafana/provisioning:ro + - ./loki/grafana/data:/var/lib/grafana + depends_on: + - loki + restart: unless-stopped + prometheus: image: prom/prometheus ports: diff --git a/loki/.gitignore b/loki/.gitignore new file mode 100644 index 0000000..06b2ab9 --- /dev/null +++ b/loki/.gitignore @@ -0,0 +1,2 @@ +data/* +!data/.gitkeep diff --git a/loki/config/loki.yml b/loki/config/loki.yml new file mode 100644 index 0000000..668e71d --- /dev/null +++ b/loki/config/loki.yml @@ -0,0 +1,42 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + log_level: info + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ingester: + wal: + dir: /loki/wal + lifecycler: + ring: + kvstore: + store: inmemory + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +limits_config: + retention_period: 168h diff --git a/loki/data/.gitkeep b/loki/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/loki/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/loki/grafana/.gitignore b/loki/grafana/.gitignore new file mode 100644 index 0000000..06b2ab9 --- /dev/null +++ b/loki/grafana/.gitignore @@ -0,0 +1,2 @@ +data/* +!data/.gitkeep diff --git a/loki/grafana/data/.gitkeep b/loki/grafana/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/loki/grafana/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/loki/grafana/provisioning/dashboards/collector-logs.json b/loki/grafana/provisioning/dashboards/collector-logs.json new file mode 100644 index 0000000..1979656 --- /dev/null +++ b/loki/grafana/provisioning/dashboards/collector-logs.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "footer": { + "reducers": [] + }, + "inspect": true + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ENV" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "severity" + }, + "properties": [ + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "body" + }, + "properties": [ + { + "id": "custom.width", + "value": 683 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "log_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 312 + }, + { + "id": "links", + "value": [ + { + "title": " 🔎 Show", + "url": "/d/collector-logs/collector-logs?orgId=1&viewPanel=2&var-log_id=${__value.raw}" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "data-links" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Details" + }, + "properties": [ + { + "id": "custom.width", + "value": 87 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "log_id" + } + ] + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{service_name=\"collector\",deployment_environment=\"local\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Latest Logs", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "json", + "jsonPaths": [ + { + "alias": "ENV", + "path": "deployment_environment" + } + ], + "keepTime": false, + "replace": false, + "source": "labels" + } + }, + { + "id": "extractFields", + "options": { + "delimiter": ",", + "format": "json", + "jsonPaths": [ + { + "alias": "severity", + "path": "severity" + }, + { + "alias": "body", + "path": "body" + }, + { + "alias": "attributes", + "path": "attributes" + }, + { + "alias": "log_id", + "path": "attributes.log_id" + } + ], + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "id": true, + "labelTypes": true, + "labels": true, + "traceID": true, + "traceID (field)": true, + "tsNs": true + }, + "includeByName": {}, + "indexByName": { + "ENV": 1, + "Line": 12, + "Time": 3, + "attributes": 11, + "body": 10, + "id": 7, + "labelTypes": 6, + "labels": 2, + "log_id": 0, + "severity": 4, + "traceID": 8, + "traceID (field)": 9, + "tsNs": 5 + }, + "renameByName": { + "ENV": "", + "labels": "", + "log_id": "Details" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "detailsMode": "inline", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "fontSize": "small", + "prettifyLogMessage": true, + "showControls": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": true + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"collector\",deployment_environment=\"local\"} | json | attributes_log_id=~\"$log_id\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Details", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ".*", + "value": ".*" + }, + "description": "", + "hide": 2, + "label": "log_id", + "name": "log_id", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Collector Logs", + "uid": "collector-logs", + "version": 1, + "weekStart": "" +} diff --git a/loki/grafana/provisioning/dashboards/dashboards.yml b/loki/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..befca0d --- /dev/null +++ b/loki/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/loki/grafana/provisioning/datasources/loki.yml b/loki/grafana/provisioning/datasources/loki.yml new file mode 100644 index 0000000..aba8b4d --- /dev/null +++ b/loki/grafana/provisioning/datasources/loki.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true diff --git a/loki/otel-collector/config.yml b/loki/otel-collector/config.yml new file mode 100644 index 0000000..f4e602e --- /dev/null +++ b/loki/otel-collector/config.yml @@ -0,0 +1,52 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + transform/add-log-id: + error_mode: ignore + log_statements: + - context: log + statements: + - set(attributes["log_id"], UUID()) + resource/ensure-service-name: + attributes: + - action: insert + key: service.name + value: collector + - action: insert + key: deployment_environment + value: local + - action: insert + key: loki.resource.labels + value: service.name, deployment_environment + batch: + +extensions: + basicauth/grafana_cloud: + client_auth: + username: ${env:GRAFANA_CLOUD_LOKI_USER} + password: ${env:GRAFANA_CLOUD_LOKI_API_KEY} + +exporters: + debug: + verbosity: detailed + loki: + # Example: https://logs-prod-012.grafana.net/loki/api/v1/push + endpoint: ${env:LOKI_ENDPOINT} + auth: + authenticator: basicauth/grafana_cloud + + default_labels_enabled: + exporter: false + job: false + +service: + extensions: [basicauth/grafana_cloud] + pipelines: + logs: + receivers: [otlp] + processors: [transform/add-log-id, resource/ensure-service-name, batch] + exporters: [debug, loki] From 51e586e18086fef15932a862cef19d8892aed347 Mon Sep 17 00:00:00 2001 From: Aleksandr Menshchikov Date: Fri, 30 Jan 2026 14:57:05 +0300 Subject: [PATCH 2/2] Update docker-compose.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9786a40..142b9bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,6 @@ services: - 4318:4318 environment: - LOKI_ENDPOINT=${LOKI_ENDPOINT} - - LOKI_AUTH_HEADER=${LOKI_AUTH_HEADER} - GRAFANA_CLOUD_LOKI_USER=${GRAFANA_CLOUD_LOKI_USER} - GRAFANA_CLOUD_LOKI_API_KEY=${GRAFANA_CLOUD_LOKI_API_KEY} volumes: