From 2415e5e8c3139a3021fe2849fda874941607e7ea Mon Sep 17 00:00:00 2001 From: "laochou.sc" Date: Tue, 14 Apr 2026 10:58:24 +0800 Subject: [PATCH 1/2] feat(fluent-plugin-oceanbase-logs): split Loki demo into two Fluentd services Replace single fluentd-to-loki.conf with fluentd-to-loki1.conf and fluentd-to-loki2.conf; add fluentd-oceanbase-demo1 and demo2 services in docker-compose. Update .env.example for the demo environment variables. Made-with: Cursor --- fluent-plugin-oceanbase-logs/.env.example | 3 - .../oceanbase2loki-docker/docker-compose.yml | 41 +++++++++-- ...ntd-to-loki.conf => fluentd-to-loki1.conf} | 49 ++++--------- .../fluentd-to-loki2.conf | 72 +++++++++++++++++++ 4 files changed, 120 insertions(+), 45 deletions(-) rename fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/{fluentd-to-loki.conf => fluentd-to-loki1.conf} (75%) create mode 100644 fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf diff --git a/fluent-plugin-oceanbase-logs/.env.example b/fluent-plugin-oceanbase-logs/.env.example index 3bb5f308..d478316d 100644 --- a/fluent-plugin-oceanbase-logs/.env.example +++ b/fluent-plugin-oceanbase-logs/.env.example @@ -1,9 +1,6 @@ # --- Required (OceanBase Cloud API) --- OCEANBASE_ACCESS_KEY_ID=your_access_key_id OCEANBASE_ACCESS_KEY_SECRET=your_access_key_secret -# Instance ID and tenant ID must both come from the console; do not reuse the same value for both. -OCEANBASE_INSTANCE_ID=ob317v4uif**** -OCEANBASE_TENANT_ID=t4louaeei**** # --- Optional --- # LOKI_URL=http://loki:3100 diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml index 594ee5ba..82c2f4f4 100644 --- a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml @@ -24,10 +24,10 @@ services: networks: - obs - fluentd-oceanbase-demo: + fluentd-oceanbase-demo1: image: fluent/fluentd:v1.16-1 platform: linux/amd64 - container_name: fluentd-oceanbase-demo + container_name: fluentd-oceanbase-demo1 user: root command: - /bin/sh @@ -37,13 +37,40 @@ services: mkdir -p /var/log/fluentd exec fluentd -c /fluentd/etc/fluentd-to-loki.conf volumes: - - ./fluentd-to-loki.conf:/fluentd/etc/fluentd-to-loki.conf:ro + - ./fluentd-to-loki1.conf:/fluentd/etc/fluentd-to-loki.conf:ro + environment: + LOKI_URL: "${LOKI_URL:-http://loki:3100}" + OCEANBASE_ACCESS_KEY_ID: "${OCEANBASE_ACCESS_KEY_ID}" + OCEANBASE_ACCESS_KEY_SECRET: "${OCEANBASE_ACCESS_KEY_SECRET}" + OCEANBASE_ENDPOINT: "${OCEANBASE_ENDPOINT:-api-cloud-cn.oceanbase.com}" + OCEANBASE_FETCH_INTERVAL: "${OCEANBASE_FETCH_INTERVAL:-60}" + OCEANBASE_LOOKBACK_SECONDS: "${OCEANBASE_LOOKBACK_SECONDS:-600}" + OCEANBASE_DB_NAME: "${OCEANBASE_DB_NAME:-}" + OCEANBASE_SEARCH_KEYWORD: "${OCEANBASE_SEARCH_KEYWORD:-}" + OCEANBASE_PROJECT_ID: "${OCEANBASE_PROJECT_ID:-}" + networks: + - obs + depends_on: + - loki + + fluentd-oceanbase-demo2: + image: fluent/fluentd:v1.16-1 + platform: linux/amd64 + container_name: fluentd-oceanbase-demo2 + user: root + command: + - /bin/sh + - -c + - | + gem install fluent-plugin-oceanbase-logs fluent-plugin-grafana-loki --no-document + mkdir -p /var/log/fluentd + exec fluentd -c /fluentd/etc/fluentd-to-loki.conf + volumes: + - ./fluentd-to-loki2.conf:/fluentd/etc/fluentd-to-loki.conf:ro environment: LOKI_URL: "${LOKI_URL:-http://loki:3100}" OCEANBASE_ACCESS_KEY_ID: "${OCEANBASE_ACCESS_KEY_ID}" OCEANBASE_ACCESS_KEY_SECRET: "${OCEANBASE_ACCESS_KEY_SECRET}" - OCEANBASE_INSTANCE_ID: "${OCEANBASE_INSTANCE_ID}" - OCEANBASE_TENANT_ID: "${OCEANBASE_TENANT_ID}" OCEANBASE_ENDPOINT: "${OCEANBASE_ENDPOINT:-api-cloud-cn.oceanbase.com}" OCEANBASE_FETCH_INTERVAL: "${OCEANBASE_FETCH_INTERVAL:-60}" OCEANBASE_LOOKBACK_SECONDS: "${OCEANBASE_LOOKBACK_SECONDS:-600}" @@ -67,7 +94,9 @@ services: - obs depends_on: - loki + - fluentd-oceanbase-demo1 + - fluentd-oceanbase-demo2 networks: obs: - driver: bridge + driver: bridge \ No newline at end of file diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki.conf b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf similarity index 75% rename from fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki.conf rename to fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf index beb8dd1e..1eb49dba 100644 --- a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki.conf +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf @@ -3,56 +3,33 @@ tag oceanbase.slow_sql log_type slow_sql - access_key_id "#{ENV['OCEANBASE_ACCESS_KEY_ID']}" - access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}" + access_key_id "" + access_key_secret "" - instance_id "#{ENV['OCEANBASE_INSTANCE_ID']}" - tenant_id "#{ENV['OCEANBASE_TENANT_ID']}" + + instance_id "" + tenant_id "" + - endpoint "#{ENV['OCEANBASE_ENDPOINT'] || 'api-cloud-cn.oceanbase.com'}" - fetch_interval "#{ENV['OCEANBASE_FETCH_INTERVAL'] || 30}" - lookback_seconds "#{ENV['OCEANBASE_LOOKBACK_SECONDS'] || 600}" - - sql_text_length "#{ENV['OCEANBASE_SQL_TEXT_LENGTH'] || 65535}" + + instance_id "" + tenant_id "" + - deduplicate true - include_metadata true - - @type local - persistent true - path /var/log/fluentd/oceanbase_slow_sql.state - - - - - @type oceanbase_logs - tag oceanbase.top_sql - log_type top_sql - - access_key_id "#{ENV['OCEANBASE_ACCESS_KEY_ID']}" - access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}" - - instance_id "#{ENV['OCEANBASE_INSTANCE_ID']}" - tenant_id "#{ENV['OCEANBASE_TENANT_ID']}" - - endpoint "#{ENV['OCEANBASE_ENDPOINT'] || 'api-cloud-cn.oceanbase.com'}" - fetch_interval "#{ENV['OCEANBASE_FETCH_INTERVAL'] || 60}" + endpoint "api-cloud-cn.oceanbase.com" + fetch_interval "#{ENV['OCEANBASE_FETCH_INTERVAL'] || 30}" lookback_seconds "#{ENV['OCEANBASE_LOOKBACK_SECONDS'] || 600}" sql_text_length "#{ENV['OCEANBASE_SQL_TEXT_LENGTH'] || 65535}" - db_name "#{ENV['OCEANBASE_DB_NAME'] || 'test'}" - search_keyword "#{ENV['OCEANBASE_SEARCH_KEYWORD'] || 'SELECT'}" - project_id "#{ENV['OCEANBASE_PROJECT_ID']}" - deduplicate true include_metadata true @type local persistent true - path /var/log/fluentd/oceanbase_top_sql.state + path /var/log/fluentd/oceanbase_slow_sql.state diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf new file mode 100644 index 00000000..b5d7ce58 --- /dev/null +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf @@ -0,0 +1,72 @@ + + @type oceanbase_logs + tag oceanbase.slow_sql + log_type slow_sql + + access_key_id "" + access_key_secret "" + + + instance_id "" + tenant_id "" + + + + endpoint "api-cloud-cn.oceanbase.com" + fetch_interval "#{ENV['OCEANBASE_FETCH_INTERVAL'] || 30}" + lookback_seconds "#{ENV['OCEANBASE_LOOKBACK_SECONDS'] || 600}" + + sql_text_length "#{ENV['OCEANBASE_SQL_TEXT_LENGTH'] || 65535}" + + deduplicate true + include_metadata true + + + @type local + persistent true + path /var/log/fluentd/oceanbase_slow_sql.state + + + + + @type record_transformer + enable_ruby true + renew_record true + + log ${r = record; require('json'); u = (r.dig("userName") || "").to_s; u = "unknown" if u.empty?; ip = (r.dig("userClientIp") || r.dig("clientIp") || "").to_s; tid = (r.dig("traceId") || "").to_s; t = (r.dig("requestTime") || "").to_s; et_ms = r.dig("executeTime"); qt_s = et_ms.nil? ? 0.0 : et_ms.to_f / 1000.0; lock_s = (r.dig("queueTime") || 0).to_f; rs = (r.dig("returnRows") || 0); mr = (r.dig("memstoreReadRows") || 0).to_f; sr = (r.dig("ssstoreReadRows") || 0).to_f; re = (mr + sr).to_i; db = (r.dig("dbName") || "").to_s; sql = (r.dig("fullSqlText") || "").to_s; rid = r.dig("requestId"); ltype = (r.dig("ob_log_type") || "").to_s; ts_unix = 0; begin; ts_unix = Time.parse(t).utc.to_i if !t.empty?; rescue; end; ub = sprintf("\u0023 User@Host: %s%c%s%c @ %c%s%c", u, 91, u, 93, 91, ip, 93); ub = ub + sprintf(" Id: %s", rid) if rid; parts = Array.new; parts.push("\u0023 Time: " + t) unless t.empty?; parts.push(sprintf("\u0023 Log_type: %s", ltype)) unless ltype.empty?; parts.push(ub); parts.push("\u0023 Trace_id: " + tid) unless tid.to_s.empty?; parts.push(sprintf("\u0023 Query_time: %.6f Lock_time: %.6f Rows_sent: %s Rows_examined: %s", qt_s, lock_s, rs.to_s, re.to_s)); parts.push(sprintf("use %s%c", db, 59)) unless db.empty?; parts.push(sprintf("SET timestamp=%s%c", ts_unix.to_s, 59)) if ts_unix > 0; parts.push(sql); mysql_part = parts.join("\n"); keys = %w(requestTime clientIp traceId executeTime queueTime returnRows rowsExaminedApprox dbName fullSqlText userName userClientIp clientId clientPort requestId sqlTextShort elapsedTime cpuTime memstoreReadRows ssstoreReadRows diskReads affectedRows blockCacheHit blockIndexCacheHit bloomFilterCacheHit decodeTime executorRpc expectedWorkerCount getPlanTime hitPlan inner netTime netWaitTime partitionCount planId planType retCode retryCount rowCacheHit rpcCount server sqlId sqlType tableScan transHash usedWorkerCount waitCount waitEvent ob_instance_id ob_tenant_id ob_log_type query_start_time query_end_time); h = Hash.new; keys.each do |k|; if k == "clientId"; h.store(k, r.dig("clientIp") || r.dig("userClientIp")); elsif k == "rowsExaminedApprox"; h.store(k, ((r.dig("memstoreReadRows") || 0).to_f + (r.dig("ssstoreReadRows") || 0).to_f).to_i); elsif r.key?(k); h.store(k, r.dig(k)); end; end; mysql_part + "\n\n" + "\u0023 ob_api_payload\uFF1A" + JSON.pretty_generate(h)} + ob_instance_id ${record.dig("ob_instance_id")} + ob_tenant_id ${record.dig("ob_tenant_id")} + dbName ${record.dig("dbName")} + sqlType ${record.dig("sqlType")} + userName ${record.dig("userName")} + ob_log_type ${record.dig("ob_log_type")} + + + + + @type loki + url "#{ENV['LOKI_URL'] || 'http://loki:3100'}" + + drop_single_key true + + extra_labels {"job":"oceanbase-logs"} + + + + remove_keys ob_log_type,message + + + @type memory + flush_interval 10s + chunk_limit_size 1m + retry_max_interval 30s + retry_forever true + + \ No newline at end of file From a84cb6eb6e6af4820f978a8745d771cf6a3ca8c5 Mon Sep 17 00:00:00 2001 From: "laochou.sc" Date: Tue, 14 Apr 2026 14:42:19 +0800 Subject: [PATCH 2/2] feat(oceanbase-logs): wire multi-target examples to env and AccessKey - Use blocks in example fluentd.conf instead of top-level instance/tenant. - Read AccessKey and dual instance/tenant pairs from env in Loki Docker examples. - Pass OCEANBASE_INSTANCE1/2 and OCEANBASE_TENANT1/2 into both Fluentd services. Made-with: Cursor --- fluent-plugin-oceanbase-logs/example/fluentd.conf | 6 ++++-- .../oceanbase2loki-docker/docker-compose.yml | 8 ++++++++ .../oceanbase2loki-docker/fluentd-to-loki1.conf | 12 ++++++------ .../oceanbase2loki-docker/fluentd-to-loki2.conf | 13 +++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/fluent-plugin-oceanbase-logs/example/fluentd.conf b/fluent-plugin-oceanbase-logs/example/fluentd.conf index ed630681..8ba1a58c 100644 --- a/fluent-plugin-oceanbase-logs/example/fluentd.conf +++ b/fluent-plugin-oceanbase-logs/example/fluentd.conf @@ -10,8 +10,10 @@ access_key_id "#{ENV['OCEANBASE_ACCESS_KEY_ID']}" access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}" - instance_id "#{ENV['OCEANBASE_INSTANCE_ID']}" - tenant_id "#{ENV['OCEANBASE_TENANT_ID']}" + + instance_id "" + tenant_id "" + # API endpoint (override with OCEANBASE_ENDPOINT) endpoint "#{ENV['OCEANBASE_ENDPOINT'] || 'api-cloud-cn.oceanbase.com'}" diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml index 82c2f4f4..fa79fa21 100644 --- a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/docker-compose.yml @@ -41,6 +41,10 @@ services: environment: LOKI_URL: "${LOKI_URL:-http://loki:3100}" OCEANBASE_ACCESS_KEY_ID: "${OCEANBASE_ACCESS_KEY_ID}" + OCEANBASE_INSTANCE1: "${OCEANBASE_INSTANCE1}" + OCEANBASE_TENANT1: "${OCEANBASE_TENANT1}" + OCEANBASE_INSTANCE2: "${OCEANBASE_INSTANCE2}" + OCEANBASE_TENANT2: "${OCEANBASE_TENANT2}" OCEANBASE_ACCESS_KEY_SECRET: "${OCEANBASE_ACCESS_KEY_SECRET}" OCEANBASE_ENDPOINT: "${OCEANBASE_ENDPOINT:-api-cloud-cn.oceanbase.com}" OCEANBASE_FETCH_INTERVAL: "${OCEANBASE_FETCH_INTERVAL:-60}" @@ -70,6 +74,10 @@ services: environment: LOKI_URL: "${LOKI_URL:-http://loki:3100}" OCEANBASE_ACCESS_KEY_ID: "${OCEANBASE_ACCESS_KEY_ID}" + OCEANBASE_INSTANCE1: "${OCEANBASE_INSTANCE1}" + OCEANBASE_TENANT1: "${OCEANBASE_TENANT1}" + OCEANBASE_INSTANCE2: "${OCEANBASE_INSTANCE2}" + OCEANBASE_TENANT2: "${OCEANBASE_TENANT2}" OCEANBASE_ACCESS_KEY_SECRET: "${OCEANBASE_ACCESS_KEY_SECRET}" OCEANBASE_ENDPOINT: "${OCEANBASE_ENDPOINT:-api-cloud-cn.oceanbase.com}" OCEANBASE_FETCH_INTERVAL: "${OCEANBASE_FETCH_INTERVAL:-60}" diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf index 1eb49dba..53e2c8ac 100644 --- a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki1.conf @@ -3,17 +3,17 @@ tag oceanbase.slow_sql log_type slow_sql - access_key_id "" - access_key_secret "" + access_key_id "#{ENV['OCEANBASE_ACCESS_KEY_ID']}" + access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}" - instance_id "" - tenant_id "" + instance_id "#{ENV['OCEANBASE_INSTANCE1']}" + tenant_id "#{ENV['OCEANBASE_TENANT1']}" - instance_id "" - tenant_id "" + instance_id "#{ENV['OCEANBASE_INSTANCE2']}" + tenant_id "#{ENV['OCEANBASE_TENANT2']}" diff --git a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf index b5d7ce58..53e2c8ac 100644 --- a/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf +++ b/fluent-plugin-oceanbase-logs/example/oceanbase2loki-docker/fluentd-to-loki2.conf @@ -3,12 +3,17 @@ tag oceanbase.slow_sql log_type slow_sql - access_key_id "" - access_key_secret "" + access_key_id "#{ENV['OCEANBASE_ACCESS_KEY_ID']}" + access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}" - instance_id "" - tenant_id "" + instance_id "#{ENV['OCEANBASE_INSTANCE1']}" + tenant_id "#{ENV['OCEANBASE_TENANT1']}" + + + + instance_id "#{ENV['OCEANBASE_INSTANCE2']}" + tenant_id "#{ENV['OCEANBASE_TENANT2']}"