WIP Add examples of log tagging

compare-buckets.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -eo pipefail
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 <file.json> [file.json...]"
+  echo "Compares GCS bucket listings exported via: gcloud storage ls --json 'gs://bucket/**' > file.json"
+  exit 1
+fi
+
+for f in "$@"; do
+  abs=$(cd "$(dirname "$f")" && pwd)/$(basename "$f")
+
+  if [ ! -s "$abs" ]; then
+    echo "-- ${f}: empty file (no objects found)"
+    echo ""
+    continue
+  fi
+
+  duckdb -markdown -c "
+SELECT
+  '${f}' as source,
+  count(*) as objects,
+  sum(metadata.size::BIGINT) as total_bytes,
+  printf('%,.1f MiB', sum(metadata.size::BIGINT) / 1048576.0) as total_size,
+  max(metadata.timeCreated::TIMESTAMPTZ) as newest,
+  age(now(), max(metadata.timeCreated::TIMESTAMPTZ)) as newest_age,
+  min(metadata.timeCreated::TIMESTAMPTZ) as oldest
+FROM read_json_auto('${abs}');
+"
+done

k3d-example/fluentbit/fluent-bit.conf

@@ -25,6 +25,51 @@
     Match    kube.*
     Add      cluster ${CLUSTER_NAME}
 
+# --- CI failure detection (moved from logs-server to the pipeline) ---
+# Detect turborepo task failures on stderr.
+# rewrite_tag copies matching records to a new tag for low-latency output.
+# The original record continues through to bulk storage outputs unchanged.
+# Two rules: hard error and soft error (--continue flag).
+# Only stderr carries the canonical error signal from turborepo.
+[FILTER]
+    Name          rewrite_tag
+    Match         kube.*
+    Rule          $message ^.+command finished with error ci_failure false
+    Emitter_Name  ci_failure_emitter
+
+# Add context from the original kube tag to ci_failure records.
+# The original tag segments are lost after rewrite, but the record
+# already has 'cluster' from the modify filter above.
+[FILTER]
+    Name     modify
+    Match    ci_failure
+    Add      detection ci_task_failure
+
+# --- Low-latency output for CI failures ---
+# Separate S3 prefix for failures — small files, fast upload.
+# In production this would be a webhook or Slack output.
+[OUTPUT]
+    Name                         s3
+    Alias                        s3-ci-failures
+    Match                        ci_failure
+    bucket                       fluentbit-logs
+    endpoint                     http://versitygw:7070
+    tls                          Off
+    use_put_object               On
+    compression                  arrow-compact
+    json_date_key                false
+    total_file_size              256K
+    upload_timeout               5s
+    s3_key_format                /ci-failures/%Y/%m/%d/%H/%M/$UUID.arrow
+
+# For debugging: also log CI failures to stdout
+[OUTPUT]
+    Name                         stdout
+    Alias                        ci-failures-debug
+    Match                        ci_failure
+    Format                       json_lines
+
+# --- Bulk storage outputs (high latency to reduce GCS operations cost) ---
 # Arrow-compact: CRI 'time' as Timestamp(ns), stream+logtag dictionary-encoded
 # json_date_key=false suppresses the internal timestamp
 [OUTPUT]

k3d-example/workload/kustomization.yaml

@@ -4,3 +4,4 @@ kind: Kustomization
 
 resources:
 - workload.yaml
+- turbo-stub.yaml

k3d-example/workload/turbo-stub.yaml

@@ -0,0 +1,55 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: turbo-stub
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: turbo-stub
+    spec:
+      restartPolicy: Never
+      securityContext:
+        runAsUser: 65534
+        runAsGroup: 65534
+      containers:
+      - name: step-build
+        image: busybox:1.37
+        command:
+        - /bin/sh
+        - -c
+        args:
+        - |
+          # Simulate a turborepo build with mixed results.
+          # These patterns match detect-ci.js from checkit/logs/server.
+          sleep 2
+
+          echo "@yolean/ui:lint: cache hit, replaying output 49382abc"
+          sleep 0.1
+          echo "@yolean/ui:lint: cache hit, suppressing logs 49382abc"
+          sleep 0.1
+          echo "@yolean/bin-lib:lint: cache hit, replaying output a1b2c3d4"
+          sleep 0.5
+
+          echo "@yolean/bin-lib:build: Done in 2.1s"
+          sleep 0.1
+          echo "@yolean/ui:build: Done in 4.7s"
+          sleep 0.5
+
+          echo "@yolean/api:build: building..."
+          sleep 1
+          echo "@yolean/api:build: src/handler.ts(42,5): error TS2345: Argument of type" >&2
+          echo "@yolean/api:build: ERROR: command finished with error:" >&2
+          sleep 0.1
+          echo "@yolean/e2e:test: running..."
+          sleep 0.5
+          echo "@yolean/e2e:test: command finished with error, but continuing..." >&2
+          sleep 0.5
+
+          echo ""
+          echo " Tasks:    3 successful, 2 failed, 2 cached, 7 total"
+          echo " Failed:   @yolean/api:build, @yolean/e2e:test"
+          echo " Cached:   2 cached, 7 total"
+          echo " Time:     9.4s"
+          sleep 1