diff --git a/compare-buckets.sh b/compare-buckets.sh new file mode 100755 index 0000000..0cc708a --- /dev/null +++ b/compare-buckets.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -eo pipefail + +if [ $# -lt 1 ]; then + echo "Usage: $0 [file.json...]" + echo "Compares GCS bucket listings exported via: gcloud storage ls --json 'gs://bucket/**' > file.json" + exit 1 +fi + +for f in "$@"; do + abs=$(cd "$(dirname "$f")" && pwd)/$(basename "$f") + + if [ ! -s "$abs" ]; then + echo "-- ${f}: empty file (no objects found)" + echo "" + continue + fi + + duckdb -markdown -c " +SELECT + '${f}' as source, + count(*) as objects, + sum(metadata.size::BIGINT) as total_bytes, + printf('%,.1f MiB', sum(metadata.size::BIGINT) / 1048576.0) as total_size, + max(metadata.timeCreated::TIMESTAMPTZ) as newest, + age(now(), max(metadata.timeCreated::TIMESTAMPTZ)) as newest_age, + min(metadata.timeCreated::TIMESTAMPTZ) as oldest +FROM read_json_auto('${abs}'); +" +done diff --git a/k3d-example/fluentbit/fluent-bit.conf b/k3d-example/fluentbit/fluent-bit.conf index 91b17c2..2b1b739 100644 --- a/k3d-example/fluentbit/fluent-bit.conf +++ b/k3d-example/fluentbit/fluent-bit.conf @@ -25,6 +25,51 @@ Match kube.* Add cluster ${CLUSTER_NAME} +# --- CI failure detection (moved from logs-server to the pipeline) --- +# Detect turborepo task failures on stderr. +# rewrite_tag copies matching records to a new tag for low-latency output. +# The original record continues through to bulk storage outputs unchanged. +# Two rules: hard error and soft error (--continue flag). +# Only stderr carries the canonical error signal from turborepo. +[FILTER] + Name rewrite_tag + Match kube.* + Rule $message ^.+command finished with error ci_failure false + Emitter_Name ci_failure_emitter + +# Add context from the original kube tag to ci_failure records. +# The original tag segments are lost after rewrite, but the record +# already has 'cluster' from the modify filter above. +[FILTER] + Name modify + Match ci_failure + Add detection ci_task_failure + +# --- Low-latency output for CI failures --- +# Separate S3 prefix for failures — small files, fast upload. +# In production this would be a webhook or Slack output. +[OUTPUT] + Name s3 + Alias s3-ci-failures + Match ci_failure + bucket fluentbit-logs + endpoint http://versitygw:7070 + tls Off + use_put_object On + compression arrow-compact + json_date_key false + total_file_size 256K + upload_timeout 5s + s3_key_format /ci-failures/%Y/%m/%d/%H/%M/$UUID.arrow + +# For debugging: also log CI failures to stdout +[OUTPUT] + Name stdout + Alias ci-failures-debug + Match ci_failure + Format json_lines + +# --- Bulk storage outputs (high latency to reduce GCS operations cost) --- # Arrow-compact: CRI 'time' as Timestamp(ns), stream+logtag dictionary-encoded # json_date_key=false suppresses the internal timestamp [OUTPUT] diff --git a/k3d-example/workload/kustomization.yaml b/k3d-example/workload/kustomization.yaml index 2b3e638..11d0173 100644 --- a/k3d-example/workload/kustomization.yaml +++ b/k3d-example/workload/kustomization.yaml @@ -4,3 +4,4 @@ kind: Kustomization resources: - workload.yaml +- turbo-stub.yaml diff --git a/k3d-example/workload/turbo-stub.yaml b/k3d-example/workload/turbo-stub.yaml new file mode 100644 index 0000000..5e0e9f0 --- /dev/null +++ b/k3d-example/workload/turbo-stub.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: turbo-stub +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: turbo-stub + spec: + restartPolicy: Never + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + containers: + - name: step-build + image: busybox:1.37 + command: + - /bin/sh + - -c + args: + - | + # Simulate a turborepo build with mixed results. + # These patterns match detect-ci.js from checkit/logs/server. + sleep 2 + + echo "@yolean/ui:lint: cache hit, replaying output 49382abc" + sleep 0.1 + echo "@yolean/ui:lint: cache hit, suppressing logs 49382abc" + sleep 0.1 + echo "@yolean/bin-lib:lint: cache hit, replaying output a1b2c3d4" + sleep 0.5 + + echo "@yolean/bin-lib:build: Done in 2.1s" + sleep 0.1 + echo "@yolean/ui:build: Done in 4.7s" + sleep 0.5 + + echo "@yolean/api:build: building..." + sleep 1 + echo "@yolean/api:build: src/handler.ts(42,5): error TS2345: Argument of type" >&2 + echo "@yolean/api:build: ERROR: command finished with error:" >&2 + sleep 0.1 + echo "@yolean/e2e:test: running..." + sleep 0.5 + echo "@yolean/e2e:test: command finished with error, but continuing..." >&2 + sleep 0.5 + + echo "" + echo " Tasks: 3 successful, 2 failed, 2 cached, 7 total" + echo " Failed: @yolean/api:build, @yolean/e2e:test" + echo " Cached: 2 cached, 7 total" + echo " Time: 9.4s" + sleep 1