diff --git a/Cargo.lock b/Cargo.lock index ff31a11..3582156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,6 +760,22 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "ljd" +version = "0.1.0" +dependencies = [ + "logjet", + "opentelemetry-proto", + "prost", + "rustls", + "rustls-pemfile", + "serde", + "serde_yaml", + "tiny_http", + "tokio", + "tonic", +] + [[package]] name = "ljx" version = "0.1.0" @@ -801,22 +817,6 @@ dependencies = [ "prost", ] -[[package]] -name = "logjetd" -version = "0.1.0" -dependencies = [ - "logjet", - "opentelemetry-proto", - "prost", - "rustls", - "rustls-pemfile", - "serde", - "serde_yaml", - "tiny_http", - "tokio", - "tonic", -] - [[package]] name = "lru" version = "0.12.5" diff --git a/Makefile b/Makefile index 551660b..25bffe4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ fix: setup cargo clippy $(CORE_WORKSPACE) --fix --all-targets --all-features --allow-dirty --allow-staged -- -D warnings test: setup - cargo build -p logjetd -p ljx + cargo build -p ljd -p ljx cargo build -p otlp-demo --bin otlp-bofh-emitter @if command -v cargo-nextest >/dev/null 2>&1; then \ cargo nextest run $(CORE_WORKSPACE); \ @@ -33,23 +33,23 @@ test: setup test-unit: setup @if command -v cargo-nextest >/dev/null 2>&1; then \ - cargo nextest run -p logjet --lib -p logjetd --bins -p ljx --bin ljx; \ + cargo nextest run -p logjet --lib -p ljd --bins -p ljx --bin ljx; \ else \ echo "cargo-nextest not available, falling back to cargo test unit-only targets"; \ cargo test -p logjet --lib; \ - cargo test -p logjetd --bin logjetd; \ + cargo test -p ljd --bin ljd; \ cargo test -p ljx --bin ljx; \ fi test-integration: setup - cargo build -p logjetd -p ljx + cargo build -p ljd -p ljx cargo build -p otlp-demo --bin otlp-bofh-emitter @if command -v cargo-nextest >/dev/null 2>&1; then \ - cargo nextest run -p logjetd --test bridge_flows; \ + cargo nextest run -p ljd --test bridge_flows; \ cargo nextest run -p logjet --test ljx_cli; \ else \ echo "cargo-nextest not available, falling back to cargo test integration targets"; \ - cargo test -p logjetd --test bridge_flows; \ + cargo test -p ljd --test bridge_flows; \ cargo test -p logjet --test ljx_cli; \ fi diff --git a/demo/README.md b/demo/README.md index 72248fd..69c7b14 100644 --- a/demo/README.md +++ b/demo/README.md @@ -18,9 +18,9 @@ This directory contains two binaries for a two-terminal OTLP demo over TCP. It also contains scenario demos under subdirectories: - [`logjet-file`](./logjet-file) - - OTLP/HTTP emitter into file-backed `logjetd` + - OTLP/HTTP emitter into file-backed `ljd` - [`logjet-grpc-file`](./logjet-grpc-file) - - OTLP/gRPC emitter into file-backed `logjetd` + - OTLP/gRPC emitter into file-backed `ljd` - [`kill-bill`](./kill-bill) - cut a `.logjet` file down to its middle third and recover later good blocks - [`memory-buffer`](./memory-buffer) @@ -28,9 +28,9 @@ It also contains scenario demos under subdirectories: - [`drain-once`](./drain-once) - preserved startup messages are consumed on the first drain and do not appear on the second - [`multi-emitter`](./multi-emitter) - - five emitters into one `logjetd`, then late replay into one collector + - five emitters into one `ljd`, then late replay into one collector - [`multi-emitter-continuous`](./multi-emitter-continuous) - - five emitters running continuously into one `logjetd` and one live collector + - five emitters running continuously into one `ljd` and one live collector - [`multi-client-behaviour`](./multi-client-behaviour) - one replay client stalls while another keeps flowing - [`replay-handoff`](./replay-handoff) @@ -52,11 +52,11 @@ It also contains scenario demos under subdirectories: - [`ingest-overload`](./ingest-overload) - rate-limited ingest with operator-visible counters and severity-aware shedding - [`remote-drain`](./remote-drain) - - appliance-side `logjetd` drained by a remote-side `logjetd bridge` + - appliance-side `ljd` drained by a remote-side `ljd bridge` - [`remote-drain-tls`](./remote-drain-tls) - same remote-drain topology, but with TLS and mutual TLS on the replay link - [`secure-pipeline`](./secure-pipeline) - - HTTPS OTLP ingest into `logjetd`, then HTTPS collector export on replay + - HTTPS OTLP ingest into `ljd`, then HTTPS collector export on replay ## Enjoy It diff --git a/demo/backpressure/README.md b/demo/backpressure/README.md index 7de9529..f83ade5 100644 --- a/demo/backpressure/README.md +++ b/demo/backpressure/README.md @@ -35,7 +35,7 @@ From this directory: This starts: -1. appliance-side `logjetd` +1. appliance-side `ljd` 2. a fast continuous emitter The emitter sends every 200 ms. @@ -118,7 +118,7 @@ Expected result: - the collector still prints slowly - the bridge keeps forwarding whatever fits into the three-record queue - newer records are dropped once the queue is full -- terminal output from `logjetd` shows dropped sequence numbers +- terminal output from `ljd` shows dropped sequence numbers ## Point of the Demo diff --git a/demo/backpressure/run-appliance.sh b/demo/backpressure/run-appliance.sh index c890f65..88bc590 100755 --- a/demo/backpressure/run-appliance.sh +++ b/demo/backpressure/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,17 +17,17 @@ done cd "$SCRIPT_DIR" -echo "starting appliance-side logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting appliance-side ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM sleep 1 -echo "starting fast BOFH traffic toward appliance-side logjetd" +echo "starting fast BOFH traffic toward appliance-side ljd" "$EMITTER" 127.0.0.1:4318 --service-name "backpressure-emitter" --interval-ms 200 diff --git a/demo/backpressure/run-consumer-block.sh b/demo/backpressure/run-consumer-block.sh index 6d800ff..a45dd31 100755 --- a/demo/backpressure/run-consumer-block.sh +++ b/demo/backpressure/run-consumer-block.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/consumer-block.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting consumer-side bridge in block mode using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/backpressure/run-consumer-disconnect.sh b/demo/backpressure/run-consumer-disconnect.sh index d518fcc..723ffe5 100755 --- a/demo/backpressure/run-consumer-disconnect.sh +++ b/demo/backpressure/run-consumer-disconnect.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/consumer-disconnect.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting consumer-side bridge in disconnect mode using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/backpressure/run-consumer-drop-newest.sh b/demo/backpressure/run-consumer-drop-newest.sh index 81c80f5..4094d7f 100755 --- a/demo/backpressure/run-consumer-drop-newest.sh +++ b/demo/backpressure/run-consumer-drop-newest.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/consumer-drop-newest.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting consumer-side bridge in drop-newest mode using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/bridge-resume/README.md b/demo/bridge-resume/README.md index d6b618e..8acc5e2 100644 --- a/demo/bridge-resume/README.md +++ b/demo/bridge-resume/README.md @@ -3,7 +3,7 @@ This demo shows the point of `upstream.state-file`. The appliance side keeps producing logs. The consumer side can die and come -back. When it returns, `logjetd bridge` resumes from the last successfully +back. When it returns, `ljd bridge` resumes from the last successfully forwarded sequence instead of replaying from zero. That means: @@ -21,13 +21,13 @@ appliance side. It is written on the downstream consumer side. The split is: -- appliance-side `logjetd` +- appliance-side `ljd` - receives OTLP - keeps backlog in memory - exposes a replay listener - does not need to write a bridge checkpoint file -- consumer-side `logjetd bridge` +- consumer-side `ljd bridge` - connects to the appliance replay listener - forwards logs to the collector - stores the last forwarded sequence in `upstream.state-file` @@ -53,7 +53,7 @@ From this directory: This starts: -1. appliance-side `logjetd` +1. appliance-side `ljd` 2. a dialogue-style message emitter The appliance side uses in-memory retention only. @@ -72,7 +72,7 @@ From this directory: This starts: 1. the collector -2. consumer-side `logjetd bridge` +2. consumer-side `ljd bridge` The consumer config contains: diff --git a/demo/bridge-resume/run-appliance.sh b/demo/bridge-resume/run-appliance.sh index dfb0f1b..64a4601 100755 --- a/demo/bridge-resume/run-appliance.sh +++ b/demo/bridge-resume/run-appliance.sh @@ -3,12 +3,12 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" STATE_FILE="$SCRIPT_DIR/bridge.state" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -20,12 +20,12 @@ cd "$SCRIPT_DIR" rm -f "$STATE_FILE" -echo "starting appliance-side logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting appliance-side ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -45,7 +45,7 @@ dialogue_line() { esac } -echo "starting dialogue traffic toward appliance-side logjetd" +echo "starting dialogue traffic toward appliance-side ljd" seq_no=1 while :; do diff --git a/demo/bridge-resume/run-consumer.sh b/demo/bridge-resume/run-consumer.sh index 4a2bd6c..d62e9e8 100755 --- a/demo/bridge-resume/run-consumer.sh +++ b/demo/bridge-resume/run-consumer.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/consumer-logjetd.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting consumer-side bridge using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/drain-once/README.md b/demo/drain-once/README.md index 58197aa..02da682 100644 --- a/demo/drain-once/README.md +++ b/demo/drain-once/README.md @@ -24,7 +24,7 @@ From this directory: This starts: -1. appliance-side `logjetd` +1. appliance-side `ljd` 2. three preserved startup messages: - `BOOT MESSAGE #1` - `BOOT MESSAGE #2` diff --git a/demo/drain-once/run-appliance.sh b/demo/drain-once/run-appliance.sh index 092a5ad..efd3755 100755 --- a/demo/drain-once/run-appliance.sh +++ b/demo/drain-once/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,12 +17,12 @@ done cd "$SCRIPT_DIR" -echo "starting appliance-side logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting appliance-side ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -39,5 +39,5 @@ for i in 4 5 6; do "$EMITTER" 127.0.0.1:4318 --once --message "BOOT MESSAGE #$i" done -echo "starting continuous BOFH traffic toward appliance-side logjetd" +echo "starting continuous BOFH traffic toward appliance-side ljd" "$EMITTER" 127.0.0.1:4318 --interval-ms 700 diff --git a/demo/drain-once/run-drain-once.sh b/demo/drain-once/run-drain-once.sh index aaed31e..3123642 100755 --- a/demo/drain-once/run-drain-once.sh +++ b/demo/drain-once/run-drain-once.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/remote-logjetd.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -35,7 +35,7 @@ run_pass() { echo echo "===== $label =====" echo "starting bridge in drain mode using $CONFIG" - "$LOGJETD" --config "$CONFIG" bridge & + "$LJD" --config "$CONFIG" bridge & BRIDGE_PID=$! sleep 4 kill "$BRIDGE_PID" 2>/dev/null || true diff --git a/demo/file-replay/README.md b/demo/file-replay/README.md index e4c2b64..6e633f6 100644 --- a/demo/file-replay/README.md +++ b/demo/file-replay/README.md @@ -13,7 +13,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-demo-collector` @@ -45,7 +45,7 @@ From this directory: The script: 1. starts the OTLP collector mockup -2. runs `logjetd --config ./logjetd.conf replay --path ./logs --name bofh.logjet` +2. runs `ljd --config ./logjetd.conf replay --path ./logs --name bofh.logjet` 3. blasts all recorded OTLP batches to the collector Expected result: diff --git a/demo/file-replay/run-demo.sh b/demo/file-replay/run-demo.sh index 380d608..4a97e10 100755 --- a/demo/file-replay/run-demo.sh +++ b/demo/file-replay/run-demo.sh @@ -4,7 +4,7 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" COLLECTOR="$TARGET_DIR/otlp-demo-collector" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" CONFIG="$SCRIPT_DIR/logjetd.conf" if [ ! -x "$COLLECTOR" ]; then @@ -13,8 +13,8 @@ if [ ! -x "$COLLECTOR" ]; then exit 1 fi -if [ ! -x "$LOGJETD" ]; then - echo "missing $LOGJETD" +if [ ! -x "$LJD" ]; then + echo "missing $LJD" echo "build it first with: make demo" exit 1 fi @@ -43,4 +43,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "replaying bofh.logjet from ./logs using collector.url from $CONFIG" -"$LOGJETD" --config "$CONFIG" replay --path "./logs" --name "bofh.logjet" +"$LJD" --config "$CONFIG" replay --path "./logs" --name "bofh.logjet" diff --git a/demo/file-tooling/README.md b/demo/file-tooling/README.md index c2034eb..7e8ad5e 100644 --- a/demo/file-tooling/README.md +++ b/demo/file-tooling/README.md @@ -2,8 +2,8 @@ This demo shows the operational file-management commands for file mode: -- `logjetd segments` -- `logjetd prune` +- `ljd segments` +- `ljd prune` ## Point @@ -34,9 +34,9 @@ From this directory: The script: -1. starts file-backed `logjetd` +1. starts file-backed `ljd` 2. sends enough OTLP batches to rotate several `ops*.logjet` files -3. shows the current segment layout with `logjetd segments` +3. shows the current segment layout with `ljd segments` 4. previews pruning by file count 5. previews pruning by byte budget 6. prunes oldest segments and keeps only the newest two files @@ -47,25 +47,25 @@ The script: List the rotated spool: ```bash -../../target/debug/logjetd segments --path ./logs --name ops.logjet +../../target/debug/ljd segments --path ./logs --name ops.logjet ``` Preview pruning by file count: ```bash -../../target/debug/logjetd prune --path ./logs --name ops.logjet --keep-files 2 --dry-run +../../target/debug/ljd prune --path ./logs --name ops.logjet --keep-files 2 --dry-run ``` Preview pruning by byte budget: ```bash -../../target/debug/logjetd prune --path ./logs --name ops.logjet --keep-bytes 2048 --dry-run +../../target/debug/ljd prune --path ./logs --name ops.logjet --keep-bytes 2048 --dry-run ``` Prune for real: ```bash -../../target/debug/logjetd prune --path ./logs --name ops.logjet --keep-files 2 +../../target/debug/ljd prune --path ./logs --name ops.logjet --keep-files 2 ``` ## Expected Result diff --git a/demo/file-tooling/run-demo.sh b/demo/file-tooling/run-demo.sh index 7029835..ed3099d 100755 --- a/demo/file-tooling/run-demo.sh +++ b/demo/file-tooling/run-demo.sh @@ -2,15 +2,15 @@ set -eu ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" -LOGJETD="$ROOT_DIR/target/debug/logjetd" +LJD="$ROOT_DIR/target/debug/ljd" EMITTER="$ROOT_DIR/target/debug/otlp-bofh-emitter" CONFIG="$(cd "$(dirname "$0")" && pwd)/logjetd.conf" LOG_DIR="$(cd "$(dirname "$0")" && pwd)/logs" cleanup() { - if [ -n "${LOGJETD_PID:-}" ]; then - kill "$LOGJETD_PID" >/dev/null 2>&1 || true - wait "$LOGJETD_PID" 2>/dev/null || true + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" >/dev/null 2>&1 || true + wait "$LJD_PID" 2>/dev/null || true fi } @@ -19,9 +19,9 @@ trap cleanup EXIT mkdir -p "$LOG_DIR" rm -f "$LOG_DIR"/ops*.logjet "$LOG_DIR"/ops*.state "$LOG_DIR"/ops*.stream-id -echo "starting file-backed logjetd" -"$LOGJETD" --config "$CONFIG" serve & -LOGJETD_PID=$! +echo "starting file-backed ljd" +"$LJD" --config "$CONFIG" serve & +LJD_PID=$! sleep 1 echo "sending enough messages to force file rotation" @@ -31,20 +31,20 @@ sleep 1 echo echo "current segment layout" -"$LOGJETD" segments --path "$LOG_DIR" --name ops.logjet +"$LJD" segments --path "$LOG_DIR" --name ops.logjet echo echo "dry-run prune by file count" -"$LOGJETD" prune --path "$LOG_DIR" --name ops.logjet --keep-files 2 --dry-run +"$LJD" prune --path "$LOG_DIR" --name ops.logjet --keep-files 2 --dry-run echo echo "dry-run prune by byte budget" -"$LOGJETD" prune --path "$LOG_DIR" --name ops.logjet --keep-bytes 2048 --dry-run +"$LJD" prune --path "$LOG_DIR" --name ops.logjet --keep-bytes 2048 --dry-run echo echo "pruning oldest segments and keeping only the newest two files" -"$LOGJETD" prune --path "$LOG_DIR" --name ops.logjet --keep-files 2 +"$LJD" prune --path "$LOG_DIR" --name ops.logjet --keep-files 2 echo echo "segment layout after prune" -"$LOGJETD" segments --path "$LOG_DIR" --name ops.logjet +"$LJD" segments --path "$LOG_DIR" --name ops.logjet diff --git a/demo/ingest-guardrails/README.md b/demo/ingest-guardrails/README.md index 3223939..3397000 100644 --- a/demo/ingest-guardrails/README.md +++ b/demo/ingest-guardrails/README.md @@ -1,6 +1,6 @@ # Ingest Guardrails Demo -This demo shows the two ingest guardrails that protect `logjetd` on weak +This demo shows the two ingest guardrails that protect `ljd` on weak appliances: - `ingest.max-batch-bytes` @@ -36,7 +36,7 @@ ingest.max-batch-bytes: 300 What happens: -1. `logjetd` starts with a tiny OTLP batch limit. +1. `ljd` starts with a tiny OTLP batch limit. 2. A small OTLP batch is sent and accepted. 3. A large OTLP batch is sent and rejected. @@ -76,14 +76,14 @@ Expected result: ## Point of the Demo -This shows that `logjetd` can now reject bad overload patterns early: +This shows that `ljd` can now reject bad overload patterns early: - too large - too many at once The concurrent-client part uses plain TCP. That means the refused client can still complete `connect()` and may even get its first write into the local -socket buffer before `logjetd` closes the connection. The important signal is +socket buffer before `ljd` closes the connection. The important signal is that the connection does not stay alive for the second record. That is only the first overload-control layer. It does not yet rate-limit, diff --git a/demo/ingest-guardrails/run-max-clients.sh b/demo/ingest-guardrails/run-max-clients.sh index e40dd95..69e6e94 100755 --- a/demo/ingest-guardrails/run-max-clients.sh +++ b/demo/ingest-guardrails/run-max-clients.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" WIRE_EMITTER="$TARGET_DIR/wire-hold-emitter" CONFIG="$SCRIPT_DIR/wire-limit.conf" -for bin in "$LOGJETD" "$WIRE_EMITTER"; do +for bin in "$LJD" "$WIRE_EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -20,14 +20,14 @@ cd "$SCRIPT_DIR" cleanup() { kill "${FIRST_PID:-}" 2>/dev/null || true kill "${SECOND_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM -echo "starting logjetd with ingest.max-clients: 1" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with ingest.max-clients: 1" +"$LJD" --config "$CONFIG" & +LJD_PID=$! sleep 1 diff --git a/demo/ingest-guardrails/run-oversize.sh b/demo/ingest-guardrails/run-oversize.sh index 35df90b..f24fea2 100755 --- a/demo/ingest-guardrails/run-oversize.sh +++ b/demo/ingest-guardrails/run-oversize.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/http-limit.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -18,14 +18,14 @@ done cd "$SCRIPT_DIR" cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM -echo "starting logjetd with tiny ingest.max-batch-bytes" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with tiny ingest.max-batch-bytes" +"$LJD" --config "$CONFIG" & +LJD_PID=$! sleep 1 diff --git a/demo/ingest-overload/README.md b/demo/ingest-overload/README.md index 75b8eb0..124cb35 100644 --- a/demo/ingest-overload/README.md +++ b/demo/ingest-overload/README.md @@ -14,7 +14,7 @@ The intended effect is: - the first few `WARN` batches fit under the rate limit - later `WARN` batches are rejected while the overload window is full - `ERROR` batches still get through during that same overload window -- `logjetd` prints overload counters for operator visibility +- `ljd` prints overload counters for operator visibility ## Build First @@ -32,7 +32,7 @@ From this directory: ./run-appliance.sh ``` -This starts `logjetd`, then sends: +This starts `ljd`, then sends: 1. a fast `WARN` burst from `service=overload-warn` 2. a fast `ERROR` burst from `service=overload-error` @@ -42,10 +42,10 @@ Expected appliance-side output: - many `WARN` sends fail with `HTTP/1.1 429 Too Many Requests` - `ERROR` sends succeed even though the daemon is already overloaded -- `logjetd` prints lines such as: +- `ljd` prints lines such as: ```text -logjetd ingest overload stats accepted=... priority-bypass=... rate-limited=... +ljd ingest overload stats accepted=... priority-bypass=... rate-limited=... ``` ## Terminal 2: Consumer Side @@ -59,7 +59,7 @@ From this directory: This starts: 1. the OTel Collector mock -2. the wire forwarder from `logjetd` replay into the collector +2. the wire forwarder from `ljd` replay into the collector Expected collector output: diff --git a/demo/ingest-overload/run-appliance.sh b/demo/ingest-overload/run-appliance.sh index f1b78f5..e550c3b 100755 --- a/demo/ingest-overload/run-appliance.sh +++ b/demo/ingest-overload/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,12 +17,12 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with ingest overload policy from $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with ingest overload policy from $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -39,4 +39,4 @@ echo "sending another WARN burst that should still be shed" "$EMITTER" 127.0.0.1:4318 --service-name overload-warn --severity warn --count 5 --interval-ms 0 echo "appliance side stays up so a consumer can drain retained records" -wait "$LOGJETD_PID" +wait "$LJD_PID" diff --git a/demo/kill-bill/README.md b/demo/kill-bill/README.md index daf92e6..620d66b 100644 --- a/demo/kill-bill/README.md +++ b/demo/kill-bill/README.md @@ -29,9 +29,9 @@ From this directory: The script does this: -1. starts `logjetd` in file mode +1. starts `ljd` in file mode 2. writes 100 numbered OTLP log messages into one `killbill.logjet` file -3. stops `logjetd` +3. stops `ljd` 4. cuts out the middle third of the file by raw bytes 5. saves only that byte slice as `./damaged/killbill.logjet` 6. inspects the original file diff --git a/demo/kill-bill/run-demo.sh b/demo/kill-bill/run-demo.sh index d2432b6..7dd4e7d 100755 --- a/demo/kill-bill/run-demo.sh +++ b/demo/kill-bill/run-demo.sh @@ -3,7 +3,7 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/logjetd.conf" @@ -12,7 +12,7 @@ DAMAGED_DIR="$SCRIPT_DIR/damaged" ORIGINAL_FILE="$LOG_DIR/killbill.logjet" DAMAGED_FILE="$DAMAGED_DIR/killbill.logjet" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -24,7 +24,7 @@ cd "$SCRIPT_DIR" cleanup() { kill "${COLLECTOR_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -33,9 +33,9 @@ echo "cleaning previous demo files" rm -rf "$LOG_DIR" "$DAMAGED_DIR" mkdir -p "$LOG_DIR" "$DAMAGED_DIR" -echo "starting logjetd to write one .logjet file with 100 messages" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd to write one .logjet file with 100 messages" +"$LJD" --config "$CONFIG" & +LJD_PID=$! sleep 1 @@ -47,9 +47,9 @@ while [ "$i" -le 100 ]; do done sleep 1 -kill "$LOGJETD_PID" -wait "$LOGJETD_PID" 2>/dev/null || true -unset LOGJETD_PID +kill "$LJD_PID" +wait "$LJD_PID" 2>/dev/null || true +unset LJD_PID if [ ! -f "$ORIGINAL_FILE" ]; then echo "expected $ORIGINAL_FILE to exist" @@ -71,11 +71,11 @@ fi echo echo "inspecting original file" -"$LOGJETD" inspect "$ORIGINAL_FILE" +"$LJD" inspect "$ORIGINAL_FILE" echo echo "inspecting damaged middle-third file" -"$LOGJETD" inspect "$DAMAGED_FILE" +"$LJD" inspect "$DAMAGED_FILE" echo echo "starting collector on 127.0.0.1:4321" @@ -86,10 +86,10 @@ sleep 1 echo echo "replaying only the damaged middle-third file" -"$LOGJETD" --config "$CONFIG" replay --path "$DAMAGED_DIR" --name "killbill.logjet" +"$LJD" --config "$CONFIG" replay --path "$DAMAGED_DIR" --name "killbill.logjet" echo echo "point of the demo:" echo "- the first bytes of the damaged file are not a valid file start" -echo "- logjetd scans forward until it finds the next block sync marker" +echo "- ljd scans forward until it finds the next block sync marker" echo "- records in surviving later blocks are still replayed" diff --git a/demo/logjet-file/README.md b/demo/logjet-file/README.md index 3d0831a..be10c82 100644 --- a/demo/logjet-file/README.md +++ b/demo/logjet-file/README.md @@ -1,6 +1,6 @@ # logjet File Demo -This demo wires the OTLP emitter into `logjetd`, with `logjetd` storing +This demo wires the OTLP emitter into `ljd`, with `ljd` storing raw OTLP protobuf batches into `.logjet` files in this directory. Emitter emits classic BOFH excuses as log messages. :-) @@ -15,7 +15,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-emitter` ## Run @@ -28,7 +28,7 @@ From this directory: The script: -1. starts `logjetd` with the local config file +1. starts `ljd` with the local config file 2. points it at this directory for append-only `.logjet` output 3. starts `otlp-bofh-emitter` 4. keeps the emitter in the foreground @@ -45,7 +45,7 @@ etc. While the demo is running or after stopping it: ```bash -../../target/debug/logjetd inspect . +../../target/debug/ljd inspect . ``` ## Notes diff --git a/demo/logjet-file/run-demo.sh b/demo/logjet-file/run-demo.sh index 44cfc5a..dd9129d 100755 --- a/demo/logjet-file/run-demo.sh +++ b/demo/logjet-file/run-demo.sh @@ -3,12 +3,12 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -if [ ! -x "$LOGJETD" ]; then - echo "missing $LOGJETD" +if [ ! -x "$LJD" ]; then + echo "missing $LJD" echo "build it first with: make demo" exit 1 fi @@ -21,12 +21,12 @@ fi cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "$LOGJETD_PID" 2>/dev/null || true + kill "$LJD_PID" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/logjet-grpc-file/README.md b/demo/logjet-grpc-file/README.md index 60cf195..2973061 100644 --- a/demo/logjet-grpc-file/README.md +++ b/demo/logjet-grpc-file/README.md @@ -1,6 +1,6 @@ # logjet gRPC File Demo -This demo wires the OTLP gRPC emitter into `logjetd`, with `logjetd` storing +This demo wires the OTLP gRPC emitter into `ljd`, with `ljd` storing raw OTLP protobuf batches into `.logjet` files in this directory. Emitter emits classic BOFH excuses as log messages over OTLP/gRPC. @@ -15,7 +15,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-grpc-emitter` ## Run @@ -28,7 +28,7 @@ From this directory: The script: -1. starts `logjetd` with the local config file +1. starts `ljd` with the local config file 2. points it at `./logs` for append-only `.logjet` output 3. starts `otlp-bofh-grpc-emitter` 4. keeps the emitter in the foreground @@ -43,7 +43,7 @@ Generated files appear under `./logs`, for example: While the demo is running or after stopping it: ```bash -../../target/debug/logjetd inspect ./logs +../../target/debug/ljd inspect ./logs ``` ## Notes diff --git a/demo/logjet-grpc-file/run-demo.sh b/demo/logjet-grpc-file/run-demo.sh index 59a4eb5..9bf66ab 100755 --- a/demo/logjet-grpc-file/run-demo.sh +++ b/demo/logjet-grpc-file/run-demo.sh @@ -3,12 +3,12 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-grpc-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -if [ ! -x "$LOGJETD" ]; then - echo "missing $LOGJETD" +if [ ! -x "$LJD" ]; then + echo "missing $LJD" echo "build it first with: make demo" exit 1 fi @@ -21,12 +21,12 @@ fi cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "$LOGJETD_PID" 2>/dev/null || true + kill "$LJD_PID" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/memory-buffer/README.md b/demo/memory-buffer/README.md index 443d2b5..4d053e4 100644 --- a/demo/memory-buffer/README.md +++ b/demo/memory-buffer/README.md @@ -18,7 +18,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-emitter` - `target/debug/otlp-demo-collector` - `target/debug/otlp-wire-forwarder` @@ -33,7 +33,7 @@ From this directory: The script: -1. starts `logjetd` in memory mode with: +1. starts `ljd` in memory mode with: - `buffer.keep: 3` - `buffer.messages: 10` 2. emits 10 startup messages: @@ -42,7 +42,7 @@ The script: - `this message #10 must be kept` 3. emits 100 BOFH flood messages with no delay 4. starts the OTLP collector mockup -5. connects a wire forwarder to `logjetd` replay +5. connects a wire forwarder to `ljd` replay 6. forwards exactly 13 messages to the collector Expected result: @@ -96,6 +96,6 @@ Total retained and forwarded: Why the config says `buffer.messages: 12`: -- in `logjetd`, `buffer.messages` applies only to the rotating tail +- in `ljd`, `buffer.messages` applies only to the rotating tail - the first 3 kept messages are outside that limit - so `3 kept + 12 tail = 15 total visible messages` diff --git a/demo/memory-buffer/run-demo-memshow.sh b/demo/memory-buffer/run-demo-memshow.sh index eb1b00d..3516a35 100755 --- a/demo/memory-buffer/run-demo-memshow.sh +++ b/demo/memory-buffer/run-demo-memshow.sh @@ -3,13 +3,13 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" FORWARDER="$TARGET_DIR/otlp-wire-forwarder" -CONFIG="$SCRIPT_DIR/logjetd-memshow.conf" +CONFIG="$SCRIPT_DIR/ljd-memshow.conf" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -19,14 +19,14 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { kill "${FORWARDER_PID:-}" 2>/dev/null || true kill "${COLLECTOR_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/memory-buffer/run-demo.sh b/demo/memory-buffer/run-demo.sh index 04a2027..1776be6 100755 --- a/demo/memory-buffer/run-demo.sh +++ b/demo/memory-buffer/run-demo.sh @@ -3,13 +3,13 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" FORWARDER="$TARGET_DIR/otlp-wire-forwarder" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -19,14 +19,14 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { kill "${FORWARDER_PID:-}" 2>/dev/null || true kill "${COLLECTOR_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/multi-client-behaviour/README.md b/demo/multi-client-behaviour/README.md index c8572ab..10bf388 100644 --- a/demo/multi-client-behaviour/README.md +++ b/demo/multi-client-behaviour/README.md @@ -24,7 +24,7 @@ From this directory: This starts: -1. appliance-side `logjetd` +1. appliance-side `ljd` 2. a steady message stream The replay listener is configured with: @@ -73,7 +73,7 @@ While the stalled client is running: - Terminal 2 keeps printing new `MULTI` messages - Terminal 3 reports that it received one record and then stalled -- appliance-side `logjetd` eventually logs a replay client error caused by timeout +- appliance-side `ljd` eventually logs a replay client error caused by timeout That is the point: diff --git a/demo/multi-client-behaviour/run-appliance.sh b/demo/multi-client-behaviour/run-appliance.sh index ebcb520..cc898b2 100755 --- a/demo/multi-client-behaviour/run-appliance.sh +++ b/demo/multi-client-behaviour/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,12 +17,12 @@ done cd "$SCRIPT_DIR" -echo "starting appliance-side logjetd with replay.client-timeout-ms=3000" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting appliance-side ljd with replay.client-timeout-ms=3000" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/multi-emitter-continuous/README.md b/demo/multi-emitter-continuous/README.md index 16df92e..f9b4843 100644 --- a/demo/multi-emitter-continuous/README.md +++ b/demo/multi-emitter-continuous/README.md @@ -6,9 +6,9 @@ the same time. It uses: - five OTLP/HTTP emitters running continuously -- one `logjetd` +- one `ljd` - one OTLP collector mockup -- one wire forwarder from the `logjetd` replay listener into the collector +- one wire forwarder from the `ljd` replay listener into the collector Each emitter identifies itself through `service.name`: @@ -38,7 +38,7 @@ Terminal 1, from this directory: This starts: -1. starts `logjetd` +1. starts `ljd` 2. starts five continuous emitters with different service names Terminal 2, from this directory: @@ -68,6 +68,6 @@ Stop it with `Ctrl+C`. This demonstrates: -- several emitters can connect to one `logjetd` at the same time -- `logjetd` merges their records into one retained stream +- several emitters can connect to one `ljd` at the same time +- `ljd` merges their records into one retained stream - one downstream consumer can watch that mixed stream live diff --git a/demo/multi-emitter-continuous/run-demo.sh b/demo/multi-emitter-continuous/run-demo.sh index 4a85f4b..b8010a4 100755 --- a/demo/multi-emitter-continuous/run-demo.sh +++ b/demo/multi-emitter-continuous/run-demo.sh @@ -3,13 +3,13 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" FORWARDER="$TARGET_DIR/otlp-wire-forwarder" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -19,9 +19,9 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { kill "${ALICE_PID:-}" 2>/dev/null || true @@ -31,7 +31,7 @@ cleanup() { kill "${EVE_PID:-}" 2>/dev/null || true kill "${FORWARDER_PID:-}" 2>/dev/null || true kill "${COLLECTOR_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/multi-emitter-continuous/run-emitters.sh b/demo/multi-emitter-continuous/run-emitters.sh index 3629e16..1ae396e 100755 --- a/demo/multi-emitter-continuous/run-emitters.sh +++ b/demo/multi-emitter-continuous/run-emitters.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,9 +17,9 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { kill "${ALICE_PID:-}" 2>/dev/null || true @@ -27,7 +27,7 @@ cleanup() { kill "${CAROL_PID:-}" 2>/dev/null || true kill "${DAVE_PID:-}" 2>/dev/null || true kill "${EVE_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -48,6 +48,6 @@ start_emitter CAROL 1100 start_emitter DAVE 1300 start_emitter EVE 1500 -echo "logjetd and five emitters are running; start ./run-consumer.sh in another terminal" +echo "ljd and five emitters are running; start ./run-consumer.sh in another terminal" echo "press Ctrl+C here to stop the emitters side" wait diff --git a/demo/multi-emitter/README.md b/demo/multi-emitter/README.md index 2b5bf5d..4e7f01b 100644 --- a/demo/multi-emitter/README.md +++ b/demo/multi-emitter/README.md @@ -1,14 +1,14 @@ # Multi-Emitter Demo -This demo shows that several emitters can send OTLP logs into one `logjetd`, +This demo shows that several emitters can send OTLP logs into one `ljd`, and one downstream consumer can receive the merged retained stream later. The demo uses: - five separate OTLP/HTTP emitters -- one `logjetd` in memory-buffer mode +- one `ljd` in memory-buffer mode - one OTLP collector mockup -- one wire forwarder from the `logjetd` replay listener into the collector +- one wire forwarder from the `ljd` replay listener into the collector Each emitter sends one identifying message: @@ -31,7 +31,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-emitter` - `target/debug/otlp-demo-collector` - `target/debug/otlp-wire-forwarder` @@ -46,7 +46,7 @@ From this directory: The script: -1. starts `logjetd` +1. starts `ljd` 2. sends five one-shot OTLP messages from five separate emitters 3. starts the collector 4. connects the wire forwarder to the replay listener @@ -60,6 +60,6 @@ Expected result: This demonstrates: -- many emitters can connect to one `logjetd` -- `logjetd` stores one merged retained stream +- many emitters can connect to one `ljd` +- `ljd` stores one merged retained stream - one downstream consumer can receive that merged stream later diff --git a/demo/multi-emitter/run-demo.sh b/demo/multi-emitter/run-demo.sh index 9fedbce..1406a2f 100755 --- a/demo/multi-emitter/run-demo.sh +++ b/demo/multi-emitter/run-demo.sh @@ -3,13 +3,13 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" FORWARDER="$TARGET_DIR/otlp-wire-forwarder" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR" "$FORWARDER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -19,14 +19,14 @@ done cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { kill "${FORWARDER_PID:-}" 2>/dev/null || true kill "${COLLECTOR_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/remote-drain-tls/README.md b/demo/remote-drain-tls/README.md index bb1160d..7930804 100644 --- a/demo/remote-drain-tls/README.md +++ b/demo/remote-drain-tls/README.md @@ -4,15 +4,15 @@ This demo shows the same network shape as the plain remote-drain demo, but with TLS enabled on the daemon-to-daemon replay link: ```text -OA -> logjetd <=TLS=> logjetd -> OTel Collector +OA -> ljd <=TLS=> ljd -> OTel Collector ``` In this demo: -- the appliance-side `logjetd` accepts OTLP/HTTP logs locally +- the appliance-side `ljd` accepts OTLP/HTTP logs locally - the appliance-side emitter stands in for `OA` -- the remote-side `logjetd` runs in `bridge` mode -- the replay connection between the two `logjetd` instances is protected with TLS +- the remote-side `ljd` runs in `bridge` mode +- the replay connection between the two `ljd` instances is protected with TLS - the remote-side collector mockup stands in for an OTel Collector This demo exercises all current replay/bridge TLS features: @@ -43,7 +43,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-emitter` - `target/debug/otlp-demo-collector` @@ -57,7 +57,7 @@ From this directory: This starts: -1. appliance-side `logjetd` with local OTLP/HTTP ingest +1. appliance-side `ljd` with local OTLP/HTTP ingest 2. memory retention with: - `buffer.keep: 3` - `buffer.messages: 5` @@ -84,7 +84,7 @@ From this directory: This starts: 1. the OTLP collector mockup on `127.0.0.1:4320` -2. remote-side `logjetd bridge` +2. remote-side `ljd bridge` 3. a TLS connection from the remote side into the appliance replay listener Expected result: diff --git a/demo/remote-drain-tls/run-appliance.sh b/demo/remote-drain-tls/run-appliance.sh index 17bc9bc..95c0ba3 100755 --- a/demo/remote-drain-tls/run-appliance.sh +++ b/demo/remote-drain-tls/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -28,13 +28,13 @@ done cd "$SCRIPT_DIR" -echo "starting appliance-side logjetd with TLS replay listener and required client certs" +echo "starting appliance-side ljd with TLS replay listener and required client certs" echo "config: $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -47,5 +47,5 @@ for i in 1 2 3 4 5 6 7 8; do "$EMITTER" 127.0.0.1:4318 --once --message "BOOT MESSAGE #$i" done -echo "starting continuous BOFH traffic toward appliance-side logjetd on 127.0.0.1:4318" +echo "starting continuous BOFH traffic toward appliance-side ljd on 127.0.0.1:4318" "$EMITTER" 127.0.0.1:4318 diff --git a/demo/remote-drain-tls/run-remote.sh b/demo/remote-drain-tls/run-remote.sh index 2e5346e..72fc927 100755 --- a/demo/remote-drain-tls/run-remote.sh +++ b/demo/remote-drain-tls/run-remote.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/remote-logjetd.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -43,4 +43,4 @@ sleep 1 echo "starting remote-side bridge with TLS and client certificate" echo "it connects to 127.0.0.1:7002 but validates server name appliance.demo.logjet" echo "config: $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/remote-drain/README.md b/demo/remote-drain/README.md index c01c4ef..b723b26 100644 --- a/demo/remote-drain/README.md +++ b/demo/remote-drain/README.md @@ -3,14 +3,14 @@ This demo shows the intended network shape: ```text -OA -> logjetd <- network <- logjetd -> OTel Collector +OA -> ljd <- network <- ljd -> OTel Collector ``` In this demo: -- the appliance-side `logjetd` accepts OTLP/HTTP logs locally +- the appliance-side `ljd` accepts OTLP/HTTP logs locally - the appliance-side emitter stands in for `OA` -- the remote-side `logjetd` runs in `bridge` mode +- the remote-side `ljd` runs in `bridge` mode - the remote-side collector mockup stands in for an OTel Collector - the appliance-side daemon uses memory retention with a permanent kept prefix @@ -29,7 +29,7 @@ make demo That gives you: -- `target/debug/logjetd` +- `target/debug/ljd` - `target/debug/otlp-bofh-emitter` - `target/debug/otlp-demo-collector` @@ -43,7 +43,7 @@ From this directory: This starts: -1. appliance-side `logjetd` with local OTLP/HTTP ingest +1. appliance-side `ljd` with local OTLP/HTTP ingest 2. memory retention with: - `buffer.keep: 3` - `buffer.messages: 5` @@ -70,7 +70,7 @@ From this directory: This starts: 1. the OTLP collector mockup on `127.0.0.1:4320` -2. remote-side `logjetd bridge` +2. remote-side `ljd bridge` 3. a connection from the remote side into the appliance replay listener Expected result: diff --git a/demo/remote-drain/run-appliance.sh b/demo/remote-drain/run-appliance.sh index f7142ac..69eabab 100755 --- a/demo/remote-drain/run-appliance.sh +++ b/demo/remote-drain/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -17,12 +17,12 @@ done cd "$SCRIPT_DIR" -echo "starting appliance-side logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting appliance-side ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -35,5 +35,5 @@ for i in 1 2 3 4 5 6 7 8; do "$EMITTER" 127.0.0.1:4318 --once --message "BOOT MESSAGE #$i" done -echo "starting continuous BOFH traffic toward appliance-side logjetd on 127.0.0.1:4318" +echo "starting continuous BOFH traffic toward appliance-side ljd on 127.0.0.1:4318" "$EMITTER" 127.0.0.1:4318 diff --git a/demo/remote-drain/run-remote.sh b/demo/remote-drain/run-remote.sh index 18e0ffd..990f9f7 100755 --- a/demo/remote-drain/run-remote.sh +++ b/demo/remote-drain/run-remote.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/remote-logjetd.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting remote-side bridge using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/demo/replay-handoff/README.md b/demo/replay-handoff/README.md index 797b96e..9736098 100644 --- a/demo/replay-handoff/README.md +++ b/demo/replay-handoff/README.md @@ -1,6 +1,6 @@ # Replay Handoff Demo -This demo isolates the backlog-to-live handoff inside `logjetd serve`. +This demo isolates the backlog-to-live handoff inside `ljd serve`. It proves one replay client can: @@ -25,7 +25,7 @@ Terminal 1, from this directory: ./run-appliance.sh ``` -This starts `logjetd`, writes three retained messages before any replay client +This starts `ljd`, writes three retained messages before any replay client connects, and then keeps sending one live message per second. Terminal 2, from this directory: diff --git a/demo/replay-handoff/run-appliance.sh b/demo/replay-handoff/run-appliance.sh index 78fc1b1..b3b8600 100755 --- a/demo/replay-handoff/run-appliance.sh +++ b/demo/replay-handoff/run-appliance.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -18,13 +18,13 @@ done mkdir -p "$SCRIPT_DIR/spool" cd "$SCRIPT_DIR" -echo "starting logjetd with config $CONFIG" -"$LOGJETD" --config "$CONFIG" serve & -LOGJETD_PID=$! +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" serve & +LJD_PID=$! cleanup() { kill "${LIVE_PID:-}" 2>/dev/null || true - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/secure-pipeline/README.md b/demo/secure-pipeline/README.md index 20ebb99..7e508cb 100644 --- a/demo/secure-pipeline/README.md +++ b/demo/secure-pipeline/README.md @@ -3,7 +3,7 @@ This demo shows secure OTLP features in one path: ```text -OA -- HTTPS OTLP --> logjetd -- HTTPS OTLP --> collector +OA -- HTTPS OTLP --> ljd -- HTTPS OTLP --> collector ``` It demonstrates: @@ -35,17 +35,17 @@ From this directory: The script starts: 1. a demo HTTPS collector on `127.0.0.1:4321` -2. `logjetd` with: +2. `ljd` with: - HTTPS OTLP ingest on `127.0.0.1:4319` - HTTPS collector export to the collector 3. removes old generated `secure*.logjet` demo files from `./logs` -4. the BOFH emitter, sending OTLP/HTTP over HTTPS into `logjetd` -5. `logjetd replay` blasting the stored `.logjet` file into the HTTPS collector +4. the BOFH emitter, sending OTLP/HTTP over HTTPS into `ljd` +5. `ljd replay` blasting the stored `.logjet` file into the HTTPS collector Expected result: - the emitter prints plain BOFH log batches it sends over HTTPS -- `logjetd` stores them in `./logs` +- `ljd` stores them in `./logs` - the collector prints the same 5 records after replay ## Certificates diff --git a/demo/secure-pipeline/run-demo.sh b/demo/secure-pipeline/run-demo.sh index 8f47eee..0baa333 100755 --- a/demo/secure-pipeline/run-demo.sh +++ b/demo/secure-pipeline/run-demo.sh @@ -3,12 +3,12 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/logjetd.conf" -for bin in "$LOGJETD" "$EMITTER" "$COLLECTOR"; do +for bin in "$LJD" "$EMITTER" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -41,12 +41,12 @@ echo "starting HTTPS collector on 127.0.0.1:4321" --key-file "$SCRIPT_DIR/certs/collector.key" & COLLECTOR_PID=$! -echo "starting logjetd with HTTPS OTLP ingest and HTTPS collector export" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +echo "starting ljd with HTTPS OTLP ingest and HTTPS collector export" +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true kill "${COLLECTOR_PID:-}" 2>/dev/null || true } @@ -64,4 +64,4 @@ echo "sending 5 BOFH batches to https://127.0.0.1:4319/v1/logs" sleep 1 echo "replaying stored records to HTTPS collector" -"$LOGJETD" --config "$CONFIG" replay --path ./logs --name secure.logjet +"$LJD" --config "$CONFIG" replay --path ./logs --name secure.logjet diff --git a/demo/src/bin/otlp-random-logjet-generator.rs b/demo/src/bin/otlp-random-logjet-generator.rs index 9926925..b6439e0 100644 --- a/demo/src/bin/otlp-random-logjet-generator.rs +++ b/demo/src/bin/otlp-random-logjet-generator.rs @@ -15,16 +15,8 @@ fn main() -> Result<(), Box> { std::process::exit(2); } }; - let count = args - .next() - .map(|value| value.parse::()) - .transpose()? - .unwrap_or(1000); - let seed = args - .next() - .map(|value| value.parse::()) - .transpose()? - .unwrap_or(0x5eed_1234_u64); + let count = args.next().map(|value| value.parse::()).transpose()?.unwrap_or(1000); + let seed = args.next().map(|value| value.parse::()).transpose()?.unwrap_or(0x5eed_1234_u64); let file = File::create(&output)?; let writer = BufWriter::new(file); @@ -45,14 +37,7 @@ fn main() -> Result<(), Box> { Ok(()) } -const SERVICES: &[&str] = &[ - "bofh-emitter", - "kill-bill", - "garage-rig", - "bridge-alpha", - "night-shift", - "coffee-daemon", -]; +const SERVICES: &[&str] = &["bofh-emitter", "kill-bill", "garage-rig", "bridge-alpha", "night-shift", "coffee-daemon"]; const LEVELS: &[&str] = &["trace", "debug", "info", "warn", "error"]; @@ -111,10 +96,7 @@ impl Lcg { } fn next(&mut self) -> u64 { - self.state = self - .state - .wrapping_mul(6364136223846793005) - .wrapping_add(1); + self.state = self.state.wrapping_mul(6364136223846793005).wrapping_add(1); self.state } diff --git a/demo/upstream-reset-resume/README.md b/demo/upstream-reset-resume/README.md index 109b2e7..5a699e3 100644 --- a/demo/upstream-reset-resume/README.md +++ b/demo/upstream-reset-resume/README.md @@ -34,7 +34,7 @@ From this directory: This starts: -1. appliance-side `logjetd` +1. appliance-side `ljd` 2. a simple `ALPHA 001`, `ALPHA 002`, `ALPHA 003` message stream This script also removes any old `bridge.state` file before the first run. @@ -50,7 +50,7 @@ From this directory: This starts: 1. the collector -2. consumer-side `logjetd bridge` +2. consumer-side `ljd bridge` The consumer keeps its state in: diff --git a/demo/upstream-reset-resume/run-appliance-alpha.sh b/demo/upstream-reset-resume/run-appliance-alpha.sh index 4de888c..01532d7 100755 --- a/demo/upstream-reset-resume/run-appliance-alpha.sh +++ b/demo/upstream-reset-resume/run-appliance-alpha.sh @@ -3,12 +3,12 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" STATE_FILE="$SCRIPT_DIR/bridge.state" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -21,11 +21,11 @@ cd "$SCRIPT_DIR" rm -f "$STATE_FILE" echo "starting ALPHA appliance stream and clearing old bridge.state" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/upstream-reset-resume/run-appliance-bravo.sh b/demo/upstream-reset-resume/run-appliance-bravo.sh index 56edfe1..76b2563 100755 --- a/demo/upstream-reset-resume/run-appliance-bravo.sh +++ b/demo/upstream-reset-resume/run-appliance-bravo.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" EMITTER="$TARGET_DIR/otlp-bofh-emitter" CONFIG="$SCRIPT_DIR/appliance-logjetd.conf" -for bin in "$LOGJETD" "$EMITTER"; do +for bin in "$LJD" "$EMITTER"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -18,11 +18,11 @@ done cd "$SCRIPT_DIR" echo "starting BRAVO appliance stream with a fresh in-memory upstream" -"$LOGJETD" --config "$CONFIG" & -LOGJETD_PID=$! +"$LJD" --config "$CONFIG" & +LJD_PID=$! cleanup() { - kill "${LOGJETD_PID:-}" 2>/dev/null || true + kill "${LJD_PID:-}" 2>/dev/null || true } trap cleanup EXIT INT TERM diff --git a/demo/upstream-reset-resume/run-consumer.sh b/demo/upstream-reset-resume/run-consumer.sh index 4a2bd6c..d62e9e8 100755 --- a/demo/upstream-reset-resume/run-consumer.sh +++ b/demo/upstream-reset-resume/run-consumer.sh @@ -3,11 +3,11 @@ set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) TARGET_DIR="$SCRIPT_DIR/../../target/debug" -LOGJETD="$TARGET_DIR/logjetd" +LJD="$TARGET_DIR/ljd" COLLECTOR="$TARGET_DIR/otlp-demo-collector" CONFIG="$SCRIPT_DIR/consumer-logjetd.conf" -for bin in "$LOGJETD" "$COLLECTOR"; do +for bin in "$LJD" "$COLLECTOR"; do if [ ! -x "$bin" ]; then echo "missing $bin" echo "build everything first with: make demo" @@ -30,4 +30,4 @@ trap cleanup EXIT INT TERM sleep 1 echo "starting consumer-side bridge using $CONFIG" -"$LOGJETD" --config "$CONFIG" bridge +"$LJD" --config "$CONFIG" bridge diff --git a/doc/README.md b/doc/README.md index 4627ad5..7d496d3 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,10 +2,10 @@ - [overview.md](./overview.md): project overview - [ljx.md](./ljx.md): `ljx` offline CLI scope and command plan -- [daemon.md](./daemon.md): `logjetd` behaviour and current limits +- [daemon.md](./daemon.md): `ljd` behaviour and current limits - [configuration.md](./configuration.md): YAML keys and defaults - [features.md](./features.md): current implemented daemon features and use cases - [config-samples.md](./config-samples.md): ready-to-use config examples -- [protocol.md](./protocol.md): current TCP wire framing used by `logjetd` +- [protocol.md](./protocol.md): current TCP wire framing used by `ljd` - [manpage/ljx.1.md](./manpage/ljx.1.md): Markdown manpage source -- [manpage/logjetd.1.md](./manpage/logjetd.1.md): Markdown manpage source +- [manpage/ljd.1.md](./manpage/ljd.1.md): Markdown manpage source diff --git a/doc/configuration.md b/doc/configuration.md index e412f20..4a26d70 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,6 +1,6 @@ # Configuration -`logjetd` reads YAML configuration from: +`ljd` reads YAML configuration from: - `/etc/logjet.conf` by default - a custom path passed through `-c` or `--config` @@ -120,7 +120,7 @@ Important: Maximum size of one `.logjet` file segment in KiB before rotation. -When the current file exceeds this size, `logjetd` opens the next file: +When the current file exceeds this size, `ljd` opens the next file: - `name.logjet` - `name-1.logjet` @@ -130,9 +130,9 @@ Important: - applies only when `output: file` - old rotated files are kept -- use `logjetd segments --path ... --name ...` to inspect one rotated spool -- use `logjetd prune --path ... --name ... --keep-files ` to trim oldest archived segments by file count -- use `logjetd prune --path ... --name ... --keep-bytes --dry-run` to preview byte-budget pruning without deleting files +- use `ljd segments --path ... --name ...` to inspect one rotated spool +- use `ljd prune --path ... --name ... --keep-files ` to trim oldest archived segments by file count +- use `ljd prune --path ... --name ... --keep-bytes --dry-run` to preview byte-budget pruning without deleting files ### `file.name` @@ -150,7 +150,7 @@ Important: ### `collector.url` -Default destination used by `logjetd replay` when `--dest` is not provided. +Default destination used by `ljd replay` when `--dest` is not provided. Accepted forms: @@ -170,10 +170,10 @@ If the URL starts with `https://`, collector export uses TLS. ### `collector.timeout-ms` -Socket timeout in milliseconds used by `logjetd replay` when posting stored +Socket timeout in milliseconds used by `ljd replay` when posting stored OTLP payloads to `collector.url`. -It is also used by `logjetd bridge` when posting replayed OTLP payloads to the +It is also used by `ljd bridge` when posting replayed OTLP payloads to the collector. ### `collector.ca-file` @@ -250,7 +250,7 @@ Important: ### `ingest.overload-report-ms` -How often `logjetd` prints aggregated ingest overload counters to stderr while +How often `ljd` prints aggregated ingest overload counters to stderr while overload events are happening. Important: @@ -307,7 +307,7 @@ Important: - default is `disconnect` - this setting affects bridge export to the collector -- `logjetd replay` remains a one-shot bulk operation and does not use this policy +- `ljd replay` remains a one-shot bulk operation and does not use this policy ### `backpressure.max-buffered-records` @@ -318,7 +318,7 @@ Important: - default is `16` - must be greater than zero -- applies only to `logjetd bridge` +- applies only to `ljd bridge` - `block` waits for queue space - `disconnect` fails the bridge connection when the queue is full - `drop-newest` drops the newest queued record when the queue is full @@ -343,19 +343,19 @@ Important: ### `upstream.replay` -Source `host:port` for `logjetd bridge`. +Source `host:port` for `ljd bridge`. -This should point at another `logjetd` replay listener, not an OTLP endpoint. +This should point at another `ljd` replay listener, not an OTLP endpoint. Example: - `10.0.0.15:7002` -If this key is omitted, `logjetd bridge` requires `--source`. +If this key is omitted, `ljd bridge` requires `--source`. ### `upstream.mode` -Retention mode requested by `logjetd bridge` from the upstream replay listener. +Retention mode requested by `ljd bridge` from the upstream replay listener. Values: @@ -373,7 +373,7 @@ Important: ### `upstream.state-file` -Optional local file used by `logjetd bridge` to persist the last successfully +Optional local file used by `ljd bridge` to persist the last successfully forwarded sequence. Important: @@ -388,14 +388,14 @@ Important: ### `upstream.retry-ms` -Reconnect delay in milliseconds for `logjetd bridge`. +Reconnect delay in milliseconds for `ljd bridge`. When the upstream replay connection closes or fails, bridge mode waits this long before reconnecting. ### `upstream.connect-timeout-ms` -TCP connect timeout in milliseconds for `logjetd bridge` when opening the +TCP connect timeout in milliseconds for `ljd bridge` when opening the upstream replay connection. ### `tls.enable` @@ -443,7 +443,7 @@ When enabled: ### `tls.server-name` -Override server name used by `logjetd bridge` for TLS certificate validation. +Override server name used by `ljd bridge` for TLS certificate validation. Use this when: @@ -452,12 +452,12 @@ Use this when: ### `ingest.protocol` -Selects how `logjetd` accepts incoming telemetry. +Selects how `ljd` accepts incoming telemetry. Values: - `wire` - - the current internal framed TCP protocol used by `logjetd` replay clients and custom clients + - the current internal framed TCP protocol used by `ljd` replay clients and custom clients - this is not OTLP - `otlp-http` - OTLP over HTTP protobuf @@ -466,7 +466,7 @@ Values: - OTLP over gRPC - accepts the standard logs `Export` RPC -If you want a normal OpenTelemetry producer to send logs directly to `logjetd`, +If you want a normal OpenTelemetry producer to send logs directly to `ljd`, use either `otlp-http` or `otlp-grpc`. ### `ingest.tls-enable` @@ -475,8 +475,8 @@ Enable TLS for OTLP ingest listeners. Behaviour: -- with `ingest.protocol: otlp-http`, `logjetd` accepts HTTPS on `/v1/logs` -- with `ingest.protocol: otlp-grpc`, `logjetd` accepts gRPC over TLS +- with `ingest.protocol: otlp-http`, `ljd` accepts HTTPS on `/v1/logs` +- with `ingest.protocol: otlp-grpc`, `ljd` accepts gRPC over TLS ### `ingest.ca-file` @@ -570,13 +570,13 @@ If omitted: - set either `buffer.size` or `buffer.messages`, never both - `buffer.size` limits the rotating in-memory tail by bytes - `buffer.messages` limits the rotating in-memory tail by message count -- `collector.url` is used by `logjetd replay` when `--dest` is omitted +- `collector.url` is used by `ljd replay` when `--dest` is omitted - `collector.timeout-ms` controls replay and bridge HTTP socket timeout - `collector.ca-file`, `collector.cert-file`, `collector.key-file`, and `collector.server-name` apply to HTTPS collector export - `backpressure.enabled` enables bridge backpressure policy handling - `backpressure.mode` configures whether bridge export blocks, disconnects, or drops newest records when the collector is too slow - `backpressure.max-buffered-records` caps the bridge-side exporter queue per bridge connection -- `upstream.replay` is used by `logjetd bridge` when `--source` is omitted +- `upstream.replay` is used by `ljd bridge` when `--source` is omitted - `upstream.mode: keep` leaves upstream retained records in place after replay - `upstream.mode: drain` consumes upstream retained records after successful bridge export - `upstream.state-file` stores the last forwarded sequence and upstream stream identity on the downstream bridge side diff --git a/doc/daemon.md b/doc/daemon.md index 6370212..c3c7cec 100644 --- a/doc/daemon.md +++ b/doc/daemon.md @@ -1,10 +1,10 @@ -# logjetd Daemon +# ljd Daemon -`logjetd` is a separate binary crate under [`logjetd/`](../logjetd). +`ljd` is a separate binary crate under [`logjetd/`](../logjetd). ## What It Does -`logjetd` accepts telemetry on an ingest socket, stores backlog, and can replay +`ljd` accepts telemetry on an ingest socket, stores backlog, and can replay that backlog later. It supports two storage modes: @@ -23,44 +23,44 @@ Default config path: Run with default config path: ```bash -logjetd +ljd ``` Explicit config path: ```bash -logjetd --config /path/to/logjet.conf +ljd --config /path/to/logjet.conf ``` Inspect a file or directory: ```bash -logjetd inspect /var/lib/logjet +ljd inspect /var/lib/logjet ``` List rotated file segments for one spool: ```bash -logjetd segments --path /var/lib/logjet --name app.logjet +ljd segments --path /var/lib/logjet --name app.logjet ``` Prune oldest rotated file segments and keep only the newest two files: ```bash -logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 +ljd prune --path /var/lib/logjet --name app.logjet --keep-files 2 ``` Preview byte-budget pruning without deleting anything: ```bash -logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run +ljd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run ``` -Continuously bridge from another `logjetd` replay listener into an OTLP +Continuously bridge from another `ljd` replay listener into an OTLP collector: ```bash -logjetd --config /path/to/logjet.conf bridge --source 10.0.0.15:7002 +ljd --config /path/to/logjet.conf bridge --source 10.0.0.15:7002 ``` ## Runtime Behaviour @@ -100,7 +100,7 @@ upstream. ### Continuous Bridge -- `logjetd bridge` connects to another `logjetd` replay listener +- `ljd bridge` connects to another `ljd` replay listener - requests either `keep` or `drain` mode from the upstream side - continues forwarding newly replayed records - forwards OTLP log payloads to `collector.url` @@ -119,7 +119,7 @@ upstream. ### File Blast Replay -- `logjetd replay --path ... --name ...` +- `ljd replay --path ... --name ...` - reads ordered rotated `.logjet` files - sends stored OTLP log batches to a collector URL - uses `collector.url` by default @@ -128,16 +128,16 @@ upstream. ### File Operational Tooling -- `logjetd segments --path ... --name ...` +- `ljd segments --path ... --name ...` - prints ordered metadata for one rotated spool: - segment id - file path - file size in bytes - record count - first and last sequence -- `logjetd prune --path ... --name ... --keep-files ` +- `ljd prune --path ... --name ... --keep-files ` - removes oldest rotated segments and keeps the newest `n` segment files -- `logjetd prune --path ... --name ... --keep-bytes ` +- `ljd prune --path ... --name ... --keep-bytes ` - removes oldest rotated segments until the newest retained set fits within the byte budget - `--dry-run` prints the paths that would be removed without deleting them - the newest active segment is always retained diff --git a/doc/features.md b/doc/features.md index e3d3971..ab7aaca 100644 --- a/doc/features.md +++ b/doc/features.md @@ -1,13 +1,13 @@ -# logjetd Features +# ljd Features -This file tracks the current feature set and practical use cases of `logjetd`. +This file tracks the current feature set and practical use cases of `ljd`. It is meant to evolve as the daemon grows. ## Current Features ### 1. OTLP log ingest -`logjetd` can accept real OTLP/HTTP protobuf log export requests on: +`ljd` can accept real OTLP/HTTP protobuf log export requests on: ```text POST /v1/logs @@ -32,7 +32,7 @@ Current behaviour: ### 2. Append-only `.logjet` file output -In file mode, `logjetd` writes raw OTLP protobuf batches into `.logjet` files +In file mode, `ljd` writes raw OTLP protobuf batches into `.logjet` files using the `logjet` block format. Current behaviour: @@ -46,13 +46,13 @@ Current behaviour: - `name-1.logjet` - `name-2.logjet` - file-mode operational tooling exists through: - - `logjetd segments --path ... --name ...` - - `logjetd prune --path ... --name ... --keep-files ...` - - `logjetd prune --path ... --name ... --keep-bytes ...` + - `ljd segments --path ... --name ...` + - `ljd prune --path ... --name ... --keep-files ...` + - `ljd prune --path ... --name ... --keep-bytes ...` ### 3. In-memory ring buffer mode -In buffer mode, `logjetd` can hold retained records in RAM. +In buffer mode, `ljd` can hold retained records in RAM. Current behaviour: @@ -70,7 +70,7 @@ Memory model: ### 4. Replay listener -`logjetd` exposes a replay socket for downstream consumers. +`ljd` exposes a replay socket for downstream consumers. Current behaviour: @@ -90,15 +90,15 @@ Current limitation: ### 5. Continuous bridge mode -`logjetd` can run as a downstream bridge process with: +`ljd` can run as a downstream bridge process with: ```text -logjetd bridge [--source ] +ljd bridge [--source ] ``` Current behaviour: -- connects to another `logjetd` replay listener +- connects to another `ljd` replay listener - requests replay starting after the last sequence already forwarded - can keep upstream records or drain them, depending on `upstream.mode` - stays attached and forwards new log records live @@ -115,7 +115,7 @@ Current behaviour: This is the current path for: ```text -OA -> logjetd <- network <- logjetd -> OTel Collector +OA -> ljd <- network <- ljd -> OTel Collector ``` ### 6. Optional TLS on replay and bridge transport @@ -137,11 +137,11 @@ Current limitation: ### 7. One-shot file replay to OTLP/HTTP -`logjetd` can replay stored `.logjet` files directly into an OTLP/HTTP +`ljd` can replay stored `.logjet` files directly into an OTLP/HTTP collector with: ```text -logjetd replay --path --name [--dest ] +ljd replay --path --name [--dest ] ``` Current behaviour: @@ -156,7 +156,7 @@ Current behaviour: ### 8. YAML configuration -`logjetd` reads configuration from: +`ljd` reads configuration from: - `/etc/logjet.conf` by default - a file passed with `-c` or `--config` @@ -183,24 +183,24 @@ Current config areas: ### 9. Inspection tooling -`logjetd` can inspect stored `.logjet` files or directories and print metadata +`ljd` can inspect stored `.logjet` files or directories and print metadata about retained records. ### 10. File-mode operational tooling -`logjetd` can report and prune rotated file segments explicitly. +`ljd` can report and prune rotated file segments explicitly. Current behaviour: -- `logjetd segments --path --name ` prints ordered segment metadata +- `ljd segments --path --name ` prints ordered segment metadata - output includes: - segment id - file path - file size - record count - first and last sequence in that segment -- `logjetd prune --path --name --keep-files ` removes oldest rotated segments and keeps the newest `n` segment files -- `logjetd prune --path --name --keep-bytes ` removes oldest rotated segments until the newest retained set fits within the byte budget +- `ljd prune --path --name --keep-files ` removes oldest rotated segments and keeps the newest `n` segment files +- `ljd prune --path --name --keep-bytes ` removes oldest rotated segments until the newest retained set fits within the byte budget - `--dry-run` shows what would be removed without deleting anything - the newest active segment is always retained @@ -211,7 +211,7 @@ Current behaviour: Use case: - an emitter sends OTLP logs locally -- `logjetd` stores them in `.logjet` files +- `ljd` stores them in `.logjet` files - files can be inspected, extracted, or replayed later Useful when: @@ -250,7 +250,7 @@ Useful when: Use case: - use the OTLP demo emitter -- send logs into `logjetd` +- send logs into `ljd` - store them or inspect them locally Useful when: @@ -290,8 +290,8 @@ Useful when: Use case: -- one `logjetd` instance runs next to `OA` -- a second `logjetd` instance connects to the first over the network +- one `ljd` instance runs next to `OA` +- a second `ljd` instance connects to the first over the network - the second instance forwards retained backlog and live OTLP logs into an OTel Collector Useful when: diff --git a/doc/ljx.md b/doc/ljx.md index a4db715..aedccb8 100644 --- a/doc/ljx.md +++ b/doc/ljx.md @@ -2,10 +2,10 @@ `ljx` is the offline command-line toolbox for `.logjet` files. -It is separate from `logjetd` and must stay separate in purpose: +It is separate from `ljd` and must stay separate in purpose: - `logjet` is the Rust library and file format -- `logjetd` is the daemon for ingest, transport, replay, and spool management +- `ljd` is the daemon for ingest, transport, replay, and spool management - `ljx` is the standalone file tool for inspection and transformation `ljx` does not control the daemon and does not depend on daemon runtime state. diff --git a/doc/manpage/logjetd.1 b/doc/manpage/ljd.1 similarity index 89% rename from doc/manpage/logjetd.1 rename to doc/manpage/ljd.1 index 009b064..48a4d38 100644 --- a/doc/manpage/logjetd.1 +++ b/doc/manpage/ljd.1 @@ -1,35 +1,35 @@ .\" Automatically generated by Pandoc 2.9.2.1 .\" -.TH "LOGJETD" "1" "March 2026" "" "" +.TH "LJD" "1" "March 2026" "" "" .hy .SH NAME .PP -logjetd - OTLP ingest, \f[C].logjet\f[R] storage, replay, and file +ljd - OTLP ingest, \f[C].logjet\f[R] storage, replay, and file blasting daemon .SH SYNOPSIS .PP -\f[C]logjetd\f[R] [\f[C]serve\f[R]] [\f[C]-c\f[R]|\f[C]--config\f[R] +\f[C]ljd\f[R] [\f[C]serve\f[R]] [\f[C]-c\f[R]|\f[C]--config\f[R] \f[I]path\f[R]] .PP -\f[C]logjetd\f[R] \f[C]inspect\f[R] \f[I]path\f[R] +\f[C]ljd\f[R] \f[C]inspect\f[R] \f[I]path\f[R] .PP -\f[C]logjetd\f[R] \f[C]segments\f[R] \f[C]--path\f[R] \f[I]dir\f[R] +\f[C]ljd\f[R] \f[C]segments\f[R] \f[C]--path\f[R] \f[I]dir\f[R] \f[C]--name\f[R] \f[I]base.logjet\f[R] .PP -\f[C]logjetd\f[R] \f[C]replay\f[R] [\f[C]-c\f[R]|\f[C]--config\f[R] +\f[C]ljd\f[R] \f[C]replay\f[R] [\f[C]-c\f[R]|\f[C]--config\f[R] \f[I]path\f[R]] \f[C]--path\f[R] \f[I]dir\f[R] \f[C]--name\f[R] \f[I]base.logjet\f[R] \f[C]--dest\f[R] \f[I]url-or-host:port\f[R] .PP -\f[C]logjetd\f[R] \f[C]prune\f[R] \f[C]--path\f[R] \f[I]dir\f[R] +\f[C]ljd\f[R] \f[C]prune\f[R] \f[C]--path\f[R] \f[I]dir\f[R] \f[C]--name\f[R] \f[I]base.logjet\f[R] [\f[C]--keep-files\f[R] \f[I]n\f[R] | \f[C]--keep-bytes\f[R] \f[I]bytes\f[R]] \f[C]--dry-run\f[R] .PP -\f[C]logjetd\f[R] \f[C]bridge\f[R] [\f[C]-c\f[R]|\f[C]--config\f[R] +\f[C]ljd\f[R] \f[C]bridge\f[R] [\f[C]-c\f[R]|\f[C]--config\f[R] \f[I]path\f[R]] \f[C]--source\f[R] \f[I]host:port\f[R] .SH DESCRIPTION .PP -\f[C]logjetd\f[R] is a small telemetry daemon for constrained systems. +\f[C]ljd\f[R] is a small telemetry daemon for constrained systems. .PP It can: .IP \[bu] 2 @@ -46,7 +46,7 @@ store raw OTLP protobuf payloads either in memory or in append-only expose a replay listener for downstream consumers over the current internal wire protocol .IP \[bu] 2 -connect to another \f[C]logjetd\f[R] replay listener and forward backlog +connect to another \f[C]ljd\f[R] replay listener and forward backlog plus live records into an OTLP/HTTP collector .IP \[bu] 2 optionally protect replay and bridge transport with TLS @@ -91,7 +91,7 @@ replay listener traffic currently uses the internal framed protocol, not OTLP egress .SS bridge .PP -Connect to another \f[C]logjetd\f[R] replay listener, drain retained +Connect to another \f[C]ljd\f[R] replay listener, drain retained backlog, stay attached for live records, and forward OTLP log payloads to the configured collector. .PP @@ -99,7 +99,7 @@ Example: .IP .nf \f[C] -logjetd --config ./logjetd.conf bridge --source 10.0.0.15:7002 +ljd --config ./logjetd.conf bridge --source 10.0.0.15:7002 \f[R] .fi .PP @@ -114,7 +114,7 @@ Example: .IP .nf \f[C] -logjetd inspect /var/lib/logjet +ljd inspect /var/lib/logjet \f[R] .fi .SS segments @@ -125,7 +125,7 @@ Example: .IP .nf \f[C] -logjetd segments --path /var/lib/logjet --name app.logjet +ljd segments --path /var/lib/logjet --name app.logjet \f[R] .fi .SS replay @@ -137,7 +137,7 @@ Example: .IP .nf \f[C] -logjetd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs +ljd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs \f[R] .fi .PP @@ -162,8 +162,8 @@ Examples: .IP .nf \f[C] -logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 -logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run +ljd prune --path /var/lib/logjet --name app.logjet --keep-files 2 +ljd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run \f[R] .fi .SH OPTIONS @@ -215,7 +215,7 @@ If only \f[C]host:port\f[R] is given, replay defaults to \f[C]/v1/logs\f[R]. .SS \f[C]--source\f[R] \f[I]host:port\f[R] .PP -Upstream \f[C]logjetd\f[R] replay listener for the \f[C]bridge\f[R] +Upstream \f[C]ljd\f[R] replay listener for the \f[C]bridge\f[R] command. .PP If omitted, \f[C]bridge\f[R] uses \f[C]upstream.replay\f[R] from @@ -426,7 +426,7 @@ optional TLS on replay/bridge transport HTTPS OTLP collector export .IP \[bu] 2 one-shot file replay to OTLP/HTTP collectors with -\f[C]logjetd replay\f[R] +\f[C]ljd replay\f[R] .IP \[bu] 2 configurable replay destination via \f[C]collector.url\f[R] .IP \[bu] 2 @@ -452,7 +452,7 @@ Run the daemon with explicit config: .IP .nf \f[C] -logjetd --config ./logjetd.conf +ljd --config ./logjetd.conf \f[R] .fi .PP @@ -460,7 +460,7 @@ Inspect generated files: .IP .nf \f[C] -logjetd inspect ./logs +ljd inspect ./logs \f[R] .fi .PP @@ -468,7 +468,7 @@ Replay files to a local collector: .IP .nf \f[C] -logjetd --config ./logjetd.conf replay --path ./logs --name app.logjet +ljd --config ./logjetd.conf replay --path ./logs --name app.logjet \f[R] .fi .PP @@ -476,7 +476,7 @@ Bridge from a remote replay listener into a collector: .IP .nf \f[C] -logjetd --config ./logjetd.conf bridge --source 10.0.0.15:7002 +ljd --config ./logjetd.conf bridge --source 10.0.0.15:7002 \f[R] .fi .SH FILES @@ -487,10 +487,10 @@ default configuration file \f[B]\f[CB]*.logjet\f[B]\f[R] append-only telemetry files written by file mode .TP -\f[B]\f[CB]doc/manpage/logjetd.1.md\f[B]\f[R] +\f[B]\f[CB]doc/manpage/ljd.1.md\f[B]\f[R] Markdown manpage source .TP -\f[B]\f[CB]doc/manpage/logjetd.1\f[B]\f[R] +\f[B]\f[CB]doc/manpage/ljd.1\f[B]\f[R] generated manpage output .SH EXIT STATUS .TP diff --git a/doc/manpage/logjetd.1.md b/doc/manpage/ljd.1.md similarity index 86% rename from doc/manpage/logjetd.1.md rename to doc/manpage/ljd.1.md index bab71a8..d2770c4 100644 --- a/doc/manpage/logjetd.1.md +++ b/doc/manpage/ljd.1.md @@ -1,28 +1,28 @@ -% LOGJETD(1) +% LJD(1) % Bo Maryniuk % March 2026 # NAME -logjetd - OTLP ingest, `.logjet` storage, replay, and file blasting daemon +ljd - OTLP ingest, `.logjet` storage, replay, and file blasting daemon # SYNOPSIS -`logjetd` [`serve`] [`-c`|`--config` *path*] +`ljd` [`serve`] [`-c`|`--config` *path*] -`logjetd` `inspect` *path* +`ljd` `inspect` *path* -`logjetd` `segments` `--path` *dir* `--name` *base.logjet* +`ljd` `segments` `--path` *dir* `--name` *base.logjet* -`logjetd` `replay` [`-c`|`--config` *path*] `--path` *dir* `--name` *base.logjet* [`--dest` *url-or-host:port*] +`ljd` `replay` [`-c`|`--config` *path*] `--path` *dir* `--name` *base.logjet* [`--dest` *url-or-host:port*] -`logjetd` `prune` `--path` *dir* `--name` *base.logjet* [`--keep-files` *n* | `--keep-bytes` *bytes*] [`--dry-run`] +`ljd` `prune` `--path` *dir* `--name` *base.logjet* [`--keep-files` *n* | `--keep-bytes` *bytes*] [`--dry-run`] -`logjetd` `bridge` [`-c`|`--config` *path*] [`--source` *host:port*] +`ljd` `bridge` [`-c`|`--config` *path*] [`--source` *host:port*] # DESCRIPTION -`logjetd` is a small telemetry daemon for constrained systems. +`ljd` is a small telemetry daemon for constrained systems. It can: @@ -31,7 +31,7 @@ It can: - optionally run OTLP/HTTP ingest over HTTPS and OTLP/gRPC ingest over TLS - store raw OTLP protobuf payloads either in memory or in append-only `.logjet` files - expose a replay listener for downstream consumers over the current internal wire protocol -- connect to another `logjetd` replay listener and forward backlog plus live records into an OTLP/HTTP collector +- connect to another `ljd` replay listener and forward backlog plus live records into an OTLP/HTTP collector - optionally protect replay and bridge transport with TLS - optionally export to an HTTPS OTLP collector - inspect `.logjet` files and directories @@ -62,14 +62,14 @@ Current serve behaviour: ## bridge -Connect to another `logjetd` replay listener, drain retained backlog, stay +Connect to another `ljd` replay listener, drain retained backlog, stay attached for live records, and forward OTLP log payloads to the configured collector. Example: ```text -logjetd --config ./logjetd.conf bridge --source 10.0.0.15:7002 +ljd --config ./logjetd.conf bridge --source 10.0.0.15:7002 ``` If `--source` is omitted, bridge uses `upstream.replay` from configuration. @@ -81,7 +81,7 @@ Inspect a `.logjet` file or a directory containing `.logjet` files. Example: ```text -logjetd inspect /var/lib/logjet +ljd inspect /var/lib/logjet ``` ## segments @@ -91,7 +91,7 @@ List ordered rotated segments for one spool. Example: ```text -logjetd segments --path /var/lib/logjet --name app.logjet +ljd segments --path /var/lib/logjet --name app.logjet ``` ## replay @@ -102,7 +102,7 @@ payloads into an OTLP/HTTP collector. Example: ```text -logjetd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs +ljd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs ``` Replay order is: @@ -122,8 +122,8 @@ Remove oldest rotated file segments deliberately. Examples: ```text -logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 -logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run +ljd prune --path /var/lib/logjet --name app.logjet --keep-files 2 +ljd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run ``` # OPTIONS @@ -177,7 +177,7 @@ If only `host:port` is given, replay defaults to `/v1/logs`. ## `--source` *host:port* -Upstream `logjetd` replay listener for the `bridge` command. +Upstream `ljd` replay listener for the `bridge` command. If omitted, `bridge` uses `upstream.replay` from configuration. @@ -311,7 +311,7 @@ Append-only file behaviour: - bounded bridge-side exporter queue through `backpressure.max-buffered-records` - optional TLS on replay/bridge transport - HTTPS OTLP collector export -- one-shot file replay to OTLP/HTTP collectors with `logjetd replay` +- one-shot file replay to OTLP/HTTP collectors with `ljd replay` - configurable replay destination via `collector.url` - inspection of `.logjet` files and directories @@ -329,25 +329,25 @@ Append-only file behaviour: Run the daemon with explicit config: ```text -logjetd --config ./logjetd.conf +ljd --config ./logjetd.conf ``` Inspect generated files: ```text -logjetd inspect ./logs +ljd inspect ./logs ``` Replay files to a local collector: ```text -logjetd --config ./logjetd.conf replay --path ./logs --name app.logjet +ljd --config ./logjetd.conf replay --path ./logs --name app.logjet ``` Bridge from a remote replay listener into a collector: ```text -logjetd --config ./logjetd.conf bridge --source 10.0.0.15:7002 +ljd --config ./logjetd.conf bridge --source 10.0.0.15:7002 ``` # FILES @@ -358,10 +358,10 @@ logjetd --config ./logjetd.conf bridge --source 10.0.0.15:7002 `*.logjet` : append-only telemetry files written by file mode -`doc/manpage/logjetd.1.md` +`doc/manpage/ljd.1.md` : Markdown manpage source -`doc/manpage/logjetd.1` +`doc/manpage/ljd.1` : generated manpage output # EXIT STATUS diff --git a/doc/manpage/ljx.1 b/doc/manpage/ljx.1 index da138ea..603131b 100644 --- a/doc/manpage/ljx.1 +++ b/doc/manpage/ljx.1 @@ -27,7 +27,7 @@ files ecosystem. .PP It works directly on \f[C].logjet\f[R] files and is intentionally -separate from \f[C]logjetd\f[R]. +separate from \f[C]ljd\f[R]. .PP \f[C]ljx\f[R] does: .IP \[bu] 2 @@ -43,7 +43,7 @@ fit into ordinary UNIX pipelines .PP \f[C]ljx\f[R] does not: .IP \[bu] 2 -start or control \f[C]logjetd\f[R] +start or control \f[C]ljd\f[R] .IP \[bu] 2 depend on daemon runtime state .IP \[bu] 2 diff --git a/doc/manpage/ljx.1.md b/doc/manpage/ljx.1.md index 2e0b663..449d6b7 100644 --- a/doc/manpage/ljx.1.md +++ b/doc/manpage/ljx.1.md @@ -25,7 +25,7 @@ ljx - offline toolbox for inspecting and transforming `.logjet` files `ljx` is the standalone file tool in the `logjet` ecosystem. It works directly on `.logjet` files and is intentionally separate from -`logjetd`. +`ljd`. `ljx` does: @@ -37,7 +37,7 @@ It works directly on `.logjet` files and is intentionally separate from `ljx` does not: -- start or control `logjetd` +- start or control `ljd` - depend on daemon runtime state - grep raw bytes blindly diff --git a/doc/overview.md b/doc/overview.md index 0f9150a..8cc1c54 100644 --- a/doc/overview.md +++ b/doc/overview.md @@ -3,7 +3,7 @@ `logjet` is split into two parts: - `logjet`: a Rust library and `.logjet` block format for storing raw OTLP protobuf batches -- `logjetd`: a daemon that accepts OTLP logs, keeps a backlog, and replays or blasts stored data later +- `ljd`: a daemon that accepts OTLP logs, keeps a backlog, and replays or blasts stored data later ## Components @@ -16,7 +16,7 @@ The library provides: - corruption tolerance through sync markers and per-block CRC32C - block-local compression with `lz4` or `none` -### `logjetd` +### `ljd` The daemon provides: @@ -45,26 +45,26 @@ The daemon provides: ## Intended Use -`logjetd` is meant to sit next to a local telemetry source such as an OTel +`ljd` is meant to sit next to a local telemetry source such as an OTel Appliance (`OA`). Typical flow: 1. a local source connects to the ingest listener -2. `logjetd` stores records in memory or `.logjet` files +2. `ljd` stores records in memory or `.logjet` files 3. a downstream consumer connects to the replay listener -4. `logjetd` sends retained backlog first -5. `logjetd` continues sending newly ingested records +4. `ljd` sends retained backlog first +5. `ljd` continues sending newly ingested records -`logjetd` can also run remotely in bridge mode: +`ljd` can also run remotely in bridge mode: -1. connect to another `logjetd` replay listener +1. connect to another `ljd` replay listener 2. request records after a known sequence 3. keep or drain retained backlog 4. stay attached for live records 5. forward raw OTLP protobuf payloads into an OTLP/HTTP collector -`logjetd` can also replay stored `.logjet` files later into an OTLP/HTTP +`ljd` can also replay stored `.logjet` files later into an OTLP/HTTP collector without preserving original timing. This is optimised for: diff --git a/doc/protocol.md b/doc/protocol.md index ce9436e..0b82a64 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -1,6 +1,6 @@ # Current Wire Protocol -`logjetd` currently uses a small custom TCP wire protocol between ingest clients +`ljd` currently uses a small custom TCP wire protocol between ingest clients and replay clients. This is not OTLP/gRPC and not OTLP/HTTP. @@ -59,16 +59,16 @@ Meaning: ## Semantics -- ingest clients send framed records to `logjetd` -- replay clients first receive replay hello, then send `from_seq`, then receive framed records from `logjetd` -- `logjetd` does not decode the OTLP payload +- ingest clients send framed records to `ljd` +- replay clients first receive replay hello, then send `from_seq`, then receive framed records from `ljd` +- `ljd` does not decode the OTLP payload - sequence ordering is preserved if producers send ordered sequence numbers - reconnecting bridge clients can resume from the last forwarded sequence without restarting from zero - if the upstream stream identity changes, bridge resets stale saved sequence state automatically ## Why It Exists -The current protocol is intentionally small and dependency-light so `logjetd` +The current protocol is intentionally small and dependency-light so `ljd` can remain easy to build on constrained systems. It is a transport layer for the current daemon implementation, not the final diff --git a/ljx/src/cli.rs b/ljx/src/cli.rs index 50f86ea..1c6fdee 100644 --- a/ljx/src/cli.rs +++ b/ljx/src/cli.rs @@ -53,11 +53,7 @@ pub enum Command { Count(CountArgs), #[command(about = "Compute summary statistics for one .logjet file")] Stats(StatsArgs), - #[command( - name = "view", - alias = "cat", - about = "Interactively browse filtered records in a terminal UI" - )] + #[command(name = "view", alias = "cat", about = "Interactively browse filtered records in a terminal UI")] View(ViewArgs), } @@ -75,12 +71,7 @@ pub struct FilterArgs { #[arg(value_name = "INPUT", help = "Input .logjet file or - for stdin")] pub input: PathBuf, - #[arg( - short, - long, - value_name = "OUTPUT", - help = "Output .logjet file or - for stdout" - )] + #[arg(short, long, value_name = "OUTPUT", help = "Output .logjet file or - for stdout")] pub output: PathBuf, #[arg(long, value_enum, default_value_t = OutputCodec::Lz4)] diff --git a/ljx/src/commands/count.rs b/ljx/src/commands/count.rs index a683a41..937af89 100644 --- a/ljx/src/commands/count.rs +++ b/ljx/src/commands/count.rs @@ -12,9 +12,7 @@ pub fn run(args: CountArgs) -> Result<()> { let mut count = 0u64; while let Some(record) = reader.next_record()? { if predicate.matches(&record) { - count = count - .checked_add(1) - .ok_or(logjet::Error::NumericOverflow("count"))?; + count = count.checked_add(1).ok_or(logjet::Error::NumericOverflow("count"))?; } } diff --git a/ljx/src/commands/filter.rs b/ljx/src/commands/filter.rs index fa8bc20..3445ea1 100644 --- a/ljx/src/commands/filter.rs +++ b/ljx/src/commands/filter.rs @@ -10,23 +10,14 @@ pub fn run(args: FilterArgs) -> Result<()> { let predicate = args.predicate.build()?; let input = InputHandle::open(&args.input)?; let output = open_output(&args.output)?; - let config = WriterConfig { - block_target_size: args.block_target_size, - codec: args.codec.into(), - sync_marker: logjet::DEFAULT_SYNC_MARKER, - }; + let config = WriterConfig { block_target_size: args.block_target_size, codec: args.codec.into(), sync_marker: logjet::DEFAULT_SYNC_MARKER }; let mut reader = LogjetReader::new(input.into_buf_reader()); let mut writer = LogjetWriter::with_config(output, config); while let Some(record) = reader.next_record()? { if predicate.matches(&record) { - writer.push( - record.record_type, - record.seq, - record.ts_unix_ns, - &record.payload, - )?; + writer.push(record.record_type, record.seq, record.ts_unix_ns, &record.payload)?; } } diff --git a/ljx/src/commands/join.rs b/ljx/src/commands/join.rs index b29a88c..b2644d0 100644 --- a/ljx/src/commands/join.rs +++ b/ljx/src/commands/join.rs @@ -2,7 +2,5 @@ use crate::cli::JoinArgs; use crate::error::{Error, Result}; pub fn run(_args: JoinArgs) -> Result<()> { - Err(Error::Unimplemented( - "join is not implemented yet; the CLI shape is defined but ordered multi-segment merging still needs to be added", - )) + Err(Error::Unimplemented("join is not implemented yet; the CLI shape is defined but ordered multi-segment merging still needs to be added")) } diff --git a/ljx/src/commands/split.rs b/ljx/src/commands/split.rs index 829e05d..c524360 100644 --- a/ljx/src/commands/split.rs +++ b/ljx/src/commands/split.rs @@ -2,7 +2,5 @@ use crate::cli::SplitArgs; use crate::error::{Error, Result}; pub fn run(_args: SplitArgs) -> Result<()> { - Err(Error::Unimplemented( - "split is not implemented yet; the CLI shape is defined but block-preserving file partitioning still needs to be added", - )) + Err(Error::Unimplemented("split is not implemented yet; the CLI shape is defined but block-preserving file partitioning still needs to be added")) } diff --git a/ljx/src/commands/stats.rs b/ljx/src/commands/stats.rs index ada43dc..8dad237 100644 --- a/ljx/src/commands/stats.rs +++ b/ljx/src/commands/stats.rs @@ -2,7 +2,5 @@ use crate::cli::StatsArgs; use crate::error::{Error, Result}; pub fn run(_args: StatsArgs) -> Result<()> { - Err(Error::Unimplemented( - "stats is not implemented yet; the CLI shape is defined but streaming summaries still need to be added", - )) + Err(Error::Unimplemented("stats is not implemented yet; the CLI shape is defined but streaming summaries still need to be added")) } diff --git a/ljx/src/commands/view.rs b/ljx/src/commands/view.rs index 1d3e53f..5ae091b 100644 --- a/ljx/src/commands/view.rs +++ b/ljx/src/commands/view.rs @@ -11,9 +11,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use chrono::{TimeZone, Utc}; use crossterm::event::{self, Event, KeyCode, KeyEvent, KeyModifiers}; use crossterm::execute; -use crossterm::terminal::{ - EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode, -}; +use crossterm::terminal::{EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode}; use logjet::{LogjetReader, LogjetWriter, OwnedRecord, RecordType, WriterConfig}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use opentelemetry_proto::tonic::common::v1::any_value::Value; @@ -37,9 +35,7 @@ const TICK_RATE: Duration = Duration::from_millis(100); pub fn run(args: ViewArgs) -> Result<()> { if !io::stdin().is_terminal() || !io::stdout().is_terminal() { - return Err(Error::Usage( - "ljx view needs an interactive terminal; pipe-oriented output belongs in `ljx filter`".to_string(), - )); + return Err(Error::Usage("ljx view needs an interactive terminal; pipe-oriented output belongs in `ljx filter`".to_string())); } let mut stdout = io::stdout(); @@ -85,10 +81,7 @@ struct DetailRecord { #[derive(Debug, Clone)] enum ScanUpdate { Batch(Vec), - Finished { - scanned: u64, - matched: u64, - }, + Finished { scanned: u64, matched: u64 }, Failed(String), } @@ -173,9 +166,7 @@ impl ViewApp { } fn handle_key(&mut self, key: KeyEvent) -> Result { - if self.focus == Focus::List - && matches!(key.code, KeyCode::Char('q') | KeyCode::Char('Q')) - { + if self.focus == Focus::List && matches!(key.code, KeyCode::Char('q') | KeyCode::Char('Q')) { self.cancel_scan(); return Ok(true); } @@ -348,11 +339,7 @@ impl ViewApp { let predicate = parse_filter_query(&self.applied_query, self.filter_mode)?; let spool_path = create_temp_path()?; - let spool_reader = OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&spool_path)?; + let spool_reader = OpenOptions::new().read(true).write(true).create_new(true).open(&spool_path)?; let spool_writer = spool_reader.try_clone()?; let cancel = Arc::new(AtomicBool::new(false)); let cancel_worker = Arc::clone(&cancel); @@ -373,15 +360,7 @@ impl ViewApp { }); self.status = format!("Scanning matches for {:?}", self.applied_query); - self.current_scan = Some(ActiveScan { - rx, - cancel, - spool_path, - spool_reader, - scanned: 0, - matched: 0, - finished: false, - }); + self.current_scan = Some(ActiveScan { rx, cancel, spool_path, spool_reader, scanned: 0, matched: 0, finished: false }); Ok(()) } @@ -429,11 +408,7 @@ impl ViewApp { self.save_message = Some("No scan data to save.".to_string()); return Ok(()); }; - let output_dir = self - .input - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")); + let output_dir = self.input.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from(".")); let output_path = output_dir.join(filename); if output_path == self.input || output_path.exists() { self.save_message = Some(format!("File {filename} already exist")); @@ -441,20 +416,12 @@ impl ViewApp { return Ok(()); } - let file = OpenOptions::new() - .write(true) - .create_new(true) - .open(&output_path)?; + let file = OpenOptions::new().write(true).create_new(true).open(&output_path)?; let writer = BufWriter::new(file); let mut logjet = LogjetWriter::with_config(writer, WriterConfig::default()); for meta in &self.entries { let detail = read_spool_record(&mut scan.spool_reader, *meta)?; - logjet.push( - detail.meta.record_type, - detail.meta.seq, - detail.meta.ts_unix_ns, - &detail.payload, - )?; + logjet.push(detail.meta.record_type, detail.meta.seq, detail.meta.ts_unix_ns, &detail.payload)?; } let mut writer = logjet.into_inner()?; writer.flush()?; @@ -496,8 +463,7 @@ impl ViewApp { scan.matched = matched; scan.finished = true; finished = true; - status_override = - Some(format!("Scan complete: {matched} matches out of {scanned} records")); + status_override = Some(format!("Scan complete: {matched} matches out of {scanned} records")); } ScanUpdate::Failed(message) => { scan.finished = true; @@ -559,12 +525,7 @@ impl ViewApp { let meta = self.entries[index]; let detail = read_spool_record(&mut scan.spool_reader, meta)?; let summary = format_summary(&detail, self.hex_payload); - remember_summary( - &mut self.summary_cache, - &mut self.summary_order, - index, - summary.clone(), - ); + remember_summary(&mut self.summary_cache, &mut self.summary_order, index, summary.clone()); Ok(summary) } @@ -588,19 +549,13 @@ impl ViewApp { fn render(&mut self, frame: &mut Frame<'_>) { let areas = Layout::default() .direction(Direction::Vertical) - .constraints([ - Constraint::Length(3), - Constraint::Min(10), - Constraint::Length(1), - ]) + .constraints([Constraint::Length(3), Constraint::Min(10), Constraint::Length(1)]) .split(frame.area()); self.render_search(frame, areas[0]); - let body = Layout::default() - .direction(Direction::Horizontal) - .constraints([Constraint::Percentage(64), Constraint::Percentage(36)]) - .split(areas[1]); + let body = + Layout::default().direction(Direction::Horizontal).constraints([Constraint::Percentage(64), Constraint::Percentage(36)]).split(areas[1]); self.render_list(frame, body[0]); self.render_details(frame, body[1]); @@ -621,9 +576,7 @@ impl ViewApp { FilterMode::Regex => " Filter (regex) ", }; let block = pane_block(title, self.focus == Focus::Search); - let paragraph = Paragraph::new(self.query_input.as_str()) - .block(block) - .style(Style::default().fg(Color::White)); + let paragraph = Paragraph::new(self.query_input.as_str()).block(block).style(Style::default().fg(Color::White)); frame.render_widget(paragraph, area); if self.focus == Focus::Search { @@ -660,36 +613,23 @@ impl ViewApp { let row_width = inner.width.saturating_sub(1) as usize; for index in self.list_offset..end { let style = if index == self.selected { - Style::default() - .fg(Color::White) - .bg(Color::Indexed(28)) - .add_modifier(Modifier::BOLD) + Style::default().fg(Color::White).bg(Color::Indexed(28)).add_modifier(Modifier::BOLD) } else { Style::default().fg(Color::White) }; - let summary = self - .summary_for(index) - .unwrap_or_else(|_| "".to_string()); + let summary = self.summary_for(index).unwrap_or_else(|_| "".to_string()); let summary = fit_to_width(&summary, row_width); lines.push(Line::from(Span::styled(summary, style))); } } - let paragraph = Paragraph::new(Text::from(lines)) - .scroll((0, 0)) - .wrap(Wrap { trim: false }) - .style(Style::default().fg(Color::White)); + let paragraph = Paragraph::new(Text::from(lines)).scroll((0, 0)).wrap(Wrap { trim: false }).style(Style::default().fg(Color::White)); frame.render_widget(paragraph, inner); if !self.entries.is_empty() { - let mut scrollbar_state = ScrollbarState::new(self.entries.len()) - .position(self.selected.min(self.entries.len().saturating_sub(1))); - frame.render_stateful_widget( - Scrollbar::new(ScrollbarOrientation::VerticalRight), - inner, - &mut scrollbar_state, - ); + let mut scrollbar_state = ScrollbarState::new(self.entries.len()).position(self.selected.min(self.entries.len().saturating_sub(1))); + frame.render_stateful_widget(Scrollbar::new(ScrollbarOrientation::VerticalRight), inner, &mut scrollbar_state); } } @@ -704,10 +644,8 @@ impl ViewApp { vec![Line::from("No record selected yet.")] }; - let paragraph = Paragraph::new(Text::from(lines)) - .scroll((self.detail_scroll, 0)) - .wrap(Wrap { trim: false }) - .style(Style::default().fg(Color::White)); + let paragraph = + Paragraph::new(Text::from(lines)).scroll((self.detail_scroll, 0)).wrap(Wrap { trim: false }).style(Style::default().fg(Color::White)); frame.render_widget(paragraph, inner); } @@ -727,28 +665,12 @@ impl ViewApp { area.x, y, area.width, - &[ - status_key("ESC"), - status_text(" to close "), - status_key("UP/DOWN"), - status_text(" scroll"), - ], + &[status_key("ESC"), status_text(" to close "), status_key("UP/DOWN"), status_text(" scroll")], ); return; } if self.focus == Focus::SavePrompt { - draw_status_spans( - buf, - area.x, - y, - area.width, - &[ - status_key("ENTER"), - status_text(" save "), - status_key("ESC"), - status_text(" cancel"), - ], - ); + draw_status_spans(buf, area.x, y, area.width, &[status_key("ENTER"), status_text(" save "), status_key("ESC"), status_text(" cancel")]); return; } if self.focus == Focus::SaveError { @@ -765,15 +687,7 @@ impl ViewApp { if status_width > 0 { let status_x = area.right().saturating_sub(status_width); - buf.set_stringn( - status_x, - y, - status, - status_width as usize, - Style::default() - .fg(Color::LightGreen) - .bg(Color::Indexed(28)), - ); + buf.set_stringn(status_x, y, status, status_width as usize, Style::default().fg(Color::LightGreen).bg(Color::Indexed(28))); } } @@ -781,10 +695,7 @@ impl ViewApp { let area = centered_rect(52, 10, frame.area()); frame.render_widget(Clear, area); let block = Block::default() - .title(Span::styled( - " Save current content ", - Style::default().fg(Color::Black).bg(Color::Gray).add_modifier(Modifier::BOLD), - )) + .title(Span::styled(" Save current content ", Style::default().fg(Color::Black).bg(Color::Gray).add_modifier(Modifier::BOLD))) .borders(Borders::ALL) .border_type(BorderType::Plain) .border_style(Style::default().fg(Color::White).bg(Color::Gray)) @@ -794,19 +705,11 @@ impl ViewApp { let label = "Filename: "; let input_width = inner.width.saturating_sub(label.chars().count() as u16 + 2); - let row = Rect { - x: inner.x, - y: inner.y, - width: inner.width, - height: 1, - }; + let row = Rect { x: inner.x, y: inner.y, width: inner.width, height: 1 }; frame.render_widget( Paragraph::new(Line::from(vec![ Span::styled(label, Style::default().fg(Color::Black).bg(Color::Gray)), - Span::styled( - fit_to_width(&self.save_filename, input_width as usize), - Style::default().fg(Color::Black).bg(Color::White), - ), + Span::styled(fit_to_width(&self.save_filename, input_width as usize), Style::default().fg(Color::Black).bg(Color::White)), ])), row, ); @@ -824,13 +727,7 @@ impl ViewApp { let area = centered_rect(38, 12, frame.area()); frame.render_widget(Clear, area); let block = Block::default() - .title(Span::styled( - " Error ", - Style::default() - .fg(Color::Red) - .bg(Color::White) - .add_modifier(Modifier::BOLD), - )) + .title(Span::styled(" Error ", Style::default().fg(Color::Red).bg(Color::White).add_modifier(Modifier::BOLD))) .borders(Borders::ALL) .border_type(BorderType::Double) .border_style(Style::default().fg(Color::White).bg(Color::Red)) @@ -839,9 +736,7 @@ impl ViewApp { frame.render_widget(block, area); if let Some(message) = &self.save_message { frame.render_widget( - Paragraph::new(render_save_error_message(message)) - .style(Style::default().bg(Color::Red)) - .wrap(Wrap { trim: false }), + Paragraph::new(render_save_error_message(message)).style(Style::default().bg(Color::Red)).wrap(Wrap { trim: false }), inner, ); } @@ -852,13 +747,7 @@ impl ViewApp { frame.render_widget(Clear, area); let block = Block::default() - .title(Span::styled( - " Log record ", - Style::default() - .fg(Color::Black) - .bg(Color::Indexed(30)) - .add_modifier(Modifier::BOLD), - )) + .title(Span::styled(" Log record ", Style::default().fg(Color::Black).bg(Color::Indexed(30)).add_modifier(Modifier::BOLD))) .borders(Borders::ALL) .border_type(BorderType::Double) .border_style(Style::default().fg(Color::Indexed(30)).bg(Color::Gray)) @@ -866,57 +755,31 @@ impl ViewApp { let inner = block.inner(area); frame.render_widget(block, area); - let chunks = Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Min(1), Constraint::Length(1)]) - .split(inner); + let chunks = Layout::default().direction(Direction::Vertical).constraints([Constraint::Min(1), Constraint::Length(1)]).split(inner); let info_entries = if let Some(detail) = &self.selected_detail { render_modal_info_entries(detail) } else { vec![("info".to_string(), "No record loaded.".to_string())] }; - let key_width = info_entries - .iter() - .map(|(key, _)| key.chars().count()) - .max() - .unwrap_or(4) - .max(4); - let preferred_info_width = info_entries - .iter() - .map(|(_, value)| (key_width + 2 + value.chars().count() + 1) as u16) - .max() - .unwrap_or((key_width + 3) as u16); + let key_width = info_entries.iter().map(|(key, _)| key.chars().count()).max().unwrap_or(4).max(4); + let preferred_info_width = + info_entries.iter().map(|(_, value)| (key_width + 2 + value.chars().count() + 1) as u16).max().unwrap_or((key_width + 3) as u16); let max_info_width = chunks[0].width.saturating_div(2).max(16); let info_width = preferred_info_width.min(max_info_width).max(16); let divider_width = 1; - let message_width = chunks[0] - .width - .saturating_sub(info_width) - .saturating_sub(divider_width); + let message_width = chunks[0].width.saturating_sub(info_width).saturating_sub(divider_width); let body = Layout::default() .direction(Direction::Horizontal) - .constraints([ - Constraint::Length(message_width), - Constraint::Length(divider_width), - Constraint::Length(info_width), - ]) + .constraints([Constraint::Length(message_width), Constraint::Length(divider_width), Constraint::Length(info_width)]) .split(chunks[0]); - let divider = (0..body[1].height) - .map(|_| Line::from(Span::styled("│", Style::default().fg(Color::Indexed(30)).bg(Color::Gray)))) - .collect::>(); - frame.render_widget( - Paragraph::new(divider).style(Style::default().bg(Color::Gray)), - body[1], - ); + let divider = + (0..body[1].height).map(|_| Line::from(Span::styled("│", Style::default().fg(Color::Indexed(30)).bg(Color::Gray)))).collect::>(); + frame.render_widget(Paragraph::new(divider).style(Style::default().bg(Color::Gray)), body[1]); - let footer = if let Some(detail) = &self.selected_detail { - render_modal_footer(detail) - } else { - render_modal_footer_placeholder() - }; + let footer = if let Some(detail) = &self.selected_detail { render_modal_footer(detail) } else { render_modal_footer_placeholder() }; let message = self.modal_text.as_deref().unwrap_or("No record loaded."); let paragraph = Paragraph::new(message) .style(Style::default().fg(Color::Black).bg(Color::Gray)) @@ -924,29 +787,16 @@ impl ViewApp { .wrap(Wrap { trim: false }); frame.render_widget(paragraph, body[0]); - let value_width = info_width - .saturating_sub((key_width + 2 + 1) as u16) as usize; - let info_lines = info_entries - .into_iter() - .map(|(key, value)| modal_info_line(&key, value, key_width, value_width)) - .collect::>(); - let info = Paragraph::new(Text::from(info_lines)) - .style(Style::default().fg(Color::Black).bg(Color::Gray)) - .scroll((0, 0)); + let value_width = info_width.saturating_sub((key_width + 2 + 1) as u16) as usize; + let info_lines = info_entries.into_iter().map(|(key, value)| modal_info_line(&key, value, key_width, value_width)).collect::>(); + let info = Paragraph::new(Text::from(info_lines)).style(Style::default().fg(Color::Black).bg(Color::Gray)).scroll((0, 0)); frame.render_widget(info, body[2]); - frame.render_widget( - Paragraph::new(footer).style(Style::default().bg(Color::Indexed(30))), - chunks[1], - ); + frame.render_widget(Paragraph::new(footer).style(Style::default().bg(Color::Indexed(30))), chunks[1]); } } fn scan_matches( - input_path: &Path, - predicate: crate::predicate::RecordPredicate, - mut spool: File, - cancel: Arc, - tx: mpsc::Sender, + input_path: &Path, predicate: crate::predicate::RecordPredicate, mut spool: File, cancel: Arc, tx: mpsc::Sender, ) -> Result<(u64, u64)> { let input = InputHandle::open(input_path)?; let mut reader = LogjetReader::new(input.into_buf_reader()); @@ -958,27 +808,21 @@ fn scan_matches( let Some(record) = reader.next_record()? else { break; }; - scanned = scanned - .checked_add(1) - .ok_or(logjet::Error::NumericOverflow("view scanned"))?; + scanned = scanned.checked_add(1).ok_or(logjet::Error::NumericOverflow("view scanned"))?; if predicate.matches(&record) { let meta = write_spool_record(&mut spool, &record)?; tx_batch.push(meta); - matched = matched - .checked_add(1) - .ok_or(logjet::Error::NumericOverflow("view matched"))?; + matched = matched.checked_add(1).ok_or(logjet::Error::NumericOverflow("view matched"))?; if tx_batch.len() >= SCAN_BATCH_SIZE { - tx.send(ScanUpdate::Batch(std::mem::take(&mut tx_batch))) - .map_err(|err| Error::Usage(err.to_string()))?; + tx.send(ScanUpdate::Batch(std::mem::take(&mut tx_batch))).map_err(|err| Error::Usage(err.to_string()))?; } } } if !tx_batch.is_empty() { - tx.send(ScanUpdate::Batch(tx_batch)) - .map_err(|err| Error::Usage(err.to_string()))?; + tx.send(ScanUpdate::Batch(tx_batch)).map_err(|err| Error::Usage(err.to_string()))?; } Ok((scanned, matched)) @@ -989,19 +833,12 @@ fn write_spool_record(file: &mut File, record: &OwnedRecord) -> Result Result { @@ -1011,12 +848,7 @@ fn read_spool_record(file: &mut File, meta: EntryMeta) -> Result { Ok(DetailRecord { meta, payload }) } -fn remember_summary( - cache: &mut HashMap, - order: &mut VecDeque, - index: usize, - summary: String, -) { +fn remember_summary(cache: &mut HashMap, order: &mut VecDeque, index: usize, summary: String) { cache.insert(index, summary); order.push_back(index); while order.len() > SUMMARY_CACHE_LIMIT { @@ -1027,7 +859,6 @@ fn remember_summary( } fn format_summary(detail: &DetailRecord, hex_payload: bool) -> String { - if hex_payload { hex_preview(&detail.payload, 32) } else if let Some(message) = extract_otlp_log_message(&detail.payload) { @@ -1045,26 +876,14 @@ fn render_detail_lines(detail: &DetailRecord, hex_payload: bool) -> Vec Vec> { scope_count += 1; for log_record in &scope_logs.log_records { record_count += 1; - if !log_record.severity_text.is_empty() - && !severities.iter().any(|existing| existing == &log_record.severity_text) - { + if !log_record.severity_text.is_empty() && !severities.iter().any(|existing| existing == &log_record.severity_text) { severities.push(log_record.severity_text.clone()); } } @@ -1122,18 +939,10 @@ fn render_otlp_lines(detail: &DetailRecord) -> Vec> { ]; if !services.is_empty() { - lines.push(key_value_line( - "Services:", - services.join(", "), - Style::default().fg(Color::White), - )); + lines.push(key_value_line("Services:", services.join(", "), Style::default().fg(Color::White))); } if !severities.is_empty() { - lines.push(key_value_line( - "Severity:", - severities.join(", "), - severity_style(severities.first().map(String::as_str).unwrap_or("")), - )); + lines.push(key_value_line("Severity:", severities.join(", "), severity_style(severities.first().map(String::as_str).unwrap_or("")))); } lines @@ -1160,30 +969,17 @@ fn render_modal_message(detail: &DetailRecord, hex_payload: bool) -> String { return message; } - if hex_payload { - hex_dump(&detail.payload) - } else { - String::from_utf8_lossy(&detail.payload).into_owned() - } + if hex_payload { hex_dump(&detail.payload) } else { String::from_utf8_lossy(&detail.payload).into_owned() } } fn render_modal_footer(detail: &DetailRecord) -> Line<'static> { let (size_num, size_unit) = format_size_parts(detail.meta.payload_len); Line::from(vec![ - Span::styled( - format!("#{}", detail.meta.seq), - Style::default().fg(Color::LightGreen), - ), + Span::styled(format!("#{}", detail.meta.seq), Style::default().fg(Color::LightGreen)), footer_sep(), - Span::styled( - format_timestamp(detail.meta.ts_unix_ns), - Style::default().fg(Color::White), - ), + Span::styled(format_timestamp(detail.meta.ts_unix_ns), Style::default().fg(Color::White)), footer_sep(), - Span::styled( - record_kind_label(detail.meta.record_type).to_string(), - Style::default().fg(Color::Black).add_modifier(Modifier::BOLD), - ), + Span::styled(record_kind_label(detail.meta.record_type).to_string(), Style::default().fg(Color::Black).add_modifier(Modifier::BOLD)), footer_sep(), Span::styled(size_num, Style::default().fg(Color::Yellow)), Span::styled(size_unit, Style::default().fg(Color::Black)), @@ -1253,14 +1049,10 @@ fn render_modal_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { } for record in &scope_logs.log_records { record_attr_count += record.attributes.len(); - if !record.severity_text.is_empty() - && !severities.iter().any(|existing| existing == &record.severity_text) - { + if !record.severity_text.is_empty() && !severities.iter().any(|existing| existing == &record.severity_text) { severities.push(record.severity_text.clone()); } - if !record.event_name.is_empty() - && !event_names.iter().any(|existing| existing == &record.event_name) - { + if !record.event_name.is_empty() && !event_names.iter().any(|existing| existing == &record.event_name) { event_names.push(record.event_name.clone()); } if !record.trace_id.is_empty() { @@ -1302,10 +1094,7 @@ fn render_modal_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { fn modal_info_line(key: &str, value: String, key_width: usize, value_width: usize) -> Line<'static> { let value = trim_single_line(&value, value_width); Line::from(vec![ - Span::styled( - format!("{key: String { } fn key_value_line(label: &str, value: String, value_style: Style) -> Line<'static> { - Line::from(vec![ - Span::styled(format!("{label:<12} "), Style::default().fg(Color::Indexed(136))), - Span::styled(value, value_style), - ]) + Line::from(vec![Span::styled(format!("{label:<12} "), Style::default().fg(Color::Indexed(136))), Span::styled(value, value_style)]) } fn severity_style(value: &str) -> Style { @@ -1407,10 +1193,7 @@ fn format_timestamp(ts_unix_ns: u64) -> String { fn hex_preview(bytes: &[u8], limit: usize) -> String { let shown = bytes.iter().take(limit); - let mut out = shown - .map(|byte| format!("{byte:02x}")) - .collect::>() - .join(" "); + let mut out = shown.map(|byte| format!("{byte:02x}")).collect::>().join(" "); if bytes.len() > limit { out.push_str(" ..."); } @@ -1449,16 +1232,8 @@ fn centered_rect(width_percent: u16, height_percent: u16, area: Rect) -> Rect { } fn pane_block<'a>(title: &'a str, active: bool) -> Block<'a> { - let title_style = if active { - Style::default().fg(Color::Black).bg(Color::LightGreen) - } else { - Style::default().fg(Color::Gray) - }; - let border_style = if active { - Style::default().fg(Color::LightGreen) - } else { - Style::default().fg(Color::Gray) - }; + let title_style = if active { Style::default().fg(Color::Black).bg(Color::LightGreen) } else { Style::default().fg(Color::Gray) }; + let border_style = if active { Style::default().fg(Color::LightGreen) } else { Style::default().fg(Color::Gray) }; Block::default() .title(Span::styled(title, title_style)) @@ -1499,29 +1274,14 @@ fn status_help_spans(focus: Focus) -> Vec> { } fn status_key(text: &str) -> Span<'static> { - Span::styled( - text.to_string(), - Style::default() - .fg(Color::White) - .bg(Color::Indexed(28)) - .add_modifier(Modifier::BOLD), - ) + Span::styled(text.to_string(), Style::default().fg(Color::White).bg(Color::Indexed(28)).add_modifier(Modifier::BOLD)) } fn status_text(text: &str) -> Span<'static> { - Span::styled( - text.to_string(), - Style::default().fg(Color::Black).bg(Color::Indexed(28)), - ) + Span::styled(text.to_string(), Style::default().fg(Color::Black).bg(Color::Indexed(28))) } -fn draw_status_spans( - buf: &mut ratatui::buffer::Buffer, - x: u16, - y: u16, - width: u16, - spans: &[Span<'static>], -) { +fn draw_status_spans(buf: &mut ratatui::buffer::Buffer, x: u16, y: u16, width: u16, spans: &[Span<'static>]) { let mut cursor_x = x; let mut remaining = width; for span in spans { @@ -1538,36 +1298,21 @@ fn render_save_error_message(message: &str) -> Line<'static> { const PREFIX: &str = "File "; const SUFFIX: &str = " already exist"; - if let Some(filename) = message - .strip_prefix(PREFIX) - .and_then(|rest| rest.strip_suffix(SUFFIX)) - { + if let Some(filename) = message.strip_prefix(PREFIX).and_then(|rest| rest.strip_suffix(SUFFIX)) { return Line::from(vec![ Span::styled(PREFIX, Style::default().fg(Color::White).bg(Color::Red)), - Span::styled( - filename.to_string(), - Style::default() - .fg(Color::Yellow) - .bg(Color::Red) - .add_modifier(Modifier::BOLD), - ), + Span::styled(filename.to_string(), Style::default().fg(Color::Yellow).bg(Color::Red).add_modifier(Modifier::BOLD)), Span::styled(SUFFIX, Style::default().fg(Color::White).bg(Color::Red)), ]); } - Line::from(Span::styled( - message.to_string(), - Style::default().fg(Color::White).bg(Color::Red), - )) + Line::from(Span::styled(message.to_string(), Style::default().fg(Color::White).bg(Color::Red))) } fn create_temp_path() -> Result { let base = std::env::temp_dir(); let pid = std::process::id(); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|err| Error::Usage(format!("system clock error: {err}")))? - .as_nanos(); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|err| Error::Usage(format!("system clock error: {err}")))?.as_nanos(); for attempt in 0..1000u32 { let candidate = base.join(format!("ljx-view-{pid}-{nanos}-{attempt}.tmp")); if !candidate.exists() { @@ -1579,10 +1324,7 @@ fn create_temp_path() -> Result { #[cfg(test)] mod tests { - use super::{ - DetailRecord, EntryMeta, extract_otlp_log_message, format_summary, - render_modal_message, text_preview, - }; + use super::{DetailRecord, EntryMeta, extract_otlp_log_message, format_summary, render_modal_message, text_preview}; use logjet::RecordType; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use opentelemetry_proto::tonic::common::v1::any_value::Value; @@ -1599,13 +1341,7 @@ mod tests { #[test] fn summary_uses_trimmed_single_line_preview() { let detail = DetailRecord { - meta: EntryMeta { - offset: 0, - record_type: RecordType::Logs, - seq: 7, - ts_unix_ns: 9, - payload_len: 13, - }, + meta: EntryMeta { offset: 0, record_type: RecordType::Logs, seq: 7, ts_unix_ns: 9, payload_len: 13 }, payload: b"line one\nline two".to_vec(), }; let summary = format_summary(&detail, false); @@ -1616,10 +1352,7 @@ mod tests { fn summary_prefers_decoded_otlp_log_message() { let batch = ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: Vec::new(), - dropped_attributes_count: 0, - }), + resource: Some(Resource { attributes: Vec::new(), dropped_attributes_count: 0 }), scope_logs: vec![ScopeLogs { scope: Some(InstrumentationScope { name: "test".to_string(), @@ -1632,9 +1365,7 @@ mod tests { observed_time_unix_nano: 0, severity_number: 0, severity_text: String::new(), - body: Some(AnyValue { - value: Some(Value::StringValue("hello from body".to_string())), - }), + body: Some(AnyValue { value: Some(Value::StringValue("hello from body".to_string())) }), attributes: Vec::new(), dropped_attributes_count: 0, flags: 0, @@ -1649,13 +1380,7 @@ mod tests { }; let payload = batch.encode_to_vec(); let detail = DetailRecord { - meta: EntryMeta { - offset: 0, - record_type: RecordType::Logs, - seq: 1, - ts_unix_ns: 2, - payload_len: payload.len() as u64, - }, + meta: EntryMeta { offset: 0, record_type: RecordType::Logs, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64 }, payload, }; @@ -1666,13 +1391,7 @@ mod tests { #[test] fn modal_falls_back_to_raw_payload() { let detail = DetailRecord { - meta: EntryMeta { - offset: 0, - record_type: RecordType::Metrics, - seq: 1, - ts_unix_ns: 2, - payload_len: 5, - }, + meta: EntryMeta { offset: 0, record_type: RecordType::Metrics, seq: 1, ts_unix_ns: 2, payload_len: 5 }, payload: b"hello".to_vec(), }; let body = render_modal_message(&detail, false); diff --git a/ljx/src/input.rs b/ljx/src/input.rs index caaaeb0..3ba6b7a 100644 --- a/ljx/src/input.rs +++ b/ljx/src/input.rs @@ -12,14 +12,7 @@ pub struct InputHandle { impl InputHandle { pub fn open(path: &Path) -> Result { - if path == Path::new("-") { - Self::from_stdin() - } else { - Ok(Self { - file: File::open(path)?, - temp_path: None, - }) - } + if path == Path::new("-") { Self::from_stdin() } else { Ok(Self { file: File::open(path)?, temp_path: None }) } } pub fn into_buf_reader(self) -> BufReader { @@ -28,11 +21,7 @@ impl InputHandle { fn from_stdin() -> Result { let path = create_temp_path()?; - let file = OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&path)?; + let file = OpenOptions::new().read(true).write(true).create_new(true).open(&path)?; let mut writer = BufWriter::new(file); let mut stdin = io::stdin().lock(); @@ -41,10 +30,7 @@ impl InputHandle { let mut file = writer.into_inner().map_err(io::Error::other)?; file.seek(SeekFrom::Start(0))?; - Ok(Self { - file, - temp_path: Some(path), - }) + Ok(Self { file, temp_path: Some(path) }) } } @@ -69,37 +55,24 @@ impl Drop for InputHandle { } pub fn open_output(path: &Path) -> Result> { - if path == Path::new("-") { - Ok(Box::new(BufWriter::new(io::stdout().lock()))) - } else { - Ok(Box::new(BufWriter::new(File::create(path)?))) - } + if path == Path::new("-") { Ok(Box::new(BufWriter::new(io::stdout().lock()))) } else { Ok(Box::new(BufWriter::new(File::create(path)?))) } } fn create_temp_path() -> Result { let mut attempt = 0u32; let base = std::env::temp_dir(); let pid = std::process::id(); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|err| Error::Usage(format!("system clock error: {err}")))? - .as_nanos(); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|err| Error::Usage(format!("system clock error: {err}")))?.as_nanos(); loop { let candidate = base.join(format!("ljx-stdin-{pid}-{nanos}-{attempt}.logjet")); - match OpenOptions::new() - .write(true) - .create_new(true) - .open(&candidate) - { + match OpenOptions::new().write(true).create_new(true).open(&candidate) { Ok(file) => { drop(file); return Ok(candidate); } Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { - attempt = attempt - .checked_add(1) - .ok_or(Error::Usage("temporary file naming overflow".to_string()))?; + attempt = attempt.checked_add(1).ok_or(Error::Usage("temporary file naming overflow".to_string()))?; } Err(err) => return Err(err.into()), } diff --git a/ljx/src/main.rs b/ljx/src/main.rs index ce3f762..be7cd82 100644 --- a/ljx/src/main.rs +++ b/ljx/src/main.rs @@ -19,8 +19,7 @@ fn main() { fn run() -> Result<()> { let mut command = cli::build_cli(); let mut matches = command.get_matches_mut(); - let cli = Cli::from_arg_matches_mut(&mut matches) - .map_err(|err| crate::error::Error::Usage(err.to_string()))?; + let cli = Cli::from_arg_matches_mut(&mut matches).map_err(|err| crate::error::Error::Usage(err.to_string()))?; match cli.command { Command::Split(args) => commands::split::run(args), Command::Join(args) => commands::join::run(args), diff --git a/ljx/src/predicate.rs b/ljx/src/predicate.rs index 2ed0de5..26284f3 100644 --- a/ljx/src/predicate.rs +++ b/ljx/src/predicate.rs @@ -64,9 +64,7 @@ impl PredicateArgs { (None, Some(text)) => Some(PayloadMatcher::new(&text, true, self.ignore_case)?), (None, None) => None, (Some(_), Some(_)) => { - return Err(Error::Usage( - "choose either -e/--grep or -F/--fixed-string, not both".to_string(), - )); + return Err(Error::Usage("choose either -e/--grep or -F/--fixed-string, not both".to_string())); } }; @@ -96,52 +94,52 @@ pub fn parse_filter_query(query: &str, mode: FilterMode) -> Result PredicateArgs { - fixed_string: Some(trimmed.to_string()), - ..PredicateArgs::default() - } - .build(), - FilterMode::Regex => PredicateArgs { - grep: Some(trimmed.to_string()), - ..PredicateArgs::default() - } - .build(), + FilterMode::Strings => PredicateArgs { fixed_string: Some(trimmed.to_string()), ..PredicateArgs::default() }.build(), + FilterMode::Regex => PredicateArgs { grep: Some(trimmed.to_string()), ..PredicateArgs::default() }.build(), }; } - let argv = shlex::split(trimmed) - .ok_or_else(|| Error::Usage("invalid filter expression: unterminated quotes".to_string()))?; + let argv = shlex::split(trimmed).ok_or_else(|| Error::Usage("invalid filter expression: unterminated quotes".to_string()))?; let mut full_argv = Vec::with_capacity(argv.len() + 1); full_argv.push("view-filter".to_string()); full_argv.extend(argv); let mut command = PredicateCli::command(); - let mut matches = command - .try_get_matches_from_mut(full_argv) - .map_err(|err| Error::Usage(err.to_string()))?; - let parsed = - PredicateCli::from_arg_matches_mut(&mut matches).map_err(|err| Error::Usage(err.to_string()))?; + let mut matches = command.try_get_matches_from_mut(full_argv).map_err(|err| Error::Usage(err.to_string()))?; + let parsed = PredicateCli::from_arg_matches_mut(&mut matches).map_err(|err| Error::Usage(err.to_string()))?; parsed.predicate.build() } impl RecordPredicate { pub fn matches(&self, record: &OwnedRecord) -> bool { - if let Some(expected) = self.record_type && record.record_type != expected { + if let Some(expected) = self.record_type + && record.record_type != expected + { return false; } - if let Some(min) = self.seq_min && record.seq < min { + if let Some(min) = self.seq_min + && record.seq < min + { return false; } - if let Some(max) = self.seq_max && record.seq > max { + if let Some(max) = self.seq_max + && record.seq > max + { return false; } - if let Some(min) = self.ts_min && record.ts_unix_ns < min { + if let Some(min) = self.ts_min + && record.ts_unix_ns < min + { return false; } - if let Some(max) = self.ts_max && record.ts_unix_ns > max { + if let Some(max) = self.ts_max + && record.ts_unix_ns > max + { return false; } - if let Some(matcher) = &self.payload_matcher && !matcher.is_match(&record.payload) { + if let Some(matcher) = &self.payload_matcher + && !matcher.is_match(&record.payload) + { return false; } @@ -151,11 +149,7 @@ impl RecordPredicate { impl PayloadMatcher { fn new(pattern: &str, fixed_string: bool, ignore_case: bool) -> Result { - let source = if fixed_string { - regex::escape(pattern) - } else { - pattern.to_string() - }; + let source = if fixed_string { regex::escape(pattern) } else { pattern.to_string() }; let regex = RegexBuilder::new(&source) .case_insensitive(ignore_case) .build() @@ -191,22 +185,12 @@ mod tests { use logjet::{OwnedRecord, RecordType}; fn sample_record(payload: &[u8]) -> OwnedRecord { - OwnedRecord { - record_type: RecordType::Logs, - seq: 42, - ts_unix_ns: 1_700_000_000, - payload: payload.to_vec(), - } + OwnedRecord { record_type: RecordType::Logs, seq: 42, ts_unix_ns: 1_700_000_000, payload: payload.to_vec() } } #[test] fn fixed_string_match_is_literal() { - let predicate = PredicateArgs { - fixed_string: Some("java.crap.failed".to_string()), - ..PredicateArgs::default() - } - .build() - .unwrap(); + let predicate = PredicateArgs { fixed_string: Some("java.crap.failed".to_string()), ..PredicateArgs::default() }.build().unwrap(); assert!(predicate.matches(&sample_record(b"xxx java.crap.failed yyy"))); assert!(!predicate.matches(&sample_record(b"javaXcrapXfailed"))); @@ -214,12 +198,7 @@ mod tests { #[test] fn regex_match_supports_wildcards() { - let predicate = PredicateArgs { - grep: Some(r"java\..*\.bs".to_string()), - ..PredicateArgs::default() - } - .build() - .unwrap(); + let predicate = PredicateArgs { grep: Some(r"java\..*\.bs".to_string()), ..PredicateArgs::default() }.build().unwrap(); assert!(predicate.matches(&sample_record(b"java.very.long.bs"))); assert!(!predicate.matches(&sample_record(b"java.very.long.cs"))); @@ -227,20 +206,8 @@ mod tests { #[test] fn ignore_case_applies_to_fixed_string_and_regex() { - let fixed = PredicateArgs { - fixed_string: Some("error".to_string()), - ignore_case: true, - ..PredicateArgs::default() - } - .build() - .unwrap(); - let regex = PredicateArgs { - grep: Some("error".to_string()), - ignore_case: true, - ..PredicateArgs::default() - } - .build() - .unwrap(); + let fixed = PredicateArgs { fixed_string: Some("error".to_string()), ignore_case: true, ..PredicateArgs::default() }.build().unwrap(); + let regex = PredicateArgs { grep: Some("error".to_string()), ignore_case: true, ..PredicateArgs::default() }.build().unwrap(); let record = sample_record(b"prefix eRrOr suffix"); assert!(fixed.matches(&record)); @@ -267,12 +234,7 @@ mod tests { #[test] fn invalid_regex_is_reported() { - let error = PredicateArgs { - grep: Some("(".to_string()), - ..PredicateArgs::default() - } - .build() - .unwrap_err(); + let error = PredicateArgs { grep: Some("(".to_string()), ..PredicateArgs::default() }.build().unwrap_err(); assert!(error.to_string().contains("invalid payload matcher")); } diff --git a/logjetd/Cargo.toml b/logjetd/Cargo.toml index 330cefc..52c8ef2 100644 --- a/logjetd/Cargo.toml +++ b/logjetd/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "logjetd" +name = "ljd" version = "0.1.0" edition = "2024" license = "Apache-2.0" diff --git a/logjetd/src/config_utst.rs b/logjetd/src/config_utst.rs index d9bb7ec..ef155e2 100644 --- a/logjetd/src/config_utst.rs +++ b/logjetd/src/config_utst.rs @@ -226,5 +226,5 @@ fn write_temp_config(label: &str, body: &str) -> PathBuf { fn unique_temp_path(label: &str) -> PathBuf { let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); - std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.yaml", std::process::id())) + std::env::temp_dir().join(format!("ljd-{label}-{nanos}-{}.yaml", std::process::id())) } diff --git a/logjetd/src/daemon.rs b/logjetd/src/daemon.rs index 39fcf4b..28281e8 100644 --- a/logjetd/src/daemon.rs +++ b/logjetd/src/daemon.rs @@ -167,7 +167,7 @@ fn maybe_report_overload(config: &IngestOverloadConfig, state: &mut IngestPolicy return; } eprintln!( - "logjetd ingest overload stats accepted={} priority-bypass={} rate-limited={} oversize-rejected={} client-cap-rejected={}", + "ljd ingest overload stats accepted={} priority-bypass={} rate-limited={} oversize-rejected={} client-cap-rejected={}", state.stats.accepted, state.stats.priority_bypass, state.stats.rate_limited, state.stats.oversize_rejected, state.stats.client_cap_rejected ); state.next_report_at = now + Duration::from_millis(config.report_every_ms.max(1)); @@ -198,10 +198,10 @@ pub fn serve(config: DaemonConfig) -> io::Result<()> { let tls = config.config.tls.clone(); let replay_thread = thread::Builder::new() - .name("logjetd-replay".to_string()) + .name("ljd-replay".to_string()) .spawn(move || replay_loop(replay_addr, replay_spool, replay_max_clients, replay_client_timeout_ms, tls))?; - eprintln!("logjetd using config {}", config.config_path.display()); + eprintln!("ljd using config {}", config.config_path.display()); ingest_loop( config.config.ingest_addr, config.config.ingest_protocol, @@ -223,7 +223,7 @@ fn ingest_loop( IngestProtocol::Wire => { let listener = TcpListener::bind(&bind_addr)?; eprintln!( - "logjetd ingest listening on {bind_addr} using wire protocol max-batch-bytes={} max-clients={}", + "ljd ingest listening on {bind_addr} using wire protocol max-batch-bytes={} max-clients={}", ingest_limits.max_batch_bytes, ingest_limits.max_clients ); @@ -233,9 +233,9 @@ fn ingest_loop( let ingest_policy = Arc::clone(&ingest_policy); let limiter = Arc::clone(&limiter); let max_batch_bytes = ingest_limits.max_batch_bytes; - thread::Builder::new().name("logjetd-ingest-client".to_string()).spawn(move || { + thread::Builder::new().name("ljd-ingest-client".to_string()).spawn(move || { if let Err(err) = handle_ingest_client(stream, spool, ingest_policy, limiter, max_batch_bytes) { - eprintln!("logjetd ingest client error: {err}"); + eprintln!("ljd ingest client error: {err}"); } })?; } @@ -245,7 +245,7 @@ fn ingest_loop( return otlp_http_tls_loop(bind_addr, ingest_tls, ingest_limits, ingest_policy, spool, next_seq, limiter); } let server = Server::http(&bind_addr).map_err(|err| io::Error::other(err.to_string()))?; - eprintln!("logjetd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes); + eprintln!("ljd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes); for mut request in server.incoming_requests() { if request.method() != &Method::Post || request.url() != "/v1/logs" { @@ -292,7 +292,7 @@ fn ingest_loop( let addr: SocketAddr = bind_addr.parse().map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, format!("invalid gRPC bind addr: {err}")))?; eprintln!( - "logjetd ingest listening on {}://{bind_addr} using otlp-grpc max-batch-bytes={} max-clients={}", + "ljd ingest listening on {}://{bind_addr} using otlp-grpc max-batch-bytes={} max-clients={}", if ingest_tls.enable { "grpcs" } else { "grpc" }, ingest_limits.max_batch_bytes, ingest_limits.max_clients @@ -327,7 +327,7 @@ fn otlp_http_tls_loop( let listener = TcpListener::bind(&bind_addr)?; let tls_server = load_ingest_server_config(&ingest_tls)?; eprintln!( - "logjetd ingest listening on https://{bind_addr}/v1/logs using otlp-http max-batch-bytes={} max-clients={}", + "ljd ingest listening on https://{bind_addr}/v1/logs using otlp-http max-batch-bytes={} max-clients={}", ingest_limits.max_batch_bytes, ingest_limits.max_clients ); @@ -339,9 +339,9 @@ fn otlp_http_tls_loop( let tls_server = tls_server.clone(); let limiter = Arc::clone(&limiter); let max_batch_bytes = ingest_limits.max_batch_bytes; - thread::Builder::new().name("logjetd-otlp-http-tls-client".to_string()).spawn(move || { + thread::Builder::new().name("ljd-otlp-http-tls-client".to_string()).spawn(move || { if let Err(err) = handle_otlp_http_tls_client(stream, tls_server, spool, ingest_policy, next_seq, limiter, max_batch_bytes) { - eprintln!("logjetd otlp-http tls client error: {err}"); + eprintln!("ljd otlp-http tls client error: {err}"); } })?; } @@ -354,7 +354,7 @@ fn handle_otlp_http_tls_client( limiter: Arc, max_batch_bytes: usize, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { - eprintln!("logjetd ingest refused TLS client: ingest.max-clients reached"); + eprintln!("ljd ingest refused TLS client: ingest.max-clients reached"); ingest_policy.note_client_cap()?; return Ok(()); }; @@ -514,12 +514,12 @@ fn replay_loop( let listener = TcpListener::bind(&bind_addr)?; let limiter = Arc::new(ConnectionLimiter::new(max_clients)); let tls_server = if tls.enable { - eprintln!("logjetd replay TLS enabled on {bind_addr}"); + eprintln!("ljd replay TLS enabled on {bind_addr}"); Some(load_server_config(&tls)?) } else { None }; - eprintln!("logjetd replay listening on {bind_addr} max-clients={max_clients} client-timeout-ms={client_timeout_ms}"); + eprintln!("ljd replay listening on {bind_addr} max-clients={max_clients} client-timeout-ms={client_timeout_ms}"); for stream in listener.incoming() { let stream = stream?; @@ -529,9 +529,9 @@ fn replay_loop( let spool = Arc::clone(&spool); let tls_server = tls_server.clone(); let limiter = Arc::clone(&limiter); - thread::Builder::new().name("logjetd-replay-client".to_string()).spawn(move || { + thread::Builder::new().name("ljd-replay-client".to_string()).spawn(move || { if let Err(err) = handle_replay_client(stream, spool, tls_server, limiter) { - eprintln!("logjetd replay client error: {err}"); + eprintln!("ljd replay client error: {err}"); } })?; } @@ -543,7 +543,7 @@ fn handle_ingest_client( stream: TcpStream, spool: Arc, ingest_policy: Arc, limiter: Arc, max_batch_bytes: usize, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { - eprintln!("logjetd ingest refused wire client: ingest.max-clients reached"); + eprintln!("ljd ingest refused wire client: ingest.max-clients reached"); ingest_policy.note_client_cap()?; return Ok(()); }; @@ -552,14 +552,14 @@ fn handle_ingest_client( while let Some(record) = read_record_with_limit(&mut reader, max_batch_bytes)? { if matches!(ingest_policy.decide(BatchPriority::Unknown)?, IngestDecision::RejectRateLimited) { - eprintln!("logjetd ingest dropped wire record seq={} because ingest rate limit was exceeded", record.seq); + eprintln!("ljd ingest dropped wire record seq={} because ingest rate limit was exceeded", record.seq); continue; } append_batch_record(&spool, record)?; } if let Some(peer) = peer { - eprintln!("logjetd ingest disconnected: {peer}"); + eprintln!("ljd ingest disconnected: {peer}"); } Ok(()) } @@ -601,7 +601,7 @@ fn handle_replay_client( stream: TcpStream, spool: Arc, tls_server: Option>, limiter: Arc, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { - eprintln!("logjetd replay refused client: replay.max-clients reached"); + eprintln!("ljd replay refused client: replay.max-clients reached"); return Ok(()); }; if let Some(server_config) = tls_server { @@ -629,7 +629,7 @@ fn handle_replay_transport(transport: &mut T, spool: Ar (spool_guard.replay_cursor_after(request.from_seq)?, spool.current_generation()?) }; - eprintln!("logjetd replay client requested records after seq={} mode={}", request.from_seq, if request.consume { "drain" } else { "keep" }); + eprintln!("ljd replay client requested records after seq={} mode={}", request.from_seq, if request.consume { "drain" } else { "keep" }); if request.consume { return handle_replay_transport_drain(transport, spool, &mut cursor, &mut seen_generation); diff --git a/logjetd/src/main.rs b/logjetd/src/main.rs index 5b09320..cf4653d 100644 --- a/logjetd/src/main.rs +++ b/logjetd/src/main.rs @@ -18,7 +18,7 @@ fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("logjetd: {err}"); + eprintln!("ljd: {err}"); ExitCode::FAILURE } } @@ -176,15 +176,15 @@ fn run() -> Result<(), Box> { } fn print_usage() { - println!("logjetd"); + println!("ljd"); println!(); println!("Usage:"); - println!(" logjetd [serve] [-c|--config ]"); - println!(" logjetd inspect "); - println!(" logjetd segments --path --name "); - println!(" logjetd replay [-c|--config ] --path --name [--dest ]"); - println!(" logjetd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]"); - println!(" logjetd bridge [-c|--config ] [--source ]"); + println!(" ljd [serve] [-c|--config ]"); + println!(" ljd inspect "); + println!(" ljd segments --path --name "); + println!(" ljd replay [-c|--config ] --path --name [--dest ]"); + println!(" ljd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]"); + println!(" ljd bridge [-c|--config ] [--source ]"); println!(); println!("Commands:"); println!(" serve Run ingest and replay listeners using YAML configuration"); diff --git a/logjetd/src/replay_utst.rs b/logjetd/src/replay_utst.rs index 91d9048..a1800ea 100644 --- a/logjetd/src/replay_utst.rs +++ b/logjetd/src/replay_utst.rs @@ -123,5 +123,5 @@ fn test_collector_transport(mode: BackpressureMode, max_buffered_records: usize) fn unique_temp_path(label: &str) -> PathBuf { let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); - std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.state", std::process::id())) + std::env::temp_dir().join(format!("ljd-{label}-{nanos}-{}.state", std::process::id())) } diff --git a/logjetd/src/spool_utst.rs b/logjetd/src/spool_utst.rs index 25d43e9..e1fae41 100644 --- a/logjetd/src/spool_utst.rs +++ b/logjetd/src/spool_utst.rs @@ -273,7 +273,7 @@ fn prune_named_segments_dry_run_does_not_remove_files() { fn unique_temp_dir(label: &str) -> PathBuf { let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); - let dir = std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("ljd-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&dir).unwrap(); dir } diff --git a/logjetd/tests/bridge_flows.rs b/logjetd/tests/bridge_flows.rs index 44ced52..683665a 100644 --- a/logjetd/tests/bridge_flows.rs +++ b/logjetd/tests/bridge_flows.rs @@ -7,7 +7,7 @@ use std::thread; use std::time::Duration; use common::{ - ChildGuard, MockCollector, TestDir, connect_replay_client, free_port, logjetd_command, post_otlp_http, read_replay_message, replay_messages, + ChildGuard, MockCollector, TestDir, connect_replay_client, free_port, ljd_command, post_otlp_http, read_replay_message, replay_messages, wait_for_tcp, wait_until, }; @@ -30,7 +30,7 @@ fn bridge_keep_forwards_backlog_in_order() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -43,7 +43,7 @@ fn bridge_keep_forwards_backlog_in_order() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -76,7 +76,7 @@ fn bridge_drain_consumes_upstream_records() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -89,7 +89,7 @@ fn bridge_drain_consumes_upstream_records() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -125,7 +125,7 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -140,7 +140,7 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { { let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -153,7 +153,7 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { { let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -197,7 +197,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -211,7 +211,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -268,14 +268,14 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; { let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_alpha).arg("serve"); cmd })?; @@ -290,7 +290,7 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { { let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_bravo).arg("serve"); cmd })?; @@ -327,7 +327,7 @@ fn bridge_block_mode_handles_slow_collector_without_losing_order() -> io::Result )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -336,7 +336,7 @@ fn bridge_block_mode_handles_slow_collector_without_losing_order() -> io::Result let collector = MockCollector::start_with_delay(collector_port, Duration::from_millis(150))?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -379,7 +379,7 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { { let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -402,7 +402,7 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let status = { - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("replay").arg("--path").arg(&spool_dir).arg("--name").arg("recover.logjet").arg("--dest").arg(format!("127.0.0.1:{collector_port}")); cmd.status()? }; @@ -435,7 +435,7 @@ fn bridge_forwards_large_payloads_end_to_end() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -444,7 +444,7 @@ fn bridge_forwards_large_payloads_end_to_end() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let _bridge = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; @@ -472,7 +472,7 @@ fn multiple_replay_clients_receive_backlog_independently() -> io::Result<()> { )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; @@ -513,7 +513,7 @@ fn replay_client_receives_backlog_then_live_records_without_reconnect() -> io::R )?; let _appliance = ChildGuard::spawn({ - let mut cmd = logjetd_command(); + let mut cmd = ljd_command(); cmd.arg("--config").arg(&appliance_config).arg("serve"); cmd })?; diff --git a/logjetd/tests/common/mod.rs b/logjetd/tests/common/mod.rs index e350b5a..ffb4bdf 100644 --- a/logjetd/tests/common/mod.rs +++ b/logjetd/tests/common/mod.rs @@ -24,7 +24,7 @@ pub struct TestDir { impl TestDir { pub fn new(label: &str) -> io::Result { let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); - let path = std::env::temp_dir().join(format!("logjetd-it-{label}-{nanos}-{}", std::process::id())); + let path = std::env::temp_dir().join(format!("ljd-it-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&path)?; Ok(Self { path }) } @@ -64,8 +64,8 @@ impl Drop for ChildGuard { } } -pub fn logjetd_command() -> Command { - Command::new(env!("CARGO_BIN_EXE_logjetd")) +pub fn ljd_command() -> Command { + Command::new(env!("CARGO_BIN_EXE_ljd")) } pub fn free_port() -> io::Result { diff --git a/tests/ljx_cli.rs b/tests/ljx_cli.rs index 331aeb7..2c941a6 100644 --- a/tests/ljx_cli.rs +++ b/tests/ljx_cli.rs @@ -13,7 +13,7 @@ use opentelemetry_proto::tonic::common::v1::any_value::Value; use prost::Message; #[test] -fn ljx_filters_real_logjetd_output_from_mock_emitter() -> io::Result<()> { +fn ljx_filters_real_ljd_output_from_mock_emitter() -> io::Result<()> { ensure_test_binaries_exist()?; let dir = TestDir::new("ljx-cli")?; @@ -34,33 +34,25 @@ fn ljx_filters_real_logjetd_output_from_mock_emitter() -> io::Result<()> { ), )?; - eprintln!("starting logjetd"); + eprintln!("starting ljd"); let _daemon = ChildGuard::spawn({ - let mut cmd = Command::new(logjetd_bin()); + let mut cmd = Command::new(ljd_bin()); cmd.arg("--config").arg(&config).arg("serve"); cmd }) - .map_err(|err| io::Error::other(format!("failed to start logjetd: {err}")))?; + .map_err(|err| io::Error::other(format!("failed to start ljd: {err}")))?; eprintln!("waiting for ingest tcp"); wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5)) .map_err(|err| io::Error::other(format!("failed waiting for ingest tcp: {err}")))?; - for message in [ - "java.crap.failed", - "java.alpha.bs", - "ERROR boom", - "eRrOr splash", - "banana", - ] { + for message in ["java.crap.failed", "java.alpha.bs", "ERROR boom", "eRrOr splash", "banana"] { eprintln!("emitting {message}"); run_emitter(ingest_port, message)?; } eprintln!("waiting for spool file"); - wait_until(Duration::from_secs(5), || { - Ok(spool_path.exists() && fs::metadata(&spool_path)?.len() > 0) - }) - .map_err(|err| io::Error::other(format!("failed waiting for spool file: {err}")))?; + wait_until(Duration::from_secs(5), || Ok(spool_path.exists() && fs::metadata(&spool_path)?.len() > 0)) + .map_err(|err| io::Error::other(format!("failed waiting for spool file: {err}")))?; let records = read_logjet_records(&spool_path)?; assert_eq!(records.len(), 5); @@ -72,100 +64,29 @@ fn ljx_filters_real_logjetd_output_from_mock_emitter() -> io::Result<()> { eprintln!("running ljx assertions"); assert_eq!(run_ljx_count(&spool_path, &[])?, "5"); assert_eq!(run_ljx_count(&spool_path, &["--type", "logs"])?, "5"); - assert_eq!( - run_ljx_count(&spool_path, &["-F", "java.crap.failed"])?, - "1" - ); - assert_eq!( - run_ljx_count(&spool_path, &["-e", r"java\..*\.bs"])?, - "1" - ); + assert_eq!(run_ljx_count(&spool_path, &["-F", "java.crap.failed"])?, "1"); + assert_eq!(run_ljx_count(&spool_path, &["-e", r"java\..*\.bs"])?, "1"); assert_eq!(run_ljx_count(&spool_path, &["-F", "error", "-i"])?, "2"); - assert_eq!( - run_ljx_count( - &spool_path, - &[ - "--seq-min", - &seq_min.to_string(), - "--seq-max", - &seq_max.to_string(), - ], - )?, - "3" - ); - assert_eq!( - run_ljx_count( - &spool_path, - &[ - "--ts-min", - &ts_min.to_string(), - "--ts-max", - &ts_max.to_string(), - ], - )?, - "3" - ); + assert_eq!(run_ljx_count(&spool_path, &["--seq-min", &seq_min.to_string(), "--seq-max", &seq_max.to_string(),],)?, "3"); + assert_eq!(run_ljx_count(&spool_path, &["--ts-min", &ts_min.to_string(), "--ts-max", &ts_max.to_string(),],)?, "3"); - run_ljx_filter( - &spool_path, - &filtered_literal, - &["-F", "java.crap.failed"], - )?; - assert_eq!( - read_logjet_messages(&filtered_literal)?, - vec!["java.crap.failed".to_string()] - ); + run_ljx_filter(&spool_path, &filtered_literal, &["-F", "java.crap.failed"])?; + assert_eq!(read_logjet_messages(&filtered_literal)?, vec!["java.crap.failed".to_string()]); run_ljx_filter(&spool_path, &filtered_regex, &["-e", "error|panic", "-i"])?; - assert_eq!( - read_logjet_messages(&filtered_regex)?, - vec!["ERROR boom".to_string(), "eRrOr splash".to_string()] - ); + assert_eq!(read_logjet_messages(&filtered_regex)?, vec!["ERROR boom".to_string(), "eRrOr splash".to_string()]); - run_ljx_filter( - &spool_path, - &filtered_seq, - &[ - "--seq-min", - &seq_min.to_string(), - "--seq-max", - &seq_max.to_string(), - ], - )?; - assert_eq!( - read_logjet_messages(&filtered_seq)?, - vec![ - "java.alpha.bs".to_string(), - "ERROR boom".to_string(), - "eRrOr splash".to_string() - ] - ); + run_ljx_filter(&spool_path, &filtered_seq, &["--seq-min", &seq_min.to_string(), "--seq-max", &seq_max.to_string()])?; + assert_eq!(read_logjet_messages(&filtered_seq)?, vec!["java.alpha.bs".to_string(), "ERROR boom".to_string(), "eRrOr splash".to_string()]); - let stdout_output = run_ljx( - [ - "filter".as_ref(), - spool_path.as_os_str(), - "-o".as_ref(), - "-".as_ref(), - ], - &["-F", "error", "-i"], - )?; + let stdout_output = run_ljx(["filter".as_ref(), spool_path.as_os_str(), "-o".as_ref(), "-".as_ref()], &["-F", "error", "-i"])?; if !stdout_output.status.success() { - return Err(io::Error::other(format!( - "ljx stdout filter failed: {}", - String::from_utf8_lossy(&stdout_output.stderr) - ))); + return Err(io::Error::other(format!("ljx stdout filter failed: {}", String::from_utf8_lossy(&stdout_output.stderr)))); } fs::write(&filtered_stdout, &stdout_output.stdout)?; - assert_eq!( - read_logjet_messages(&filtered_stdout)?, - vec!["ERROR boom".to_string(), "eRrOr splash".to_string()] - ); + assert_eq!(read_logjet_messages(&filtered_stdout)?, vec!["ERROR boom".to_string(), "eRrOr splash".to_string()]); - let invalid = run_ljx( - ["count".as_ref(), spool_path.as_os_str()], - &["-F", "error", "-e", "panic"], - )?; + let invalid = run_ljx(["count".as_ref(), spool_path.as_os_str()], &["-F", "error", "-e", "panic"])?; assert!(!invalid.status.success()); assert!( String::from_utf8_lossy(&invalid.stderr).contains("cannot be used with") @@ -176,10 +97,10 @@ fn ljx_filters_real_logjetd_output_from_mock_emitter() -> io::Result<()> { } fn ensure_test_binaries_exist() -> io::Result<()> { - for path in [logjetd_bin(), ljx_bin(), emitter_bin()] { + for path in [ljd_bin(), ljx_bin(), emitter_bin()] { if !path.is_file() { return Err(io::Error::other(format!( - "missing test binary {}. build it first with: cargo build -p logjetd -p ljx -p otlp-demo --bin otlp-bofh-emitter", + "missing test binary {}. build it first with: cargo build -p ljd -p ljx -p otlp-demo --bin otlp-bofh-emitter", path.display() ))); } @@ -199,59 +120,32 @@ fn run_emitter(ingest_port: u16, message: &str) -> io::Result<()> { .stderr(Stdio::null()) .status() .map_err(|err| io::Error::other(format!("failed to start emitter: {err}")))?; - if status.success() { - Ok(()) - } else { - Err(io::Error::other(format!("emitter failed for message: {message}"))) - } + if status.success() { Ok(()) } else { Err(io::Error::other(format!("emitter failed for message: {message}"))) } } fn run_ljx_count(input: &Path, extra_args: &[&str]) -> io::Result { let output = run_ljx(["count".as_ref(), input.as_os_str()], extra_args)?; if !output.status.success() { - return Err(io::Error::other(format!( - "ljx count failed: {}", - String::from_utf8_lossy(&output.stderr) - ))); + return Err(io::Error::other(format!("ljx count failed: {}", String::from_utf8_lossy(&output.stderr)))); } Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } fn run_ljx_filter(input: &Path, output: &Path, extra_args: &[&str]) -> io::Result<()> { - let output = run_ljx( - [ - "filter".as_ref(), - input.as_os_str(), - "-o".as_ref(), - output.as_os_str(), - ], - extra_args, - )?; - if output.status.success() { - Ok(()) - } else { - Err(io::Error::other(format!( - "ljx filter failed: {}", - String::from_utf8_lossy(&output.stderr) - ))) - } + let output = run_ljx(["filter".as_ref(), input.as_os_str(), "-o".as_ref(), output.as_os_str()], extra_args)?; + if output.status.success() { Ok(()) } else { Err(io::Error::other(format!("ljx filter failed: {}", String::from_utf8_lossy(&output.stderr)))) } } fn run_ljx(prefix_args: [&OsStr; N], extra_args: &[&str]) -> io::Result { let mut command = Command::new(ljx_bin()); command.args(prefix_args); command.args(extra_args); - command - .output() - .map_err(|err| io::Error::other(format!("failed to start ljx: {err}"))) + command.output().map_err(|err| io::Error::other(format!("failed to start ljx: {err}"))) } fn read_logjet_messages(path: &Path) -> io::Result> { - Ok(read_logjet_records(path)? - .into_iter() - .map(|record| record.message) - .collect()) + Ok(read_logjet_records(path)?.into_iter().map(|record| record.message).collect()) } fn read_logjet_records(path: &Path) -> io::Result> { @@ -260,19 +154,14 @@ fn read_logjet_records(path: &Path) -> io::Result> { let mut records = Vec::new(); while let Some(record) = reader.next_record().map_err(io::Error::other)? { for message in decode_payload_messages(&record.payload)? { - records.push(DecodedRecord { - seq: record.seq, - ts_unix_ns: record.ts_unix_ns, - message, - }); + records.push(DecodedRecord { seq: record.seq, ts_unix_ns: record.ts_unix_ns, message }); } } Ok(records) } fn decode_payload_messages(payload: &[u8]) -> io::Result> { - let batch = ExportLogsServiceRequest::decode(payload) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + let batch = ExportLogsServiceRequest::decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; let mut messages = Vec::new(); for resource_logs in batch.resource_logs { for scope_logs in resource_logs.scope_logs { @@ -289,9 +178,7 @@ fn decode_payload_messages(payload: &[u8]) -> io::Result> { } fn target_dir() -> PathBuf { - std::env::var_os("CARGO_TARGET_DIR") - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target")) + std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from).unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target")) } struct DecodedRecord { @@ -300,8 +187,8 @@ struct DecodedRecord { message: String, } -fn logjetd_bin() -> PathBuf { - target_dir().join("debug").join(binary_name("logjetd")) +fn ljd_bin() -> PathBuf { + target_dir().join("debug").join(binary_name("ljd")) } fn ljx_bin() -> PathBuf { @@ -309,17 +196,11 @@ fn ljx_bin() -> PathBuf { } fn emitter_bin() -> PathBuf { - target_dir() - .join("debug") - .join(binary_name("otlp-bofh-emitter")) + target_dir().join("debug").join(binary_name("otlp-bofh-emitter")) } fn binary_name(name: &str) -> String { - if cfg!(windows) { - format!("{name}.exe") - } else { - name.to_string() - } + if cfg!(windows) { format!("{name}.exe") } else { name.to_string() } } struct TestDir { @@ -328,14 +209,8 @@ struct TestDir { impl TestDir { fn new(label: &str) -> io::Result { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let path = std::env::temp_dir().join(format!( - "logjet-ljx-it-{label}-{nanos}-{}", - std::process::id() - )); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let path = std::env::temp_dir().join(format!("logjet-ljx-it-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&path)?; Ok(Self { path }) } @@ -363,10 +238,7 @@ struct ChildGuard { impl ChildGuard { fn spawn(mut command: Command) -> io::Result { - let child = command - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; + let child = command.stdout(Stdio::null()).stderr(Stdio::null()).spawn()?; Ok(Self { child }) } } @@ -417,10 +289,7 @@ where return Ok(()); } if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "timed out waiting for condition", - )); + return Err(io::Error::new(io::ErrorKind::TimedOut, "timed out waiting for condition")); } thread::sleep(Duration::from_millis(25)); }