From 01ec261c2cd8189a0af8fe626e521eed3ce24d76 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:42:47 +0100 Subject: [PATCH 01/17] Add replay-handoff demo --- demo/README.md | 2 ++ demo/replay-handoff/README.md | 50 ++++++++++++++++++++++++++++ demo/replay-handoff/logjetd.conf | 7 ++++ demo/replay-handoff/run-appliance.sh | 44 ++++++++++++++++++++++++ demo/replay-handoff/run-consumer.sh | 37 ++++++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100644 demo/replay-handoff/README.md create mode 100644 demo/replay-handoff/logjetd.conf create mode 100755 demo/replay-handoff/run-appliance.sh create mode 100755 demo/replay-handoff/run-consumer.sh diff --git a/demo/README.md b/demo/README.md index fe4c35c..390ba14 100644 --- a/demo/README.md +++ b/demo/README.md @@ -33,6 +33,8 @@ It also contains scenario demos under subdirectories: - five emitters running continuously into one `logjetd` and one live collector - [`multi-client-behaviour`](./multi-client-behaviour) - one replay client stalls while another keeps flowing +- [`replay-handoff`](./replay-handoff) + - a late replay client drains retained backlog and then continues live on the same connection - [`file-replay`](./file-replay) - replay stored `.logjet` files into a collector - [`file-tooling`](./file-tooling) diff --git a/demo/replay-handoff/README.md b/demo/replay-handoff/README.md new file mode 100644 index 0000000..797b96e --- /dev/null +++ b/demo/replay-handoff/README.md @@ -0,0 +1,50 @@ +# Replay Handoff Demo + +This demo isolates the backlog-to-live handoff inside `logjetd serve`. + +It proves one replay client can: + +- connect after backlog already exists +- receive the retained backlog first +- stay on the same replay connection +- continue receiving new records through direct ingest wakeups + +## Build First + +From the project root: + +```bash +make demo +``` + +## Run In Two Terminals + +Terminal 1, from this directory: + +```bash +./run-appliance.sh +``` + +This starts `logjetd`, writes three retained messages before any replay client +connects, and then keeps sending one live message per second. + +Terminal 2, from this directory: + +```bash +./run-consumer.sh +``` + +This starts the demo collector and then connects a replay client late through +the internal wire forwarder. + +## What You Should See + +On the collector side: + +- `HANDOFF backlog 001` +- `HANDOFF backlog 002` +- `HANDOFF backlog 003` +- then ongoing `HANDOFF live` records every second + +That transition should happen on one replay connection, without reconnecting +between backlog replay and live delivery. diff --git a/demo/replay-handoff/logjetd.conf b/demo/replay-handoff/logjetd.conf new file mode 100644 index 0000000..5c5a0fe --- /dev/null +++ b/demo/replay-handoff/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./spool +file.size: 256 +file.name: handoff.logjet +ingest.protocol: otlp-http +ingest.listen: 127.0.0.1:4318 +replay.listen: 127.0.0.1:7002 diff --git a/demo/replay-handoff/run-appliance.sh b/demo/replay-handoff/run-appliance.sh new file mode 100755 index 0000000..78fc1b1 --- /dev/null +++ b/demo/replay-handoff/run-appliance.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LOGJETD="$TARGET_DIR/logjetd" +EMITTER="$TARGET_DIR/otlp-bofh-emitter" +CONFIG="$SCRIPT_DIR/logjetd.conf" + +for bin in "$LOGJETD" "$EMITTER"; do + if [ ! -x "$bin" ]; then + echo "missing $bin" + echo "build everything first with: make demo" + exit 1 + fi +done + +mkdir -p "$SCRIPT_DIR/spool" +cd "$SCRIPT_DIR" + +echo "starting logjetd with config $CONFIG" +"$LOGJETD" --config "$CONFIG" serve & +LOGJETD_PID=$! + +cleanup() { + kill "${LIVE_PID:-}" 2>/dev/null || true + kill "${LOGJETD_PID:-}" 2>/dev/null || true +} + +trap cleanup EXIT INT TERM + +sleep 1 + +echo "writing retained backlog before any replay client connects" +"$EMITTER" 127.0.0.1:4318 --once --service-name handoff-demo --message "HANDOFF backlog 001" +"$EMITTER" 127.0.0.1:4318 --once --service-name handoff-demo --message "HANDOFF backlog 002" +"$EMITTER" 127.0.0.1:4318 --once --service-name handoff-demo --message "HANDOFF backlog 003" + +echo "starting a live emitter after the backlog is already retained" +"$EMITTER" 127.0.0.1:4318 --service-name handoff-demo --interval-ms 1000 --message "HANDOFF live" & +LIVE_PID=$! + +echo "appliance side is running; start ./run-consumer.sh in another terminal" +wait diff --git a/demo/replay-handoff/run-consumer.sh b/demo/replay-handoff/run-consumer.sh new file mode 100755 index 0000000..979979f --- /dev/null +++ b/demo/replay-handoff/run-consumer.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +COLLECTOR="$TARGET_DIR/otlp-demo-collector" +FORWARDER="$TARGET_DIR/otlp-wire-forwarder" + +for bin in "$COLLECTOR" "$FORWARDER"; do + if [ ! -x "$bin" ]; then + echo "missing $bin" + echo "build everything first with: make demo" + exit 1 + fi +done + +cd "$SCRIPT_DIR" + +echo "starting collector on 127.0.0.1:4320" +"$COLLECTOR" 127.0.0.1:4320 & +COLLECTOR_PID=$! + +cleanup() { + kill "${FORWARDER_PID:-}" 2>/dev/null || true + kill "${COLLECTOR_PID:-}" 2>/dev/null || true +} + +trap cleanup EXIT INT TERM + +sleep 1 + +echo "connecting a late replay client to 127.0.0.1:7002" +"$FORWARDER" 127.0.0.1:7002 127.0.0.1:4320 & +FORWARDER_PID=$! + +echo "you should first see HANDOFF backlog messages, then HANDOFF live records continue" +wait From 428baca9553fd4718e18309783e305326d113601 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:44:11 +0100 Subject: [PATCH 02/17] Adjust mockups for replay-handoff demo --- demo/src/bin/otlp-bofh-emitter.rs | 23 +++++++++++++---- demo/src/bin/otlp-bofh-grpc-emitter.rs | 2 +- demo/src/bin/otlp-demo-collector.rs | 22 +++++++++++++---- demo/src/bin/otlp-wire-forwarder.rs | 29 +++++++++++++--------- demo/src/bin/replay-stall-client.rs | 29 +++++++++++++--------- demo/src/bin/wire-hold-emitter.rs | 4 ++- demo/src/lib.rs | 34 ++++++++++++++++++-------- 7 files changed, 98 insertions(+), 45 deletions(-) diff --git a/demo/src/bin/otlp-bofh-emitter.rs b/demo/src/bin/otlp-bofh-emitter.rs index 1f2bb28..56d7773 100644 --- a/demo/src/bin/otlp-bofh-emitter.rs +++ b/demo/src/bin/otlp-bofh-emitter.rs @@ -49,12 +49,16 @@ fn main() -> Result<(), Box> { severity = args.next().ok_or("missing value for --severity")?; } "--ca-file" => { - ca_file = Some(PathBuf::from(args.next().ok_or("missing value for --ca-file")?)); + ca_file = Some(PathBuf::from( + args.next().ok_or("missing value for --ca-file")?, + )); } "--server-name" => { server_name = Some(args.next().ok_or("missing value for --server-name")?); } - value if value.starts_with("--") => return Err(format!("unknown argument: {value}").into()), + value if value.starts_with("--") => { + return Err(format!("unknown argument: {value}").into()); + } value => addr = value.to_string(), } } @@ -69,9 +73,18 @@ fn main() -> Result<(), Box> { let mut sequence = 1u64; loop { let request = match &once_message { - Some(message) => build_message_request_for_service(sequence, &service_name, &severity, message.clone()), - None if service_name == "bofh-emitter" && severity == "warn" => build_excuse_request(sequence), - None => build_excuse_request_for_service_with_severity(sequence, &service_name, &severity), + Some(message) => build_message_request_for_service( + sequence, + &service_name, + &severity, + message.clone(), + ), + None if service_name == "bofh-emitter" && severity == "warn" => { + build_excuse_request(sequence) + } + None => { + build_excuse_request_for_service_with_severity(sequence, &service_name, &severity) + } }; print!("{}", format_batch_plain(&request)); match otlp_demo::post_raw_otlp_http( diff --git a/demo/src/bin/otlp-bofh-grpc-emitter.rs b/demo/src/bin/otlp-bofh-grpc-emitter.rs index 8e491ea..3e3b089 100644 --- a/demo/src/bin/otlp-bofh-grpc-emitter.rs +++ b/demo/src/bin/otlp-bofh-grpc-emitter.rs @@ -2,7 +2,7 @@ use std::env; use std::time::Duration; use opentelemetry_proto::tonic::collector::logs::v1::{ - logs_service_client::LogsServiceClient, ExportLogsServiceRequest, + ExportLogsServiceRequest, logs_service_client::LogsServiceClient, }; use tonic::Request; diff --git a/demo/src/bin/otlp-demo-collector.rs b/demo/src/bin/otlp-demo-collector.rs index 0292cc0..49a6178 100644 --- a/demo/src/bin/otlp-demo-collector.rs +++ b/demo/src/bin/otlp-demo-collector.rs @@ -30,10 +30,14 @@ fn main() -> Result<(), Box> { .parse::()?; } "--cert-file" => { - cert_file = Some(PathBuf::from(args.next().ok_or("missing value for --cert-file")?)); + cert_file = Some(PathBuf::from( + args.next().ok_or("missing value for --cert-file")?, + )); } "--key-file" => { - key_file = Some(PathBuf::from(args.next().ok_or("missing value for --key-file")?)); + key_file = Some(PathBuf::from( + args.next().ok_or("missing value for --key-file")?, + )); } value => bind_addr = value.to_string(), } @@ -157,7 +161,10 @@ fn read_http_request(transport: &mut T) -> io::Result 16 * 1024 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "http header too large")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "http header too large", + )); } } @@ -191,9 +198,14 @@ fn read_http_request(transport: &mut T) -> io::Result Result<(), Box> { let mut args = env::args().skip(1); - let source = args - .next() - .unwrap_or_else(|| "127.0.0.1:7002".to_string()); - let dest = args - .next() - .unwrap_or_else(|| "127.0.0.1:4320".to_string()); + let source = args.next().unwrap_or_else(|| "127.0.0.1:7002".to_string()); + let dest = args.next().unwrap_or_else(|| "127.0.0.1:4320".to_string()); let max_records = match args.next() { Some(value) => Some(value.parse::()?), None => None, @@ -27,7 +23,10 @@ fn main() -> Result<(), Box> { if record.record_type == RecordType::Logs { post_raw_otlp_http(&dest, &record.payload, None, None)?; forwarded += 1; - eprintln!("forwarded record seq={} to http://{dest}/v1/logs", record.seq); + eprintln!( + "forwarded record seq={} to http://{dest}/v1/logs", + record.seq + ); } if let Some(limit) = max_records { @@ -53,7 +52,10 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result<()> { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != *b"LJRPH001" { - return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay hello magic")); + return Err(io::Error::new( + ErrorKind::InvalidData, + "invalid replay hello magic", + )); } let mut header = [0u8; 32]; @@ -83,22 +85,25 @@ fn read_wire_record(reader: &mut R) -> io::Result> { } if magic != *b"LJNETV01" { - return Err(io::Error::new(ErrorKind::InvalidData, "invalid wire protocol magic")); + return Err(io::Error::new( + ErrorKind::InvalidData, + "invalid wire protocol magic", + )); } let mut header = [0u8; 24]; reader.read_exact(&mut header)?; let record_type = RecordType::from_u8(header[1]) .map_err(|err| io::Error::new(ErrorKind::InvalidData, err.to_string()))?; - let payload_len = - u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; + let payload_len = u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; let mut payload = vec![0u8; payload_len]; reader.read_exact(&mut payload)?; Ok(Some(WireRecord { record_type, seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11], + header[4], header[5], header[6], header[7], header[8], header[9], header[10], + header[11], ]), payload, })) diff --git a/demo/src/bin/replay-stall-client.rs b/demo/src/bin/replay-stall-client.rs index e87e631..e89371f 100644 --- a/demo/src/bin/replay-stall-client.rs +++ b/demo/src/bin/replay-stall-client.rs @@ -6,9 +6,7 @@ use std::time::Duration; fn main() -> Result<(), Box> { let mut args = env::args().skip(1); - let source = args - .next() - .unwrap_or_else(|| "127.0.0.1:7002".to_string()); + let source = args.next().unwrap_or_else(|| "127.0.0.1:7002".to_string()); let stall_ms = match args.next() { Some(value) => value.parse::()?, None => 10_000, @@ -62,7 +60,10 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != *b"LJRPH001" { - return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay hello magic")); + return Err(io::Error::new( + ErrorKind::InvalidData, + "invalid replay hello magic", + )); } let mut header = [0u8; 32]; @@ -76,13 +77,16 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result { Ok(ReplayHello { stream_id: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15], + header[8], header[9], header[10], header[11], header[12], header[13], header[14], + header[15], ]), first_seq: u64::from_le_bytes([ - header[16], header[17], header[18], header[19], header[20], header[21], header[22], header[23], + header[16], header[17], header[18], header[19], header[20], header[21], header[22], + header[23], ]), last_seq: u64::from_le_bytes([ - header[24], header[25], header[26], header[27], header[28], header[29], header[30], header[31], + header[24], header[25], header[26], header[27], header[28], header[29], header[30], + header[31], ]), }) } @@ -100,19 +104,22 @@ fn read_wire_record(reader: &mut R) -> io::Result> { } if magic != *b"LJNETV01" { - return Err(io::Error::new(ErrorKind::InvalidData, "invalid wire protocol magic")); + return Err(io::Error::new( + ErrorKind::InvalidData, + "invalid wire protocol magic", + )); } let mut header = [0u8; 24]; reader.read_exact(&mut header)?; - let payload_len = - u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; + let payload_len = u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; let mut payload = vec![0u8; payload_len]; reader.read_exact(&mut payload)?; Ok(Some(WireRecord { seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11], + header[4], header[5], header[6], header[7], header[8], header[9], header[10], + header[11], ]), })) } diff --git a/demo/src/bin/wire-hold-emitter.rs b/demo/src/bin/wire-hold-emitter.rs index 46f00d9..a352873 100644 --- a/demo/src/bin/wire-hold-emitter.rs +++ b/demo/src/bin/wire-hold-emitter.rs @@ -26,7 +26,9 @@ fn main() -> Result<(), Box> { "--service-name" => { service_name = args.next().ok_or("missing value for --service-name")?; } - value if value.starts_with("--") => return Err(format!("unknown argument: {value}").into()), + value if value.starts_with("--") => { + return Err(format!("unknown argument: {value}").into()); + } value => addr = value.to_string(), } } diff --git a/demo/src/lib.rs b/demo/src/lib.rs index 343543f..cbb9a07 100644 --- a/demo/src/lib.rs +++ b/demo/src/lib.rs @@ -93,9 +93,11 @@ pub fn build_message_request_for_service( severity_number, severity_text, body: Some(AnyValue { - value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - body, - )), + value: Some( + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( + body, + ), + ), }), attributes: vec![ string_attr("demo.kind", "bofh"), @@ -219,11 +221,16 @@ fn post_raw_otlp_http_transport( fn load_demo_client_config(ca_path: &Path) -> io::Result> { let roots = load_root_store(ca_path)?; Ok(Arc::new( - ClientConfig::builder().with_root_certificates(roots).with_no_client_auth(), + ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(), )) } -fn demo_server_name(endpoint: &DemoEndpoint, override_name: Option<&str>) -> io::Result> { +fn demo_server_name( + endpoint: &DemoEndpoint, + override_name: Option<&str>, +) -> io::Result> { let name = override_name.unwrap_or_else(|| endpoint.server_name()); ServerName::try_from(name.to_string()) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) @@ -284,7 +291,10 @@ impl DemoEndpoint { } fn server_name(&self) -> &str { - self.authority.rsplit_once(':').map(|(host, _)| host).unwrap_or(&self.authority) + self.authority + .rsplit_once(':') + .map(|(host, _)| host) + .unwrap_or(&self.authority) } } @@ -400,9 +410,11 @@ fn string_attr(key: &str, value: &str) -> KeyValue { KeyValue { key: key.to_string(), value: Some(AnyValue { - value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - value.to_string(), - )), + value: Some( + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( + value.to_string(), + ), + ), }), } } @@ -418,7 +430,9 @@ fn int_attr(key: &str, value: i64) -> KeyValue { fn any_value_to_string(value: &AnyValue) -> Option<&str> { match value.value.as_ref()? { - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value.as_str()), + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => { + Some(value.as_str()) + } _ => None, } } From 1b7b0fd4d126cf3d11e8bf12b7d1d5b9d981a0c0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:45:37 +0100 Subject: [PATCH 03/17] Reword documentation names --- doc/configuration.md | 6 +++--- doc/daemon.md | 6 +++--- doc/manpage/logjetd.1.md | 22 +++++++++++----------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/configuration.md b/doc/configuration.md index e7c7eee..e412f20 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -140,9 +140,9 @@ Base file name for file mode. Example: -- `bofh.logjet` -- `bofh-1.logjet` -- `bofh-2.logjet` +- `app.logjet` +- `app-1.logjet` +- `app-2.logjet` Important: diff --git a/doc/daemon.md b/doc/daemon.md index eff8148..b7c6c17 100644 --- a/doc/daemon.md +++ b/doc/daemon.md @@ -41,19 +41,19 @@ logjetd inspect /var/lib/logjet List rotated file segments for one spool: ```bash -logjetd segments --path /var/lib/logjet --name bofh.logjet +logjetd segments --path /var/lib/logjet --name app.logjet ``` Prune oldest rotated file segments and keep only the newest two files: ```bash -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-files 2 +logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 ``` Preview byte-budget pruning without deleting anything: ```bash -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-bytes 1048576 --dry-run +logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run ``` Continuously bridge from another `logjetd` replay listener into an OTLP diff --git a/doc/manpage/logjetd.1.md b/doc/manpage/logjetd.1.md index 2baa670..96b2c4e 100644 --- a/doc/manpage/logjetd.1.md +++ b/doc/manpage/logjetd.1.md @@ -89,7 +89,7 @@ List ordered rotated segments for one spool. Example: ```text -logjetd segments --path /var/lib/logjet --name bofh.logjet +logjetd segments --path /var/lib/logjet --name app.logjet ``` ## replay @@ -100,14 +100,14 @@ payloads into an OTLP/HTTP collector. Example: ```text -logjetd replay --path /var/logs --name bofh.logjet --dest http://127.0.0.1:4318/v1/logs +logjetd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs ``` Replay order is: -- `bofh.logjet` -- `bofh-1.logjet` -- `bofh-2.logjet` +- `app.logjet` +- `app-1.logjet` +- `app-2.logjet` - and so on Replay is immediate and does not preserve original timing. @@ -120,8 +120,8 @@ Remove oldest rotated file segments deliberately. Examples: ```text -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-files 2 -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-bytes 1048576 --dry-run +logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 +logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run ``` # OPTIONS @@ -146,9 +146,9 @@ Base file name used to locate replay segments. Example: -- `bofh.logjet` -- `bofh-1.logjet` -- `bofh-2.logjet` +- `app.logjet` +- `app-1.logjet` +- `app-2.logjet` Used only with the `replay` command. @@ -338,7 +338,7 @@ logjetd inspect ./logs Replay files to a local collector: ```text -logjetd --config ./logjetd.conf replay --path ./logs --name bofh.logjet +logjetd --config ./logjetd.conf replay --path ./logs --name app.logjet ``` Bridge from a remote replay listener into a collector: From 12a9cbd912ea1dd6223c8fc9caa545655853f650 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:45:51 +0100 Subject: [PATCH 04/17] Update documentation for replay-handoff --- doc/daemon.md | 2 +- doc/features.md | 2 +- doc/manpage/logjetd.1 | 34 +++++++++++++++++++++------------- doc/manpage/logjetd.1.md | 5 ++++- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/doc/daemon.md b/doc/daemon.md index b7c6c17..6370212 100644 --- a/doc/daemon.md +++ b/doc/daemon.md @@ -87,7 +87,7 @@ logjetd --config /path/to/logjet.conf bridge --source 10.0.0.15:7002 - expects a small replay request carrying `from_seq` - replay requests can ask to `keep` or `drain` - replays retained records in sequence order -- switches from retained backlog to live records through direct wakeups from ingest +- uses a per-client replay cursor so retained backlog can hand off directly into live wakeups from ingest - can optionally wrap the replay transport in TLS Replay is strictly sequential today. Resume exists per connection via `from_seq`. diff --git a/doc/features.md b/doc/features.md index 75927fd..e3d3971 100644 --- a/doc/features.md +++ b/doc/features.md @@ -79,7 +79,7 @@ Current behaviour: - clients can request `keep` or `drain` - each client keeps its own replay cursor - replays retained data in sequence order -- switches from retained backlog to live records through direct wakeups from ingest +- hands off from retained backlog to live records through direct ingest wakeups without returning to storage polling - supports multiple clients in a basic way - can cap concurrent replay clients through `replay.max-clients` - can cap blocked replay socket I/O through `replay.client-timeout-ms` diff --git a/doc/manpage/logjetd.1 b/doc/manpage/logjetd.1 index f0d8a74..009b064 100644 --- a/doc/manpage/logjetd.1 +++ b/doc/manpage/logjetd.1 @@ -81,6 +81,12 @@ OTLP/gRPC ingest is supported with \f[C]ingest.protocol: otlp-grpc\f[R] .IP \[bu] 2 a replay listener socket is exposed for downstream clients .IP \[bu] 2 +replay sends retained backlog first, then hands the same client +directly into live wakeups from ingest +.IP \[bu] 2 +replay keeps an explicit cursor per client across buffer eviction, +drain cleanup, and file rotation +.IP \[bu] 2 replay listener traffic currently uses the internal framed protocol, not OTLP egress .SS bridge @@ -119,7 +125,7 @@ Example: .IP .nf \f[C] -logjetd segments --path /var/lib/logjet --name bofh.logjet +logjetd segments --path /var/lib/logjet --name app.logjet \f[R] .fi .SS replay @@ -131,17 +137,17 @@ Example: .IP .nf \f[C] -logjetd replay --path /var/logs --name bofh.logjet --dest http://127.0.0.1:4318/v1/logs +logjetd replay --path /var/logs --name app.logjet --dest http://127.0.0.1:4318/v1/logs \f[R] .fi .PP Replay order is: .IP \[bu] 2 -\f[C]bofh.logjet\f[R] +\f[C]app.logjet\f[R] .IP \[bu] 2 -\f[C]bofh-1.logjet\f[R] +\f[C]app-1.logjet\f[R] .IP \[bu] 2 -\f[C]bofh-2.logjet\f[R] +\f[C]app-2.logjet\f[R] .IP \[bu] 2 and so on .PP @@ -156,8 +162,8 @@ Examples: .IP .nf \f[C] -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-files 2 -logjetd prune --path /var/lib/logjet --name bofh.logjet --keep-bytes 1048576 --dry-run +logjetd prune --path /var/lib/logjet --name app.logjet --keep-files 2 +logjetd prune --path /var/lib/logjet --name app.logjet --keep-bytes 1048576 --dry-run \f[R] .fi .SH OPTIONS @@ -180,11 +186,11 @@ Base file name used to locate replay segments. .PP Example: .IP \[bu] 2 -\f[C]bofh.logjet\f[R] +\f[C]app.logjet\f[R] .IP \[bu] 2 -\f[C]bofh-1.logjet\f[R] +\f[C]app-1.logjet\f[R] .IP \[bu] 2 -\f[C]bofh-2.logjet\f[R] +\f[C]app-2.logjet\f[R] .PP Used only with the \f[C]replay\f[R] command. .SS \f[C]--keep-files\f[R] \f[I]n\f[R] @@ -395,6 +401,8 @@ protocol .IP \[bu] 2 independent replay cursor per connected client .IP \[bu] 2 +backlog-to-live replay handoff through direct ingest wakeups +.IP \[bu] 2 basic replay-client caps through \f[C]replay.max-clients\f[R] .IP \[bu] 2 basic replay-client timeout through \f[C]replay.client-timeout-ms\f[R] @@ -432,8 +440,8 @@ upstream storage reset and rollover handling is still basic ingest overload handling is limited to payload-size caps and concurrent-client caps .IP \[bu] 2 -multi-client replay isolation is still basic beyond per-client cursors -and replay-client caps +multi-client replay isolation is still basic beyond per-client cursors, +wakeups, and replay-client caps .IP \[bu] 2 file mode does not delete old rotated files .IP \[bu] 2 @@ -460,7 +468,7 @@ Replay files to a local collector: .IP .nf \f[C] -logjetd --config ./logjetd.conf replay --path ./logs --name bofh.logjet +logjetd --config ./logjetd.conf replay --path ./logs --name app.logjet \f[R] .fi .PP diff --git a/doc/manpage/logjetd.1.md b/doc/manpage/logjetd.1.md index 96b2c4e..bab71a8 100644 --- a/doc/manpage/logjetd.1.md +++ b/doc/manpage/logjetd.1.md @@ -56,6 +56,8 @@ Current serve behaviour: - OTLP/HTTP ingest is supported with `ingest.protocol: otlp-http` - OTLP/gRPC ingest is supported with `ingest.protocol: otlp-grpc` - a replay listener socket is exposed for downstream clients +- replay sends retained backlog first, then hands the same client directly into live wakeups from ingest +- replay keeps an explicit cursor per client across buffer eviction, drain cleanup, and file rotation - replay listener traffic currently uses the internal framed protocol, not OTLP egress ## bridge @@ -298,6 +300,7 @@ Append-only file behaviour: - in-memory ring buffering with `buffer.keep` - replay listener for downstream consumers using the internal framed protocol - independent replay cursor per connected client +- backlog-to-live replay handoff through direct ingest wakeups - basic replay-client caps through `replay.max-clients` - basic replay-client timeout through `replay.client-timeout-ms` - continuous bridge mode from replay listener to OTLP/HTTP collectors @@ -317,7 +320,7 @@ Append-only file behaviour: - replay listener traffic is not OTLP - upstream storage reset and rollover handling is still basic - ingest overload handling is limited to payload-size caps and concurrent-client caps -- multi-client replay isolation is still basic beyond per-client cursors and replay-client caps +- multi-client replay isolation is still basic beyond per-client cursors, wakeups, and replay-client caps - file mode does not delete old rotated files - certificate management and deployment policy are still operator-managed From 56c06af155e7a1be89b1a4b09729804bafca8b6f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:46:26 +0100 Subject: [PATCH 05/17] Update rust-toolchain.toml to Rust 1.94.0 --- rust-toolchain.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 rust-toolchain.toml diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..76a06e6 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.94.0" From 9d073b8b18885fda1410f76cb9dd0fb8c877be7a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:48:22 +0100 Subject: [PATCH 06/17] Add toolchain update script --- scripts/update-rust-toolchain.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 scripts/update-rust-toolchain.sh diff --git a/scripts/update-rust-toolchain.sh b/scripts/update-rust-toolchain.sh new file mode 100755 index 0000000..02b3470 --- /dev/null +++ b/scripts/update-rust-toolchain.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash +set -e + +LATEST=$(curl -s https://static.rust-lang.org/dist/channel-rust-stable.toml | grep "^version = \"1\." | sort | uniq | tail -1 | sed -nE 's/version = "([^ ]+).*/\1/p') + +if [ -z "$LATEST" ]; then + echo "Could not fetch the latest Rust version" + exit 1 +fi +echo "Latest Rust stable version: $LATEST" + +# Write the new version to rust-toolchain.toml. +cat < rust-toolchain.toml +[toolchain] +channel = "$LATEST" +EOF + +# Check if the file has changed (this assumes you are in a git repository). +if [ -n "$(git status --porcelain rust-toolchain.toml)" ]; then + echo "rust-toolchain.toml updated. Committing changes..." + git add rust-toolchain.toml + git commit -m "Update rust-toolchain.toml to Rust $LATEST" + git push +else + echo "rust-toolchain.toml is already up-to-date." +fi From 795fc065d41134655b0c9c16496462f14bf71541 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:48:30 +0100 Subject: [PATCH 07/17] Add formatter rules --- rustfmt.toml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..5dfd661 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,23 @@ +# Don't argue on 130 width. :-) Just don't. +# No more IBM punchcards here, and your monitor +# is at least 28 inches wide with a decent resolution. +# +# Welcome to the 21st century. + +max_width = 150 +fn_call_width = 150 +chain_width = 150 +single_line_let_else_max_width = 150 +single_line_if_else_max_width = 150 +use_small_heuristics = "Max" + +# Parser version +edition = "2024" + +# Emacs! +tab_spaces = 4 + +# Perks +use_field_init_shorthand = true +fn_params_layout = "Compressed" +use_try_shorthand = true From f7f247b187aad3c94f6b08c19a4e4dde03e1f9ca Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:49:20 +0100 Subject: [PATCH 08/17] Add forward-replay --- examples/write_file.rs | 7 +- logjetd/src/config.rs | 12 ++- logjetd/src/config_utst.rs | 107 +++++++++++++++++----- logjetd/src/daemon.rs | 167 ++++++++++++++++++++++++---------- logjetd/src/main.rs | 68 ++++++++++---- logjetd/src/protocol.rs | 34 +++++-- logjetd/src/replay.rs | 168 +++++++++++++++++++++++++--------- logjetd/src/spool.rs | 181 +++++++++++++++++++++++-------------- logjetd/src/tls.rs | 31 +++++-- src/error.rs | 31 +++++-- src/format.rs | 3 +- src/lib.rs | 5 +- src/reader.rs | 12 +-- src/writer.rs | 24 ++--- 14 files changed, 599 insertions(+), 251 deletions(-) diff --git a/examples/write_file.rs b/examples/write_file.rs index f3d285b..6f5051c 100644 --- a/examples/write_file.rs +++ b/examples/write_file.rs @@ -8,7 +8,12 @@ fn main() -> Result<(), Box> { let writer = BufWriter::new(file); let mut log_writer = LogjetWriter::new(writer); - log_writer.push(RecordType::Logs, 1, 1_700_000_000_000_000_000, b"fake-otlp-logs")?; + log_writer.push( + RecordType::Logs, + 1, + 1_700_000_000_000_000_000, + b"fake-otlp-logs", + )?; log_writer.push( RecordType::Metrics, 2, diff --git a/logjetd/src/config.rs b/logjetd/src/config.rs index 7582230..8cc5ae4 100644 --- a/logjetd/src/config.rs +++ b/logjetd/src/config.rs @@ -297,7 +297,9 @@ impl Config { let ingest_overload = IngestOverloadConfig { max_batches_per_second: raw.ingest_max_batches_per_second.unwrap_or(0), priority_severity_floor: parse_severity_floor( - raw.ingest_priority_severity_floor.as_deref().unwrap_or("error"), + raw.ingest_priority_severity_floor + .as_deref() + .unwrap_or("error"), )?, report_every_ms: raw.ingest_overload_report_ms.unwrap_or(5_000), }; @@ -368,13 +370,13 @@ impl Config { keep_messages, }), "file" => { - let name = raw - .file_name - .unwrap_or_else(|| "bar.logjet".to_string()); + let name = raw.file_name.unwrap_or_else(|| "bar.logjet".to_string()); StorageConfig::File(FileConfig { dir: raw.file_path.unwrap_or_else(|| PathBuf::from(".")), name, - segment_size_bytes: u64::try_from(kib_to_bytes(raw.file_size_kb.unwrap_or(100))?)?, + segment_size_bytes: u64::try_from(kib_to_bytes( + raw.file_size_kb.unwrap_or(100), + )?)?, }) } other => return Err(format!("invalid output mode: {other}").into()), diff --git a/logjetd/src/config_utst.rs b/logjetd/src/config_utst.rs index 47a30c4..fac2735 100644 --- a/logjetd/src/config_utst.rs +++ b/logjetd/src/config_utst.rs @@ -1,4 +1,7 @@ -use super::{BackpressureMode, BufferLimit, Config, IngestProtocol, SeverityFloor, StorageConfig, UpstreamMode}; +use super::{ + BackpressureMode, BufferLimit, Config, IngestProtocol, SeverityFloor, StorageConfig, + UpstreamMode, +}; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -18,7 +21,10 @@ fn empty_config_file_uses_defaults() { assert_eq!(config.ingest_limits.max_batch_bytes, 1024 * 1024); assert_eq!(config.ingest_limits.max_clients, 32); assert_eq!(config.ingest_overload.max_batches_per_second, 0); - assert_eq!(config.ingest_overload.priority_severity_floor, SeverityFloor::Error); + assert_eq!( + config.ingest_overload.priority_severity_floor, + SeverityFloor::Error + ); assert_eq!(config.ingest_overload.report_every_ms, 5_000); assert_eq!(config.replay_addr, "0.0.0.0:7002"); assert_eq!(config.replay_max_clients, 32); @@ -73,37 +79,73 @@ fn file_mode_and_collector_settings_parse() { assert_eq!(config.ingest_protocol, IngestProtocol::OtlpGrpc); assert_eq!(config.collector.timeout_ms, 3210); - assert_eq!(config.upstream.replay_addr.as_deref(), Some("10.0.0.15:7002")); + assert_eq!( + config.upstream.replay_addr.as_deref(), + Some("10.0.0.15:7002") + ); assert_eq!(config.upstream.mode, UpstreamMode::Keep); assert!(config.upstream.state_file.is_none()); assert_eq!(config.upstream.retry_ms, 222); assert_eq!(config.upstream.connect_timeout_ms, 333); assert!(config.ingest_tls.enable); - assert_eq!(config.ingest_tls.ca_file.as_deref(), Some(Path::new("./ingest-ca.pem"))); - assert_eq!(config.ingest_tls.cert_file.as_deref(), Some(Path::new("./ingest.pem"))); - assert_eq!(config.ingest_tls.key_file.as_deref(), Some(Path::new("./ingest.key"))); + assert_eq!( + config.ingest_tls.ca_file.as_deref(), + Some(Path::new("./ingest-ca.pem")) + ); + assert_eq!( + config.ingest_tls.cert_file.as_deref(), + Some(Path::new("./ingest.pem")) + ); + assert_eq!( + config.ingest_tls.key_file.as_deref(), + Some(Path::new("./ingest.key")) + ); assert!(config.ingest_tls.require_client_cert); assert_eq!(config.ingest_limits.max_batch_bytes, 262_144); assert_eq!(config.ingest_limits.max_clients, 7); assert_eq!(config.ingest_overload.max_batches_per_second, 12); - assert_eq!(config.ingest_overload.priority_severity_floor, SeverityFloor::Fatal); + assert_eq!( + config.ingest_overload.priority_severity_floor, + SeverityFloor::Fatal + ); assert_eq!(config.ingest_overload.report_every_ms, 900); assert_eq!(config.replay_max_clients, 9); assert_eq!(config.replay_client_timeout_ms, 444); assert!(config.tls.enable); assert_eq!(config.tls.ca_file.as_deref(), Some(Path::new("./ca.pem"))); - assert_eq!(config.tls.cert_file.as_deref(), Some(Path::new("./node.pem"))); - assert_eq!(config.tls.key_file.as_deref(), Some(Path::new("./node.key"))); + assert_eq!( + config.tls.cert_file.as_deref(), + Some(Path::new("./node.pem")) + ); + assert_eq!( + config.tls.key_file.as_deref(), + Some(Path::new("./node.key")) + ); assert!(config.tls.require_client_cert); - assert_eq!(config.tls.server_name.as_deref(), Some("appliance.internal")); + assert_eq!( + config.tls.server_name.as_deref(), + Some("appliance.internal") + ); assert_eq!(config.collector.url, "https://127.0.0.1:4320/custom"); assert!(config.backpressure.enabled); assert_eq!(config.backpressure.mode, BackpressureMode::Block); assert_eq!(config.backpressure.max_buffered_records, 23); - assert_eq!(config.collector.ca_file.as_deref(), Some(Path::new("./collector-ca.pem"))); - assert_eq!(config.collector.cert_file.as_deref(), Some(Path::new("./collector.pem"))); - assert_eq!(config.collector.key_file.as_deref(), Some(Path::new("./collector.key"))); - assert_eq!(config.collector.server_name.as_deref(), Some("collector.internal")); + assert_eq!( + config.collector.ca_file.as_deref(), + Some(Path::new("./collector-ca.pem")) + ); + assert_eq!( + config.collector.cert_file.as_deref(), + Some(Path::new("./collector.pem")) + ); + assert_eq!( + config.collector.key_file.as_deref(), + Some(Path::new("./collector.key")) + ); + assert_eq!( + config.collector.server_name.as_deref(), + Some("collector.internal") + ); match config.storage { StorageConfig::File(file) => { @@ -143,9 +185,18 @@ fn https_collector_fields_parse_without_file_mode() { "collector.url: https://collector.example:443/v1/logs\ncollector.ca-file: ./ca.pem\ncollector.server-name: collector.example\n", ); let config = Config::load(&path).unwrap(); - assert_eq!(config.collector.url, "https://collector.example:443/v1/logs"); - assert_eq!(config.collector.ca_file.as_deref(), Some(Path::new("./ca.pem"))); - assert_eq!(config.collector.server_name.as_deref(), Some("collector.example")); + assert_eq!( + config.collector.url, + "https://collector.example:443/v1/logs" + ); + assert_eq!( + config.collector.ca_file.as_deref(), + Some(Path::new("./ca.pem")) + ); + assert_eq!( + config.collector.server_name.as_deref(), + Some("collector.example") + ); fs::remove_file(path).unwrap(); } @@ -157,7 +208,10 @@ fn upstream_mode_drain_parses() { ); let config = Config::load(&path).unwrap(); assert_eq!(config.upstream.mode, UpstreamMode::Drain); - assert_eq!(config.upstream.state_file.as_deref(), Some(Path::new("./bridge.state"))); + assert_eq!( + config.upstream.state_file.as_deref(), + Some(Path::new("./bridge.state")) + ); fs::remove_file(path).unwrap(); } @@ -179,7 +233,10 @@ fn invalid_backpressure_mode_is_rejected() { #[test] fn backpressure_mode_block_parses() { - let path = write_temp_config("backpressure-block", "backpressure.enabled: true\nbackpressure.mode: block\n"); + let path = write_temp_config( + "backpressure-block", + "backpressure.enabled: true\nbackpressure.mode: block\n", + ); let config = Config::load(&path).unwrap(); assert!(config.backpressure.enabled); assert_eq!(config.backpressure.mode, BackpressureMode::Block); @@ -216,7 +273,8 @@ fn invalid_ingest_limit_values_are_rejected() { assert!(replay_err.contains("replay.max-clients")); fs::remove_file(replay_path).unwrap(); - let replay_timeout_path = write_temp_config("bad-replay-timeout", "replay.client-timeout-ms: 0\n"); + let replay_timeout_path = + write_temp_config("bad-replay-timeout", "replay.client-timeout-ms: 0\n"); let replay_timeout_err = Config::load(&replay_timeout_path).unwrap_err().to_string(); assert!(replay_timeout_err.contains("replay.client-timeout-ms")); fs::remove_file(replay_timeout_path).unwrap(); @@ -225,7 +283,9 @@ fn invalid_ingest_limit_values_are_rejected() { "bad-backpressure-buffer", "backpressure.max-buffered-records: 0\n", ); - let backpressure_buffer_err = Config::load(&backpressure_buffer_path).unwrap_err().to_string(); + let backpressure_buffer_err = Config::load(&backpressure_buffer_path) + .unwrap_err() + .to_string(); assert!(backpressure_buffer_err.contains("backpressure.max-buffered-records")); fs::remove_file(backpressure_buffer_path).unwrap(); } @@ -241,5 +301,8 @@ fn unique_temp_path(label: &str) -> PathBuf { .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.yaml", std::process::id())) + std::env::temp_dir().join(format!( + "logjetd-{label}-{nanos}-{}.yaml", + std::process::id() + )) } diff --git a/logjetd/src/daemon.rs b/logjetd/src/daemon.rs index 4c5841a..438ef5a 100644 --- a/logjetd/src/daemon.rs +++ b/logjetd/src/daemon.rs @@ -1,7 +1,7 @@ use std::fs; use std::io::{self, BufReader, Read, Write}; -use std::net::{TcpListener, TcpStream}; use std::net::SocketAddr; +use std::net::{TcpListener, TcpStream}; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; @@ -9,23 +9,25 @@ use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use opentelemetry_proto::tonic::collector::logs::v1::{ - logs_service_server::{LogsService, LogsServiceServer}, ExportLogsServiceRequest, ExportLogsServiceResponse, + logs_service_server::{LogsService, LogsServiceServer}, }; use prost::Message; +use rustls::{ServerConfig, ServerConnection, StreamOwned}; use tiny_http::{Method, Response, Server, StatusCode}; use tonic::transport::{Certificate, Identity, ServerTlsConfig}; use tonic::{Request, Response as GrpcResponse, Status}; -use rustls::{ServerConfig, ServerConnection, StreamOwned}; -use crate::config::{Config, IngestLimits, IngestOverloadConfig, IngestProtocol, IngestTlsConfig, SeverityFloor}; +use crate::config::{ + Config, IngestLimits, IngestOverloadConfig, IngestProtocol, IngestTlsConfig, SeverityFloor, +}; +use crate::protocol::WireRecord; use crate::protocol::{ ReplayHello, read_record_with_limit, read_replay_ack, read_replay_request, write_record, write_replay_hello, }; use crate::spool::Spool; use crate::tls::{load_ingest_server_config, load_server_config}; -use crate::{protocol::WireRecord}; #[derive(Debug, Clone)] pub struct DaemonConfig { @@ -159,7 +161,10 @@ impl SharedIngestPolicy { IngestDecision::RejectRateLimited }; - if matches!(decision, IngestDecision::AcceptPriorityBypass | IngestDecision::RejectRateLimited) { + if matches!( + decision, + IngestDecision::AcceptPriorityBypass | IngestDecision::RejectRateLimited + ) { maybe_report_overload(&self.config, &mut state); } @@ -272,8 +277,7 @@ fn ingest_loop( let listener = TcpListener::bind(&bind_addr)?; eprintln!( "logjetd ingest listening on {bind_addr} using wire protocol max-batch-bytes={} max-clients={}", - ingest_limits.max_batch_bytes, - ingest_limits.max_clients + ingest_limits.max_batch_bytes, ingest_limits.max_clients ); for stream in listener.incoming() { @@ -285,7 +289,13 @@ fn ingest_loop( thread::Builder::new() .name("logjetd-ingest-client".to_string()) .spawn(move || { - if let Err(err) = handle_ingest_client(stream, spool, ingest_policy, limiter, max_batch_bytes) { + if let Err(err) = handle_ingest_client( + stream, + spool, + ingest_policy, + limiter, + max_batch_bytes, + ) { eprintln!("logjetd ingest client error: {err}"); } })?; @@ -293,10 +303,18 @@ fn ingest_loop( } IngestProtocol::OtlpHttp => { if ingest_tls.enable { - return otlp_http_tls_loop(bind_addr, ingest_tls, ingest_limits, ingest_policy, spool, next_seq, limiter); + return otlp_http_tls_loop( + bind_addr, + ingest_tls, + ingest_limits, + ingest_policy, + spool, + next_seq, + limiter, + ); } - let server = Server::http(&bind_addr) - .map_err(|err| io::Error::other(err.to_string()))?; + let server = + Server::http(&bind_addr).map_err(|err| io::Error::other(err.to_string()))?; eprintln!( "logjetd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes @@ -304,8 +322,8 @@ fn ingest_loop( for mut request in server.incoming_requests() { if request.method() != &Method::Post || request.url() != "/v1/logs" { - let response = Response::from_string("not found") - .with_status_code(StatusCode(404)); + let response = + Response::from_string("not found").with_status_code(StatusCode(404)); let _ = request.respond(response); continue; } @@ -326,7 +344,8 @@ fn ingest_loop( } match ExportLogsServiceRequest::decode(body.as_slice()) { Ok(batch) => { - let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; + let decision = + ingest_policy.decide(classify_otlp_batch_priority(&batch))?; if matches!(decision, IngestDecision::RejectRateLimited) { let response = Response::from_string("rate limit exceeded") .with_status_code(StatusCode(429)); @@ -338,7 +357,8 @@ fn ingest_loop( let record = WireRecord { record_type: logjet::RecordType::Logs, seq: next_seq.fetch_add(1, Ordering::Relaxed), - ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), + ts_unix_ns: extract_batch_timestamp(&batch) + .unwrap_or_else(unix_time_nanos), payload: body, }; append_batch_record(&spool, record)?; @@ -360,12 +380,14 @@ fn ingest_loop( } IngestProtocol::OtlpGrpc => { let addr: SocketAddr = bind_addr.parse().map_err(|err| { - io::Error::new(io::ErrorKind::InvalidInput, format!("invalid gRPC bind addr: {err}")) + io::Error::new( + io::ErrorKind::InvalidInput, + format!("invalid gRPC bind addr: {err}"), + ) })?; eprintln!( "logjetd ingest listening on {}://{bind_addr} using otlp-grpc max-batch-bytes={} max-clients={}", - if ingest_tls.enable { "grpcs" } else { "grpc" } - , + if ingest_tls.enable { "grpcs" } else { "grpc" }, ingest_limits.max_batch_bytes, ingest_limits.max_clients ); @@ -424,8 +446,7 @@ fn otlp_http_tls_loop( let tls_server = load_ingest_server_config(&ingest_tls)?; eprintln!( "logjetd ingest listening on https://{bind_addr}/v1/logs using otlp-http max-batch-bytes={} max-clients={}", - ingest_limits.max_batch_bytes, - ingest_limits.max_clients + ingest_limits.max_batch_bytes, ingest_limits.max_clients ); for stream in listener.incoming() { @@ -473,7 +494,13 @@ fn handle_otlp_http_tls_client( let conn = ServerConnection::new(tls_server) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); - let result = handle_otlp_http_transport(&mut transport, spool, ingest_policy, next_seq, max_batch_bytes); + let result = handle_otlp_http_transport( + &mut transport, + spool, + ingest_policy, + next_seq, + max_batch_bytes, + ); transport.conn.send_close_notify(); let _ = transport.flush(); result @@ -488,7 +515,10 @@ fn handle_otlp_http_transport( ) -> io::Result<()> { let request = match read_http_request(transport, max_batch_bytes) { Ok(request) => request, - Err(err) if err.kind() == io::ErrorKind::InvalidData && err.to_string() == "payload too large" => { + Err(err) + if err.kind() == io::ErrorKind::InvalidData + && err.to_string() == "payload too large" => + { ingest_policy.note_oversize()?; write_http_response(transport, 413, "payload too large")?; return Ok(()); @@ -541,7 +571,10 @@ struct ParsedHttpRequest { body: Vec, } -fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io::Result { +fn read_http_request( + transport: &mut T, + max_batch_bytes: usize, +) -> io::Result { const MAX_HEADER_BYTES: usize = 16 * 1024; let mut buffer = Vec::new(); let mut byte = [0u8; 1]; @@ -550,7 +583,10 @@ fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io:: transport.read_exact(&mut byte)?; buffer.push(byte[0]); if buffer.len() > MAX_HEADER_BYTES { - return Err(io::Error::new(io::ErrorKind::InvalidData, "http header too large")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "http header too large", + )); } if buffer.ends_with(b"\r\n\r\n") { break; @@ -558,8 +594,9 @@ fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io:: } let header_end = buffer.len(); - let header_text = std::str::from_utf8(&buffer[..header_end - 4]) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "http header is not valid utf-8"))?; + let header_text = std::str::from_utf8(&buffer[..header_end - 4]).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "http header is not valid utf-8") + })?; let mut lines = header_text.lines(); let request_line = lines .next() @@ -588,14 +625,20 @@ fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io:: let content_length = content_length .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; if content_length > max_batch_bytes { - return Err(io::Error::new(io::ErrorKind::InvalidData, "payload too large")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "payload too large", + )); } let mut body = Vec::with_capacity(content_length); transport .take(content_length as u64) .read_to_end(&mut body)?; if body.len() != content_length { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "short http body")); + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "short http body", + )); } Ok(ParsedHttpRequest { method, path, body }) @@ -709,8 +752,14 @@ fn handle_ingest_client( let mut reader = BufReader::new(stream); while let Some(record) = read_record_with_limit(&mut reader, max_batch_bytes)? { - if matches!(ingest_policy.decide(BatchPriority::Unknown)?, IngestDecision::RejectRateLimited) { - eprintln!("logjetd ingest dropped wire record seq={} because ingest rate limit was exceeded", record.seq); + if matches!( + ingest_policy.decide(BatchPriority::Unknown)?, + IngestDecision::RejectRateLimited + ) { + eprintln!( + "logjetd ingest dropped wire record seq={} because ingest rate limit was exceeded", + record.seq + ); continue; } append_batch_record(&spool, record)?; @@ -805,8 +854,16 @@ fn handle_replay_transport( transport.flush()?; let request = read_replay_request(transport)?; - let mut last_seq = request.from_seq; - let mut seen_generation = spool.current_generation()?; + let (mut cursor, mut seen_generation) = { + let spool_guard = spool + .spool + .lock() + .map_err(|_| io::Error::other("spool mutex poisoned"))?; + ( + spool_guard.replay_cursor_after(request.from_seq)?, + spool.current_generation()?, + ) + }; eprintln!( "logjetd replay client requested records after seq={} mode={}", @@ -815,29 +872,41 @@ fn handle_replay_transport( ); if request.consume { - return handle_replay_transport_drain(transport, spool, &mut last_seq, &mut seen_generation); + return handle_replay_transport_drain(transport, spool, &mut cursor, &mut seen_generation); } loop { - let sent_any = { - let spool = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; - spool.replay_since(transport, &mut last_seq)? - }; + let mut sent_any = false; + loop { + let next_record = { + let spool = spool + .spool + .lock() + .map_err(|_| io::Error::other("spool mutex poisoned"))?; + spool.next_for_cursor(&mut cursor)? + }; - transport.flush()?; - if !sent_any { - spool.wait_for_change(&mut seen_generation)?; + let Some(record) = next_record else { + break; + }; + + write_record(transport, &record)?; + sent_any = true; } + + if sent_any { + transport.flush()?; + continue; + } + + spool.wait_for_change(&mut seen_generation)?; } } fn handle_replay_transport_drain( transport: &mut T, spool: Arc, - last_seq: &mut u64, + cursor: &mut crate::spool::ReplayCursor, seen_generation: &mut u64, ) -> io::Result<()> { loop { @@ -846,7 +915,7 @@ fn handle_replay_transport_drain( .spool .lock() .map_err(|_| io::Error::other("spool mutex poisoned"))?; - spool.next_after(*last_seq)? + spool.next_for_cursor(cursor)? }; let Some(record) = next_record else { @@ -861,7 +930,10 @@ fn handle_replay_transport_drain( if ack.ack_seq != record.seq { return Err(io::Error::new( io::ErrorKind::InvalidData, - format!("replay ack out of order: expected {} got {}", record.seq, ack.ack_seq), + format!( + "replay ack out of order: expected {} got {}", + record.seq, ack.ack_seq + ), )); } @@ -872,7 +944,6 @@ fn handle_replay_transport_drain( .map_err(|_| io::Error::other("spool mutex poisoned"))?; spool.consume_through(ack.ack_seq)?; } - *last_seq = ack.ack_seq; } } diff --git a/logjetd/src/main.rs b/logjetd/src/main.rs index 36c5137..0b3a5f8 100644 --- a/logjetd/src/main.rs +++ b/logjetd/src/main.rs @@ -64,9 +64,17 @@ fn run() -> Result<(), Box> { command = Some("replay"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => replay_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), - "--name" => replay_name = Some(args.next().ok_or("missing value for --name")?), - "--dest" => replay_dest = Some(args.next().ok_or("missing value for --dest")?), + "--path" => { + replay_path = Some(PathBuf::from( + args.next().ok_or("missing value for --path")?, + )) + } + "--name" => { + replay_name = Some(args.next().ok_or("missing value for --name")?) + } + "--dest" => { + replay_dest = Some(args.next().ok_or("missing value for --dest")?) + } _ => return Err(format!("unknown replay argument: {flag}").into()), } } @@ -76,8 +84,14 @@ fn run() -> Result<(), Box> { command = Some("segments"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => segment_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), - "--name" => segment_name = Some(args.next().ok_or("missing value for --name")?), + "--path" => { + segment_path = Some(PathBuf::from( + args.next().ok_or("missing value for --path")?, + )) + } + "--name" => { + segment_name = Some(args.next().ok_or("missing value for --name")?) + } _ => return Err(format!("unknown segments argument: {flag}").into()), } } @@ -87,8 +101,14 @@ fn run() -> Result<(), Box> { command = Some("prune"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => prune_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), - "--name" => prune_name = Some(args.next().ok_or("missing value for --name")?), + "--path" => { + prune_path = Some(PathBuf::from( + args.next().ok_or("missing value for --path")?, + )) + } + "--name" => { + prune_name = Some(args.next().ok_or("missing value for --name")?) + } "--keep-files" => { prune_keep_files = Some( args.next() @@ -113,7 +133,9 @@ fn run() -> Result<(), Box> { command = Some("bridge"); while let Some(flag) = args.next() { match flag.as_str() { - "--source" => bridge_source = Some(args.next().ok_or("missing value for --source")?), + "--source" => { + bridge_source = Some(args.next().ok_or("missing value for --source")?) + } _ => return Err(format!("unknown bridge argument: {flag}").into()), } } @@ -174,12 +196,14 @@ fn run() -> Result<(), Box> { None => return Err("missing bridge source; set --source or upstream.replay".into()), }; - eprintln!( - "bridging from {} to {}", - source, - config.collector.url - ); - bridge_wire_to_otlp_http(&source, &config.collector, &config.backpressure, &config.upstream, &config.tls)?; + eprintln!("bridging from {} to {}", source, config.collector.url); + bridge_wire_to_otlp_http( + &source, + &config.collector, + &config.backpressure, + &config.upstream, + &config.tls, + )?; } Some("prune") => { let path = prune_path.ok_or("missing --path")?; @@ -213,17 +237,25 @@ fn print_usage() { println!(" logjetd [serve] [-c|--config ]"); println!(" logjetd inspect "); println!(" logjetd segments --path --name "); - println!(" logjetd replay [-c|--config ] --path --name [--dest ]"); - println!(" logjetd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]"); + println!( + " logjetd replay [-c|--config ] --path --name [--dest ]" + ); + println!( + " logjetd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]" + ); println!(" logjetd bridge [-c|--config ] [--source ]"); println!(); println!("Commands:"); println!(" serve Run ingest and replay listeners using YAML configuration"); println!(" inspect Print stored record metadata from a .logjet file or spool directory"); println!(" segments Print ordered file-segment metadata for one rotated .logjet spool"); - println!(" replay Read ordered .logjet files and blast OTLP log batches to an OTLP/HTTP collector"); + println!( + " replay Read ordered .logjet files and blast OTLP log batches to an OTLP/HTTP collector" + ); println!(" prune Remove oldest rotated file segments by count or total bytes"); - println!(" bridge Connect to a replay listener, drain backlog, stay attached, and forward OTLP logs to the configured collector"); + println!( + " bridge Connect to a replay listener, drain backlog, stay attached, and forward OTLP logs to the configured collector" + ); println!(); println!("Config path defaults to /etc/logjet.conf."); } diff --git a/logjetd/src/protocol.rs b/logjetd/src/protocol.rs index 7328c11..5286e6d 100644 --- a/logjetd/src/protocol.rs +++ b/logjetd/src/protocol.rs @@ -41,7 +41,10 @@ pub fn read_record(reader: &mut R) -> io::Result> { read_record_with_limit(reader, usize::MAX) } -pub fn read_record_with_limit(reader: &mut R, max_payload_len: usize) -> io::Result> { +pub fn read_record_with_limit( + reader: &mut R, + max_payload_len: usize, +) -> io::Result> { let mut magic = [0u8; 8]; match reader.read_exact(&mut magic) { Ok(()) => {} @@ -87,18 +90,24 @@ pub fn read_record_with_limit(reader: &mut R, max_payload_len: usize) - Ok(Some(WireRecord { record_type, seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11], + header[4], header[5], header[6], header[7], header[8], header[9], header[10], + header[11], ]), ts_unix_ns: u64::from_le_bytes([ - header[12], header[13], header[14], header[15], header[16], header[17], header[18], header[19], + header[12], header[13], header[14], header[15], header[16], header[17], header[18], + header[19], ]), payload, })) } pub fn write_record(writer: &mut W, record: &WireRecord) -> io::Result<()> { - let payload_len = u32::try_from(record.payload.len()) - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "payload too large for wire protocol"))?; + let payload_len = u32::try_from(record.payload.len()).map_err(|_| { + io::Error::new( + ErrorKind::InvalidInput, + "payload too large for wire protocol", + ) + })?; writer.write_all(&WIRE_MAGIC)?; writer.write_all(&[WIRE_VERSION, record.record_type as u8])?; @@ -133,7 +142,8 @@ pub fn read_replay_request(reader: &mut R) -> io::Result Ok(ReplayRequest { consume: header[1] & 0x01 != 0, from_seq: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15], + header[8], header[9], header[10], header[11], header[12], header[13], header[14], + header[15], ]), }) } @@ -169,13 +179,16 @@ pub fn read_replay_hello(reader: &mut R) -> io::Result { Ok(ReplayHello { stream_id: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15], + header[8], header[9], header[10], header[11], header[12], header[13], header[14], + header[15], ]), first_seq: u64::from_le_bytes([ - header[16], header[17], header[18], header[19], header[20], header[21], header[22], header[23], + header[16], header[17], header[18], header[19], header[20], header[21], header[22], + header[23], ]), last_seq: u64::from_le_bytes([ - header[24], header[25], header[26], header[27], header[28], header[29], header[30], header[31], + header[24], header[25], header[26], header[27], header[28], header[29], header[30], + header[31], ]), }) } @@ -212,7 +225,8 @@ pub fn read_replay_ack(reader: &mut R) -> io::Result { Ok(ReplayAck { ack_seq: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15], + header[8], header[9], header[10], header[11], header[12], header[13], header[14], + header[15], ]), }) } diff --git a/logjetd/src/replay.rs b/logjetd/src/replay.rs index 1d82dbf..2e92eb1 100644 --- a/logjetd/src/replay.rs +++ b/logjetd/src/replay.rs @@ -9,15 +9,24 @@ use std::time::Duration; use logjet::{LogjetReader, ReaderConfig, RecordType}; use rustls::{ClientConfig, ClientConnection, StreamOwned}; -use crate::config::{BackpressureConfig, BackpressureMode, CollectorConfig, TlsConfig, UpstreamConfig, UpstreamMode}; +use crate::config::{ + BackpressureConfig, BackpressureMode, CollectorConfig, TlsConfig, UpstreamConfig, UpstreamMode, +}; use crate::protocol::{ ReplayAck, ReplayHello, ReplayRequest, read_record, read_replay_hello, write_replay_ack, write_replay_request, }; use crate::spool::list_named_segments; -use crate::tls::{load_client_config, load_collector_client_config, parse_collector_server_name, parse_server_name}; +use crate::tls::{ + load_client_config, load_collector_client_config, parse_collector_server_name, + parse_server_name, +}; -pub fn replay_path_to_otlp_http(path: &Path, name: &str, collector: &CollectorConfig) -> io::Result { +pub fn replay_path_to_otlp_http( + path: &Path, + name: &str, + collector: &CollectorConfig, +) -> io::Result { let mut sent = 0u64; let endpoint = CollectorEndpoint::parse(&collector.url)?; let transport = CollectorTransport { @@ -92,7 +101,8 @@ pub fn bridge_wire_to_otlp_http( "bridge resume state file {} loaded seq={} stream-id={}", path.display(), state.last_seq, - state.stream_id + state + .stream_id .map(|value| value.to_string()) .unwrap_or_else(|| "unset".to_string()) ); @@ -111,15 +121,13 @@ pub fn bridge_wire_to_otlp_http( Ok(()) => { eprintln!( "bridge source {source} closed after seq={}; reconnecting in {} ms", - state.last_seq, - upstream.retry_ms + state.last_seq, upstream.retry_ms ); } Err(err) => { eprintln!( "bridge source {source} error after seq={}: {err}; reconnecting in {} ms", - state.last_seq, - upstream.retry_ms + state.last_seq, upstream.retry_ms ); } } @@ -145,11 +153,23 @@ fn bridge_once( let conn = ClientConnection::new(client_config, server_name) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); - return bridge_transport(source, state, state_file, &mut transport, collector_transport); + return bridge_transport( + source, + state, + state_file, + &mut transport, + collector_transport, + ); } let mut transport = stream; - bridge_transport(source, state, state_file, &mut transport, collector_transport) + bridge_transport( + source, + state, + state_file, + &mut transport, + collector_transport, + ) } fn bridge_transport( @@ -164,7 +184,13 @@ fn bridge_transport( write_bridge_state(state_file, state)?; let consume = collector_transport.upstream_mode == UpstreamMode::Drain; - write_replay_request(transport, &ReplayRequest { from_seq: state.last_seq, consume })?; + write_replay_request( + transport, + &ReplayRequest { + from_seq: state.last_seq, + consume, + }, + )?; transport.flush()?; eprintln!( "bridge connected to {} and requested records after seq={} mode={} backpressure={}", @@ -191,7 +217,15 @@ fn bridge_transport( let mut pending = std::collections::VecDeque::new(); while let Some(record) = read_record(transport)? { - flush_ready_results(transport, state, state_file, consume, &mut pending, &result_rx, false)?; + flush_ready_results( + transport, + state, + state_file, + consume, + &mut pending, + &result_rx, + false, + )?; if record.record_type != RecordType::Logs { commit_record(transport, state, state_file, consume, record.seq)?; @@ -199,7 +233,14 @@ fn bridge_transport( } let seq = record.seq; - match enqueue_export_task(&task_tx, collector_transport, ExportTask { seq, payload: record.payload }) { + match enqueue_export_task( + &task_tx, + collector_transport, + ExportTask { + seq, + payload: record.payload, + }, + ) { Ok(EnqueueOutcome::Queued) => pending.push_back(PendingExport::Queued(seq)), Ok(EnqueueOutcome::DroppedNewest) => pending.push_back(PendingExport::Dropped(seq)), Err(err) => { @@ -211,7 +252,15 @@ fn bridge_transport( } drop(task_tx); - flush_ready_results(transport, state, state_file, consume, &mut pending, &result_rx, true)?; + flush_ready_results( + transport, + state, + state_file, + consume, + &mut pending, + &result_rx, + true, + )?; match exporter.join() { Ok(Ok(())) => Ok(()), Ok(Err(err)) => Err(err), @@ -225,19 +274,24 @@ fn enqueue_export_task( task: ExportTask, ) -> io::Result { match collector_transport.backpressure_mode { - BackpressureMode::Block => task_tx - .send(task) - .map(|()| EnqueueOutcome::Queued) - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")), + BackpressureMode::Block => { + task_tx + .send(task) + .map(|()| EnqueueOutcome::Queued) + .map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped") + }) + } BackpressureMode::Disconnect => match task_tx.try_send(task) { Ok(()) => Ok(EnqueueOutcome::Queued), Err(mpsc::TrySendError::Full(_)) => Err(io::Error::new( io::ErrorKind::TimedOut, "collector export buffer is full; disconnecting bridge", )), - Err(mpsc::TrySendError::Disconnected(_)) => { - Err(io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")) - } + Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "collector export worker stopped", + )), }, BackpressureMode::DropNewest => match task_tx.try_send(task) { Ok(()) => Ok(EnqueueOutcome::Queued), @@ -248,9 +302,10 @@ fn enqueue_export_task( ); Ok(EnqueueOutcome::DroppedNewest) } - Err(mpsc::TrySendError::Disconnected(_)) => { - Err(io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")) - } + Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "collector export worker stopped", + )), }, } } @@ -261,9 +316,16 @@ fn export_worker( result_tx: mpsc::Sender, ) -> io::Result<()> { while let Ok(task) = task_rx.recv() { - let outcome = post_raw_otlp_http(&collector_transport, &task.payload).map(|()| ExportOutcome::Delivered); + let outcome = post_raw_otlp_http(&collector_transport, &task.payload) + .map(|()| ExportOutcome::Delivered); let failed = outcome.is_err(); - if result_tx.send(ExportResult { seq: task.seq, outcome }).is_err() { + if result_tx + .send(ExportResult { + seq: task.seq, + outcome, + }) + .is_err() + { break; } if failed { @@ -294,9 +356,9 @@ fn flush_ready_results( }, PendingExport::Queued(expected_seq) => { let result = if block { - result_rx - .recv() - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped"))? + result_rx.recv().map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped") + })? } else { match result_rx.try_recv() { Ok(result) => result, @@ -305,7 +367,7 @@ fn flush_ready_results( return Err(io::Error::new( io::ErrorKind::BrokenPipe, "collector export worker stopped", - )) + )); } } }; @@ -324,7 +386,9 @@ fn flush_ready_results( pending.pop_front(); match result.outcome { - Ok(ExportOutcome::Delivered) => commit_record(transport, state, state_file, consume, result.seq)?, + Ok(ExportOutcome::Delivered) => { + commit_record(transport, state, state_file, consume, result.seq)? + } Ok(ExportOutcome::DroppedNewest) => { commit_record(transport, state, state_file, consume, result.seq)?; } @@ -350,7 +414,10 @@ fn commit_record( } fn post_raw_otlp_http(collector_transport: &CollectorTransport, payload: &[u8]) -> io::Result<()> { - let stream = connect_with_timeout(&collector_transport.endpoint.authority, collector_transport.timeout)?; + let stream = connect_with_timeout( + &collector_transport.endpoint.authority, + collector_transport.timeout, + )?; if !collector_transport.backpressure_enabled { stream.set_write_timeout(Some(collector_transport.timeout))?; stream.set_read_timeout(Some(collector_transport.timeout))?; @@ -375,7 +442,11 @@ fn post_raw_otlp_http(collector_transport: &CollectorTransport, payload: &[u8]) let conn = ClientConnection::new(client_config.clone(), server_name) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut tls_transport = StreamOwned::new(conn, stream); - return post_raw_otlp_http_transport(&collector_transport.endpoint, payload, &mut tls_transport); + return post_raw_otlp_http_transport( + &collector_transport.endpoint, + payload, + &mut tls_transport, + ); } let mut plain_transport = stream; @@ -470,43 +541,54 @@ fn parse_bridge_state(text: &str) -> io::Result { Some(None) } else { Some(Some(value.parse::().map_err(|err| { - io::Error::new(io::ErrorKind::InvalidData, format!("invalid bridge state stream_id: {err}")) + io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid bridge state stream_id: {err}"), + ) })?)) }; } "last_seq" => { last_seq = Some(value.trim().parse::().map_err(|err| { - io::Error::new(io::ErrorKind::InvalidData, format!("invalid bridge state last_seq: {err}")) + io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid bridge state last_seq: {err}"), + ) })?); } _ => {} } } - let last_seq = last_seq - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid bridge state: missing last_seq"))?; + let last_seq = last_seq.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "invalid bridge state: missing last_seq", + ) + })?; Ok(BridgeState { stream_id: stream_id.unwrap_or(None), last_seq, }) } -fn reconcile_bridge_state(source: &str, state: &mut BridgeState, hello: &ReplayHello) -> io::Result<()> { +fn reconcile_bridge_state( + source: &str, + state: &mut BridgeState, + hello: &ReplayHello, +) -> io::Result<()> { if let Some(saved_stream_id) = state.stream_id { if saved_stream_id != hello.stream_id { eprintln!( "bridge source {source} changed stream identity {} -> {}; resetting saved seq from {} to 0", - saved_stream_id, - hello.stream_id, - state.last_seq + saved_stream_id, hello.stream_id, state.last_seq ); state.last_seq = 0; } } else if state.last_seq > 0 && hello.last_seq > 0 && hello.last_seq < state.last_seq { eprintln!( "bridge source {source} appears to have reset or been replaced; upstream last_seq={} is below saved seq={}; resetting to 0", - hello.last_seq, - state.last_seq + hello.last_seq, state.last_seq ); state.last_seq = 0; } diff --git a/logjetd/src/spool.rs b/logjetd/src/spool.rs index 3ed872c..650fd0b 100644 --- a/logjetd/src/spool.rs +++ b/logjetd/src/spool.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use logjet::{LogjetReader, LogjetWriter, OwnedRecord, ReaderConfig}; use crate::config::{BufferConfig, BufferLimit, FileConfig, StorageConfig}; -use crate::protocol::{WireRecord, write_record}; +use crate::protocol::WireRecord; #[derive(Debug)] pub enum Spool { @@ -53,6 +53,24 @@ pub struct SegmentSummary { pub last_seq: Option, } +#[derive(Debug, Clone)] +pub enum ReplayCursor { + Buffer(BufferReplayCursor), + File(FileReplayCursor), +} + +#[derive(Debug, Clone)] +pub struct BufferReplayCursor { + next_index: usize, + last_seq: u64, +} + +#[derive(Debug, Clone)] +pub struct FileReplayCursor { + next_segment_id_hint: u64, + last_seq: u64, +} + impl Spool { pub fn open(config: StorageConfig) -> io::Result { match config { @@ -71,17 +89,25 @@ impl Spool { } } - pub fn replay_since(&self, writer: &mut W, last_seq: &mut u64) -> io::Result { + pub fn replay_cursor_after(&self, last_seq: u64) -> io::Result { match self { - Self::Buffer(spool) => spool.replay_since(writer, last_seq), - Self::File(spool) => spool.replay_since(writer, last_seq), + Self::Buffer(spool) => Ok(ReplayCursor::Buffer(spool.replay_cursor_after(last_seq))), + Self::File(spool) => Ok(ReplayCursor::File(spool.replay_cursor_after(last_seq))), } } - pub fn next_after(&self, last_seq: u64) -> io::Result> { - match self { - Self::Buffer(spool) => Ok(spool.next_after(last_seq)), - Self::File(spool) => spool.next_after(last_seq), + pub fn next_for_cursor(&self, cursor: &mut ReplayCursor) -> io::Result> { + match (self, cursor) { + (Self::Buffer(spool), ReplayCursor::Buffer(cursor)) => { + Ok(spool.next_for_cursor(cursor)) + } + (Self::File(spool), ReplayCursor::File(cursor)) => spool.next_for_cursor(cursor), + (Self::Buffer(_), ReplayCursor::File(_)) | (Self::File(_), ReplayCursor::Buffer(_)) => { + Err(io::Error::new( + ErrorKind::InvalidInput, + "replay cursor type does not match spool type", + )) + } } } @@ -141,27 +167,45 @@ impl BufferSpool { self.enforce_limits(); } - fn replay_since(&self, writer: &mut W, last_seq: &mut u64) -> io::Result { - let mut sent_any = false; + fn replay_cursor_after(&self, last_seq: u64) -> BufferReplayCursor { + let next_index = self + .records + .iter() + .position(|record| record.seq > last_seq) + .unwrap_or(self.records.len()); + BufferReplayCursor { + next_index, + last_seq, + } + } - for record in &self.records { - if record.seq <= *last_seq { - continue; - } + fn next_for_cursor(&self, cursor: &mut BufferReplayCursor) -> Option { + let index_still_aligned = match cursor.next_index { + 0 => true, + value => self + .records + .get(value.saturating_sub(1)) + .map(|record| record.seq <= cursor.last_seq) + .unwrap_or(false), + }; - write_record(writer, record)?; - *last_seq = record.seq; - sent_any = true; + if index_still_aligned + && let Some(record) = self.records.get(cursor.next_index) + && record.seq > cursor.last_seq + { + cursor.next_index += 1; + cursor.last_seq = record.seq; + return Some(record.clone()); } - Ok(sent_any) - } - - fn next_after(&self, last_seq: u64) -> Option { - self.records + let next_index = self + .records .iter() - .find(|record| record.seq > last_seq) - .cloned() + .position(|record| record.seq > cursor.last_seq)?; + let record = self.records.get(next_index)?.clone(); + cursor.next_index = next_index + 1; + cursor.last_seq = record.seq; + Some(record) } fn consume_through(&mut self, seq: u64) { @@ -225,10 +269,9 @@ impl FileSpool { if size < config.segment_size_bytes { (segment.id, segment.path.clone(), size) } else { - let next_id = segment - .id - .checked_add(1) - .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; + let next_id = segment.id.checked_add(1).ok_or_else(|| { + io::Error::new(ErrorKind::InvalidData, "segment id overflow") + })?; (next_id, segment_path(&config.dir, &base_stem, next_id), 0) } } @@ -262,7 +305,12 @@ impl FileSpool { } self.active_writer - .push(record.record_type, record.seq, record.ts_unix_ns, &record.payload) + .push( + record.record_type, + record.seq, + record.ts_unix_ns, + &record.payload, + ) .map_err(to_io_error)?; self.active_writer.flush_block().map_err(to_io_error)?; self.refresh_active_size()?; @@ -274,48 +322,32 @@ impl FileSpool { Ok(()) } - fn replay_since(&self, writer: &mut W, last_seq: &mut u64) -> io::Result { - let mut sent_any = false; - let floor_seq = (*last_seq).max(self.consumed_through_seq); - - for segment in list_segments(&self.dir, &self.base_stem)? { - let file = File::open(&segment.path)?; - let mut reader = LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); - - while let Some(record) = reader.next_record().map_err(to_io_error)? { - if record.seq <= floor_seq || record.seq <= *last_seq { - continue; - } - - write_record( - writer, - &WireRecord { - record_type: record.record_type, - seq: record.seq, - ts_unix_ns: record.ts_unix_ns, - payload: record.payload, - }, - )?; - *last_seq = record.seq; - sent_any = true; - } + fn replay_cursor_after(&self, last_seq: u64) -> FileReplayCursor { + FileReplayCursor { + next_segment_id_hint: 0, + last_seq, } - - Ok(sent_any) } - fn next_after(&self, last_seq: u64) -> io::Result> { - let floor_seq = last_seq.max(self.consumed_through_seq); + fn next_for_cursor(&self, cursor: &mut FileReplayCursor) -> io::Result> { + let floor_seq = cursor.last_seq.max(self.consumed_through_seq); for segment in list_segments(&self.dir, &self.base_stem)? { + if segment.id < cursor.next_segment_id_hint { + continue; + } + let file = File::open(&segment.path)?; - let mut reader = LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); + let mut reader = + LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); while let Some(record) = reader.next_record().map_err(to_io_error)? { if record.seq <= floor_seq { continue; } + cursor.last_seq = record.seq; + cursor.next_segment_id_hint = segment.id; return Ok(Some(WireRecord { record_type: record.record_type, seq: record.seq, @@ -323,6 +355,8 @@ impl FileSpool { payload: record.payload, })); } + + cursor.next_segment_id_hint = segment.id; } Ok(None) @@ -407,7 +441,8 @@ impl FileSpool { for segment in list_segments(&self.dir, &self.base_stem)? { let file = File::open(&segment.path)?; - let mut reader = LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); + let mut reader = + LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); while let Some(record) = reader.next_record().map_err(to_io_error)? { if record.seq <= self.consumed_through_seq { @@ -431,7 +466,9 @@ pub fn inspect_path(path: &Path) -> io::Result<()> { if path.is_dir() { for entry in fs::read_dir(path)? { let entry = entry?; - if entry.file_type()?.is_file() && entry.path().extension().and_then(|ext| ext.to_str()) == Some("logjet") { + if entry.file_type()?.is_file() + && entry.path().extension().and_then(|ext| ext.to_str()) == Some("logjet") + { inspect_file(&entry.path())?; } } @@ -597,7 +634,10 @@ fn list_segments(dir: &Path, base_stem: &str) -> io::Result> { continue; }; - segments.push(SegmentInfo { id, path: entry.path() }); + segments.push(SegmentInfo { + id, + path: entry.path(), + }); } segments.sort_by_key(|segment| segment.id); @@ -654,10 +694,12 @@ fn read_consumed_state(path: &Path) -> io::Result { } let text = fs::read_to_string(path)?; - let seq = text - .trim() - .parse::() - .map_err(|err| io::Error::new(ErrorKind::InvalidData, format!("invalid consumed state: {err}")))?; + let seq = text.trim().parse::().map_err(|err| { + io::Error::new( + ErrorKind::InvalidData, + format!("invalid consumed state: {err}"), + ) + })?; Ok(seq) } @@ -668,10 +710,9 @@ fn write_consumed_state(path: &Path, seq: u64) -> io::Result<()> { fn read_or_create_stream_id(path: &Path) -> io::Result { if path.exists() { let text = fs::read_to_string(path)?; - let stream_id = text - .trim() - .parse::() - .map_err(|err| io::Error::new(ErrorKind::InvalidData, format!("invalid stream id: {err}")))?; + let stream_id = text.trim().parse::().map_err(|err| { + io::Error::new(ErrorKind::InvalidData, format!("invalid stream id: {err}")) + })?; return Ok(stream_id); } diff --git a/logjetd/src/tls.rs b/logjetd/src/tls.rs index 9139aa1..33b6352 100644 --- a/logjetd/src/tls.rs +++ b/logjetd/src/tls.rs @@ -52,7 +52,10 @@ pub fn parse_server_name(tls: &TlsConfig, authority: &str) -> io::Result io::Result> { +pub fn parse_collector_server_name( + collector: &CollectorConfig, + authority: &str, +) -> io::Result> { parse_server_name_override(collector.server_name.as_deref(), authority) } @@ -60,10 +63,16 @@ pub fn authority_host(authority: &str) -> &str { if let Some(rest) = authority.strip_prefix('[') { return rest.split(']').next().unwrap_or(authority); } - authority.rsplit_once(':').map(|(host, _)| host).unwrap_or(authority) + authority + .rsplit_once(':') + .map(|(host, _)| host) + .unwrap_or(authority) } -fn parse_server_name_override(override_name: Option<&str>, authority: &str) -> io::Result> { +fn parse_server_name_override( + override_name: Option<&str>, + authority: &str, +) -> io::Result> { let name = override_name.unwrap_or_else(|| authority_host(authority)); if let Ok(ip) = name.parse::() { return Ok(ServerName::IpAddress(ip.into())); @@ -144,7 +153,9 @@ fn load_client_config_from_parts( _ => { return Err(io::Error::new( io::ErrorKind::InvalidInput, - format!("{namespace}.cert-file and {namespace}.key-file must either both be set or both be unset"), + format!( + "{namespace}.cert-file and {namespace}.key-file must either both be set or both be unset" + ), )); } }; @@ -159,7 +170,10 @@ fn load_root_store(path: &Path) -> io::Result { if ignored > 0 { return Err(io::Error::new( io::ErrorKind::InvalidInput, - format!("failed to parse {ignored} CA certificate(s) from {}", path.display()), + format!( + "failed to parse {ignored} CA certificate(s) from {}", + path.display() + ), )); } Ok(roots) @@ -176,7 +190,12 @@ fn load_private_key(path: &Path) -> io::Result> { let mut reader = BufReader::new(File::open(path)?); rustls_pemfile::private_key(&mut reader) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, format!("no private key found in {}", path.display()))) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("no private key found in {}", path.display()), + ) + }) } #[cfg(test)] diff --git a/src/error.rs b/src/error.rs index aeec346..167fd35 100644 --- a/src/error.rs +++ b/src/error.rs @@ -8,8 +8,14 @@ pub enum Error { UnknownCodec(u8), UnknownVersion(u8), HeaderTooShort(u16), - HeaderCrcMismatch { expected: u32, actual: u32 }, - BlockCrcMismatch { expected: u32, actual: u32 }, + HeaderCrcMismatch { + expected: u32, + actual: u32, + }, + BlockCrcMismatch { + expected: u32, + actual: u32, + }, LengthTooLarge { field: &'static str, value: u64, @@ -19,7 +25,10 @@ pub enum Error { Truncated(&'static str), VarintTooLong, NumericOverflow(&'static str), - RecordTooLarge { encoded_len: usize, block_target_size: usize }, + RecordTooLarge { + encoded_len: usize, + block_target_size: usize, + }, Codec(String), } @@ -34,12 +43,22 @@ impl Display for Error { Self::UnknownVersion(value) => write!(f, "unknown version: {value}"), Self::HeaderTooShort(value) => write!(f, "header too short: {value}"), Self::HeaderCrcMismatch { expected, actual } => { - write!(f, "header crc mismatch: expected {expected:#010x}, got {actual:#010x}") + write!( + f, + "header crc mismatch: expected {expected:#010x}, got {actual:#010x}" + ) } Self::BlockCrcMismatch { expected, actual } => { - write!(f, "block crc mismatch: expected {expected:#010x}, got {actual:#010x}") + write!( + f, + "block crc mismatch: expected {expected:#010x}, got {actual:#010x}" + ) } - Self::LengthTooLarge { field, value, limit } => { + Self::LengthTooLarge { + field, + value, + limit, + } => { write!(f, "{field} too large: {value} > {limit}") } Self::InvalidHeader(msg) => write!(f, "invalid header: {msg}"), diff --git a/src/format.rs b/src/format.rs index 1ba656d..34762c7 100644 --- a/src/format.rs +++ b/src/format.rs @@ -123,7 +123,8 @@ impl BlockHeaderExt { bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]), base_ts_unix_ns: u64::from_le_bytes([ - bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], + bytes[15], ]), }) } diff --git a/src/lib.rs b/src/lib.rs index 3460bca..d17eaf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,8 +14,9 @@ pub mod writer; pub use codec::Codec; pub use error::{Error, Result}; pub use format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, DEFAULT_BLOCK_TARGET_SIZE, - DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, FORMAT_VERSION, BlockHeader, BlockHeaderExt, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, + BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, + FORMAT_VERSION, }; pub use reader::{LogjetReader, ReaderConfig, ReaderStats}; pub use record::{OwnedRecord, Record, RecordType}; diff --git a/src/reader.rs b/src/reader.rs index 8700715..57b79bf 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -4,8 +4,8 @@ use std::io::{ErrorKind, Read, Seek, SeekFrom}; use crate::crc::crc32c; use crate::error::{Error, Result}; use crate::format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, DEFAULT_MAX_BLOCK_SIZE, - DEFAULT_SYNC_MARKER, BlockHeader, BlockHeaderExt, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, + BlockHeaderExt, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, }; use crate::record::{OwnedRecord, RecordType}; @@ -226,11 +226,9 @@ impl LogjetReader { } let mut payload = Vec::with_capacity(header.uncompressed_len as usize); - header.codec.decompress( - &compressed, - header.uncompressed_len as usize, - &mut payload, - )?; + header + .codec + .decompress(&compressed, header.uncompressed_len as usize, &mut payload)?; let records = parse_records(&payload, header.record_count, ext)?; Ok(records) diff --git a/src/writer.rs b/src/writer.rs index 07c64c8..8106e22 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -4,8 +4,8 @@ use crate::codec::Codec; use crate::crc::crc32c; use crate::error::{Error, Result}; use crate::format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, DEFAULT_BLOCK_TARGET_SIZE, - DEFAULT_SYNC_MARKER, FORMAT_VERSION, BlockHeader, BlockHeaderExt, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, + BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, DEFAULT_SYNC_MARKER, FORMAT_VERSION, }; use crate::record::RecordType; @@ -76,12 +76,12 @@ impl LogjetWriter { } }; - let seq_delta = seq - .checked_sub(base_seq) - .ok_or(Error::InvalidHeader("sequence must be monotonic within block"))?; - let ts_delta = ts_unix_ns - .checked_sub(base_ts) - .ok_or(Error::InvalidHeader("timestamp must be monotonic within block"))?; + let seq_delta = seq.checked_sub(base_seq).ok_or(Error::InvalidHeader( + "sequence must be monotonic within block", + ))?; + let ts_delta = ts_unix_ns.checked_sub(base_ts).ok_or(Error::InvalidHeader( + "timestamp must be monotonic within block", + ))?; self.encoded_record_buf.push(record_type as u8); encode_varint(seq_delta, &mut self.encoded_record_buf)?; @@ -141,14 +141,14 @@ impl LogjetWriter { .codec .compress(&self.payload_buf, &mut self.compressed_buf)?; - let uncompressed_len = u32::try_from(self.payload_buf.len()) - .map_err(|_| Error::LengthTooLarge { + let uncompressed_len = + u32::try_from(self.payload_buf.len()).map_err(|_| Error::LengthTooLarge { field: "uncompressed_len", value: self.payload_buf.len() as u64, limit: u32::MAX as usize, })?; - let compressed_len = u32::try_from(self.compressed_buf.len()) - .map_err(|_| Error::LengthTooLarge { + let compressed_len = + u32::try_from(self.compressed_buf.len()).map_err(|_| Error::LengthTooLarge { field: "compressed_len", value: self.compressed_buf.len() as u64, limit: u32::MAX as usize, From b7df0558b95fc75ff402c5b624b17b252d691620 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:49:38 +0100 Subject: [PATCH 09/17] Update unit tests for forward replay --- logjetd/src/daemon_utst.rs | 14 +- logjetd/src/replay_utst.rs | 15 +- logjetd/src/spool_utst.rs | 268 ++++++++++++++++++++++++---------- logjetd/src/tls_utst.rs | 3 +- logjetd/tests/bridge_flows.rs | 155 +++++++++++++++++--- logjetd/tests/common/mod.rs | 60 ++++++-- tests/logjet.rs | 7 +- 7 files changed, 393 insertions(+), 129 deletions(-) diff --git a/logjetd/src/daemon_utst.rs b/logjetd/src/daemon_utst.rs index 7d5582b..0330d46 100644 --- a/logjetd/src/daemon_utst.rs +++ b/logjetd/src/daemon_utst.rs @@ -1,6 +1,6 @@ use super::{ - BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, classify_otlp_batch_priority, - read_http_request, write_http_response, + BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, + classify_otlp_batch_priority, read_http_request, write_http_response, }; use crate::config::{IngestOverloadConfig, SeverityFloor}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; @@ -117,7 +117,10 @@ fn ingest_policy_rate_limits_low_priority_batches() { priority_severity_floor: SeverityFloor::Error, report_every_ms: 0, }); - assert_eq!(policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::Accept); + assert_eq!( + policy.decide(BatchPriority::Warn).unwrap(), + IngestDecision::Accept + ); assert_eq!( policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::RejectRateLimited @@ -131,7 +134,10 @@ fn ingest_policy_allows_priority_bypass_above_threshold() { priority_severity_floor: SeverityFloor::Error, report_every_ms: 0, }); - assert_eq!(policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::Accept); + assert_eq!( + policy.decide(BatchPriority::Warn).unwrap(), + IngestDecision::Accept + ); assert_eq!( policy.decide(BatchPriority::Error).unwrap(), IngestDecision::AcceptPriorityBypass diff --git a/logjetd/src/replay_utst.rs b/logjetd/src/replay_utst.rs index ac55222..5a998b8 100644 --- a/logjetd/src/replay_utst.rs +++ b/logjetd/src/replay_utst.rs @@ -1,6 +1,7 @@ use super::{ - BridgeState, CollectorEndpoint, CollectorTransport, EnqueueOutcome, ExportTask, enqueue_export_task, - parse_bridge_state, read_bridge_state, reconcile_bridge_state, write_bridge_state, + BridgeState, CollectorEndpoint, CollectorTransport, EnqueueOutcome, ExportTask, + enqueue_export_task, parse_bridge_state, read_bridge_state, reconcile_bridge_state, + write_bridge_state, }; use crate::config::{BackpressureMode, CollectorConfig, UpstreamMode}; use crate::protocol::ReplayHello; @@ -191,7 +192,10 @@ fn drop_newest_mode_reports_drop_when_export_queue_is_full() { assert_eq!(outcome, EnqueueOutcome::DroppedNewest); } -fn test_collector_transport(mode: BackpressureMode, max_buffered_records: usize) -> CollectorTransport { +fn test_collector_transport( + mode: BackpressureMode, + max_buffered_records: usize, +) -> CollectorTransport { CollectorTransport { endpoint: CollectorEndpoint { authority: "127.0.0.1:4318".to_string(), @@ -220,5 +224,8 @@ fn unique_temp_path(label: &str) -> PathBuf { .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.state", std::process::id())) + std::env::temp_dir().join(format!( + "logjetd-{label}-{nanos}-{}.state", + std::process::id() + )) } diff --git a/logjetd/src/spool_utst.rs b/logjetd/src/spool_utst.rs index e0e3dba..40da180 100644 --- a/logjetd/src/spool_utst.rs +++ b/logjetd/src/spool_utst.rs @@ -48,25 +48,28 @@ fn message_limit_rotates_tail_only() { #[test] fn replay_since_only_sends_newer_records() { - let mut spool = BufferSpool::new(BufferConfig { + let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { limit: BufferLimit::Messages(8), keep_messages: 1, - }); + })) + .unwrap(); for seq in 1..=4 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8], + }) + .unwrap(); } let mut bytes = Vec::new(); - let mut last_seq = 2; - let sent_any = spool.replay_since(&mut bytes, &mut last_seq).unwrap(); - assert!(sent_any); - assert_eq!(last_seq, 4); + let mut cursor = spool.replay_cursor_after(2).unwrap(); + while let Some(record) = spool.next_for_cursor(&mut cursor).unwrap() { + crate::protocol::write_record(&mut bytes, &record).unwrap(); + } let mut reader = bytes.as_slice(); let mut seen = Vec::new(); @@ -97,6 +100,38 @@ fn consume_through_removes_buffer_records() { assert_eq!(kept, vec![4, 5]); } +#[test] +fn buffer_replay_cursor_resyncs_after_front_records_are_consumed() { + let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { + limit: BufferLimit::Messages(8), + keep_messages: 1, + })) + .unwrap(); + + for seq in 1..=5 { + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8], + }) + .unwrap(); + } + + let mut cursor = spool.replay_cursor_after(0).unwrap(); + let first = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(first.seq, 1); + + spool.consume_through(3).unwrap(); + + let next = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(next.seq, 4); + let final_record = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(final_record.seq, 5); + assert!(spool.next_for_cursor(&mut cursor).unwrap().is_none()); +} + #[test] fn file_spool_consume_state_survives_reopen() { let dir = unique_temp_dir("file-consume"); @@ -109,26 +144,64 @@ fn file_spool_consume_state_survives_reopen() { { let mut spool = Spool::open(StorageConfig::File(config.clone())).unwrap(); for seq in 1..=3 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8], + }) + .unwrap(); } spool.consume_through(2).unwrap(); } { let spool = Spool::open(StorageConfig::File(config)).unwrap(); - let next = spool.next_after(0).unwrap().unwrap(); + let mut cursor = spool.replay_cursor_after(0).unwrap(); + let next = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); assert_eq!(next.seq, 3); } fs::remove_dir_all(dir).unwrap(); } +#[test] +fn file_replay_cursor_skips_consumed_records_after_cleanup() { + let dir = unique_temp_dir("file-cursor-consume"); + let config = FileConfig { + dir: dir.clone(), + name: "bofh.logjet".to_string(), + segment_size_bytes: 1, + }; + + let mut spool = Spool::open(StorageConfig::File(config)).unwrap(); + for seq in 1..=4 { + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8; 8], + }) + .unwrap(); + } + + let mut cursor = spool.replay_cursor_after(0).unwrap(); + let first = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(first.seq, 1); + + spool.consume_through(2).unwrap(); + + let next = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(next.seq, 3); + let final_record = spool.next_for_cursor(&mut cursor).unwrap().unwrap(); + assert_eq!(final_record.seq, 4); + assert!(spool.next_for_cursor(&mut cursor).unwrap().is_none()); + + fs::remove_dir_all(dir).unwrap(); +} + #[test] fn list_named_segments_orders_numeric_suffixes() { let dir = unique_temp_dir("segments"); @@ -141,9 +214,24 @@ fn list_named_segments_orders_numeric_suffixes() { let segments = super::list_named_segments(&dir, "bofh.logjet").unwrap(); let names: Vec = segments .iter() - .map(|segment| segment.path.file_name().unwrap().to_string_lossy().into_owned()) + .map(|segment| { + segment + .path + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() + }) .collect(); - assert_eq!(names, vec!["bofh.logjet", "bofh-1.logjet", "bofh-2.logjet", "bofh-10.logjet"]); + assert_eq!( + names, + vec![ + "bofh.logjet", + "bofh-1.logjet", + "bofh-2.logjet", + "bofh-10.logjet" + ] + ); fs::remove_dir_all(dir).unwrap(); } @@ -159,13 +247,14 @@ fn file_spool_rotates_when_segment_size_is_exceeded() { .unwrap(); for seq in 1..=2 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![0u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![0u8; 8], + }) + .unwrap(); } assert!(dir.join("bofh.logjet").exists()); @@ -183,13 +272,14 @@ fn file_spool_reuses_existing_non_full_segment() { segment_size_bytes: 1024 * 1024, })) .unwrap(); - spool.append(WireRecord { - record_type: RecordType::Logs, - seq: 1, - ts_unix_ns: 1, - payload: vec![1u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq: 1, + ts_unix_ns: 1, + payload: vec![1u8; 8], + }) + .unwrap(); } { @@ -199,13 +289,14 @@ fn file_spool_reuses_existing_non_full_segment() { segment_size_bytes: 1024 * 1024, })) .unwrap(); - spool.append(WireRecord { - record_type: RecordType::Logs, - seq: 2, - ts_unix_ns: 2, - payload: vec![2u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq: 2, + ts_unix_ns: 2, + payload: vec![2u8; 8], + }) + .unwrap(); } assert!(dir.join("bofh.logjet").exists()); @@ -227,20 +318,22 @@ fn file_spool_preserves_stream_id_and_advances_sequence_seed_after_reopen() { let mut spool = Spool::open(StorageConfig::File(config.clone())).unwrap(); first_stream_id = spool.stream_id(); assert_eq!(spool.next_sequence_seed().unwrap(), 1); - spool.append(WireRecord { - record_type: RecordType::Logs, - seq: 1, - ts_unix_ns: 1, - payload: vec![1u8; 8], - }) - .unwrap(); - spool.append(WireRecord { - record_type: RecordType::Logs, - seq: 2, - ts_unix_ns: 2, - payload: vec![2u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq: 1, + ts_unix_ns: 1, + payload: vec![1u8; 8], + }) + .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq: 2, + ts_unix_ns: 2, + payload: vec![2u8; 8], + }) + .unwrap(); } { @@ -263,13 +356,14 @@ fn summarise_named_segments_reports_sequence_ranges() { .unwrap(); for seq in 1..=3 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8; 8], + }) + .unwrap(); } let summaries = super::summarise_named_segments(&dir, "bofh.logjet").unwrap(); @@ -293,13 +387,14 @@ fn prune_named_segments_by_file_count_keeps_newest_segment() { .unwrap(); for seq in 1..=4 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8; 8], + }) + .unwrap(); } let removed = super::prune_named_segments(&dir, "bofh.logjet", Some(2), None, false).unwrap(); @@ -308,7 +403,14 @@ fn prune_named_segments_by_file_count_keeps_newest_segment() { let names: Vec = super::list_named_segments(&dir, "bofh.logjet") .unwrap() .into_iter() - .map(|segment| segment.path.file_name().unwrap().to_string_lossy().into_owned()) + .map(|segment| { + segment + .path + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() + }) .collect(); assert_eq!(names.len(), 2); assert_eq!(names, vec!["bofh-3.logjet", "bofh-4.logjet"]); @@ -327,18 +429,24 @@ fn prune_named_segments_dry_run_does_not_remove_files() { .unwrap(); for seq in 1..=3 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool + .append(WireRecord { + record_type: RecordType::Logs, + seq, + ts_unix_ns: seq, + payload: vec![seq as u8; 8], + }) + .unwrap(); } let removed = super::prune_named_segments(&dir, "bofh.logjet", Some(1), None, true).unwrap(); assert_eq!(removed.len(), 3); - assert_eq!(super::list_named_segments(&dir, "bofh.logjet").unwrap().len(), 4); + assert_eq!( + super::list_named_segments(&dir, "bofh.logjet") + .unwrap() + .len(), + 4 + ); fs::remove_dir_all(dir).unwrap(); } diff --git a/logjetd/src/tls_utst.rs b/logjetd/src/tls_utst.rs index d15f7f2..d60f005 100644 --- a/logjetd/src/tls_utst.rs +++ b/logjetd/src/tls_utst.rs @@ -1,5 +1,6 @@ use super::{ - authority_host, load_client_config, load_ingest_server_config, load_server_config, parse_server_name, + authority_host, load_client_config, load_ingest_server_config, load_server_config, + parse_server_name, }; use crate::config::{IngestTlsConfig, TlsConfig}; use std::path::PathBuf; diff --git a/logjetd/tests/bridge_flows.rs b/logjetd/tests/bridge_flows.rs index e1ee0ca..a765283 100644 --- a/logjetd/tests/bridge_flows.rs +++ b/logjetd/tests/bridge_flows.rs @@ -7,8 +7,8 @@ use std::thread; use std::time::Duration; use common::{ - ChildGuard, MockCollector, TestDir, free_port, logjetd_command, post_otlp_http, replay_messages, - wait_for_tcp, wait_until, + ChildGuard, MockCollector, TestDir, connect_replay_client, free_port, logjetd_command, + post_otlp_http, read_replay_message, replay_messages, wait_for_tcp, wait_until, }; #[test] @@ -50,7 +50,9 @@ fn bridge_keep_forwards_backlog_in_order() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 3) + })?; assert_eq!( collector.messages(), vec![ @@ -112,7 +114,9 @@ fn bridge_drain_consumes_upstream_records() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 3) + })?; assert_eq!( collector.messages(), vec![ @@ -162,7 +166,11 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; for message in ["RESUME 001", "RESUME 002", "RESUME 003"] { - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-resume", message)?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-resume", + message, + )?; } { @@ -171,11 +179,17 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 3) + })?; } for message in ["RESUME 004", "RESUME 005", "RESUME 006"] { - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-resume", message)?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-resume", + message, + )?; } { @@ -184,7 +198,9 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 6))?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 6) + })?; } assert_eq!( @@ -245,7 +261,9 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 5))?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 5) + })?; assert_eq!( collector.messages(), vec![ @@ -260,10 +278,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { let rotated_count = fs::read_dir(&spool_dir)? .filter_map(Result::ok) .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with("rotation") + entry.file_name().to_string_lossy().starts_with("rotation") && entry.file_name().to_string_lossy().ends_with(".logjet") }) .count(); @@ -316,9 +331,19 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { })?; wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5))?; wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "ALPHA 001")?; - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "ALPHA 002")?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 2))?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-reset", + "ALPHA 001", + )?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-reset", + "ALPHA 002", + )?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 2) + })?; } wait_until(Duration::from_secs(5), || { @@ -333,9 +358,19 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { })?; wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5))?; wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "BRAVO 001")?; - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "BRAVO 002")?; - wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 4))?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-reset", + "BRAVO 001", + )?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-reset", + "BRAVO 002", + )?; + wait_until(Duration::from_secs(5), || { + Ok(collector.messages().len() >= 4) + })?; } assert_eq!( @@ -394,7 +429,9 @@ fn bridge_block_mode_handles_slow_collector_without_losing_order() -> io::Result )?; } - wait_until(Duration::from_secs(10), || Ok(collector.messages().len() >= 6))?; + wait_until(Duration::from_secs(10), || { + Ok(collector.messages().len() >= 6) + })?; assert_eq!( collector.messages(), vec![ @@ -436,7 +473,11 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { for index in 1..=120 { let message = format!("RECOVER {index:03} {}", noisy_message(index)); - post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "replay-corruption", &message)?; + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "replay-corruption", + &message, + )?; } } @@ -463,10 +504,16 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { }; assert!(status.success()); - wait_until(Duration::from_secs(5), || Ok(!collector.messages().is_empty()))?; + wait_until(Duration::from_secs(5), || { + Ok(!collector.messages().is_empty()) + })?; let messages = collector.messages(); assert!(messages.len() < 120); - assert!(messages.iter().any(|message| message.starts_with("RECOVER 090 "))); + assert!( + messages + .iter() + .any(|message| message.starts_with("RECOVER 090 ")) + ); Ok(()) } @@ -513,7 +560,9 @@ fn bridge_forwards_large_payloads_end_to_end() -> io::Result<()> { &large_message, )?; - wait_until(Duration::from_secs(5), || Ok(!collector.messages().is_empty()))?; + wait_until(Duration::from_secs(5), || { + Ok(!collector.messages().is_empty()) + })?; assert_eq!(collector.messages(), vec![large_message]); Ok(()) @@ -569,6 +618,64 @@ fn multiple_replay_clients_receive_backlog_independently() -> io::Result<()> { Ok(()) } +#[test] +fn replay_client_receives_backlog_then_live_records_without_reconnect() -> io::Result<()> { + let dir = TestDir::new("bridge-live-handoff")?; + let ingest_port = free_port()?; + let replay_port = free_port()?; + + let appliance_config = dir.write( + "appliance.conf", + &format!( + "output: file\nfile.path: {}\nfile.size: 64\nfile.name: handoff.logjet\ningest.protocol: otlp-http\ningest.listen: 127.0.0.1:{ingest_port}\nreplay.listen: 127.0.0.1:{replay_port}\n", + dir.path().join("spool").display() + ), + )?; + + let _appliance = ChildGuard::spawn({ + let mut cmd = logjetd_command(); + cmd.arg("--config").arg(&appliance_config).arg("serve"); + cmd + })?; + wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5))?; + wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; + + for message in ["HANDOFF 001", "HANDOFF 002", "HANDOFF 003"] { + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-live", message)?; + } + + let mut replay = connect_replay_client(&format!("127.0.0.1:{replay_port}"), 0, false)?; + assert_eq!( + read_replay_message(&mut replay)?, + Some("HANDOFF 001".to_string()) + ); + assert_eq!( + read_replay_message(&mut replay)?, + Some("HANDOFF 002".to_string()) + ); + assert_eq!( + read_replay_message(&mut replay)?, + Some("HANDOFF 003".to_string()) + ); + + thread::sleep(Duration::from_millis(200)); + assert_eq!(read_replay_message(&mut replay)?, None); + + post_otlp_http( + &format!("127.0.0.1:{ingest_port}"), + "bridge-live", + "HANDOFF 004", + )?; + + replay.set_read_timeout(Some(Duration::from_secs(5)))?; + assert_eq!( + read_replay_message(&mut replay)?, + Some("HANDOFF 004".to_string()) + ); + + Ok(()) +} + fn noisy_message(seed: usize) -> String { let mut value = (seed as u64) .wrapping_mul(0x9E37_79B9_7F4A_7C15) diff --git a/logjetd/tests/common/mod.rs b/logjetd/tests/common/mod.rs index 8630c02..b51b528 100644 --- a/logjetd/tests/common/mod.rs +++ b/logjetd/tests/common/mod.rs @@ -27,10 +27,8 @@ impl TestDir { .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - let path = std::env::temp_dir().join(format!( - "logjetd-it-{label}-{nanos}-{}", - std::process::id() - )); + let path = + std::env::temp_dir().join(format!("logjetd-it-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&path)?; Ok(Self { path }) } @@ -116,7 +114,10 @@ where return Ok(()); } if Instant::now() >= deadline { - return Err(io::Error::new(io::ErrorKind::TimedOut, "timed out waiting for condition")); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "timed out waiting for condition", + )); } thread::sleep(Duration::from_millis(25)); } @@ -200,7 +201,10 @@ fn read_http_request(stream: &mut TcpStream) -> io::Result { break; } if header.len() > 16 * 1024 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "header too large")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "header too large", + )); } } @@ -277,15 +281,11 @@ pub fn post_otlp_http(addr: &str, service_name: &str, message: &str) -> io::Resu } pub fn replay_messages(addr: &str, from_seq: u64, limit: usize) -> io::Result> { - let mut stream = TcpStream::connect(addr)?; - stream.set_read_timeout(Some(Duration::from_secs(2)))?; - read_replay_hello(&mut stream)?; - write_replay_request(&mut stream, from_seq, false)?; - stream.flush()?; + let mut stream = connect_replay_client(addr, from_seq, false)?; let mut messages = Vec::new(); for _ in 0..limit { - match read_wire_record(&mut stream)? { + match read_replay_payload(&mut stream)? { Some(record) => { let batch = ExportLogsServiceRequest::decode(record.as_slice()) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; @@ -297,11 +297,36 @@ pub fn replay_messages(addr: &str, from_seq: u64, limit: usize) -> io::Result io::Result { + let mut stream = TcpStream::connect(addr)?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + read_replay_hello(&mut stream)?; + write_replay_request(&mut stream, from_seq, consume)?; + stream.flush()?; + Ok(stream) +} + +pub fn read_replay_payload(stream: &mut TcpStream) -> io::Result>> { + read_wire_record(stream) +} + +pub fn read_replay_message(stream: &mut TcpStream) -> io::Result> { + let Some(payload) = read_replay_payload(stream)? else { + return Ok(None); + }; + let batch = ExportLogsServiceRequest::decode(payload.as_slice()) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + Ok(extract_messages(&batch).into_iter().next()) +} + fn read_replay_hello(stream: &mut TcpStream) -> io::Result<()> { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != REPLAY_HELLO_MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid replay hello magic")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid replay hello magic", + )); } let mut header = [0u8; 32]; stream.read_exact(&mut header)?; @@ -331,7 +356,10 @@ fn read_wire_record(stream: &mut TcpStream) -> io::Result>> { Err(err) => return Err(err), } if magic != WIRE_MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid wire magic")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid wire magic", + )); } let mut header = [0u8; 24]; stream.read_exact(&mut header)?; @@ -396,7 +424,9 @@ fn extract_messages(batch: &ExportLogsServiceRequest) -> Vec { for record in &scope_logs.log_records { if let Some(body) = &record.body && let Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value), + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( + value, + ), ) = &body.value { messages.push(value.clone()); diff --git a/tests/logjet.rs b/tests/logjet.rs index 6e7efd1..3a4cfc6 100644 --- a/tests/logjet.rs +++ b/tests/logjet.rs @@ -32,7 +32,12 @@ fn read_all(bytes: Vec, config: ReaderConfig) -> ReadAllOutput { let mut reader = LogjetReader::with_config(Cursor::new(bytes), config); let mut out = Vec::new(); while let Some(record) = reader.next_record().unwrap() { - out.push((record.record_type, record.seq, record.ts_unix_ns, record.payload)); + out.push(( + record.record_type, + record.seq, + record.ts_unix_ns, + record.payload, + )); } (out, reader.stats()) } From 07dec96881c1a9fce069758935592194610c8197 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 12:51:13 +0100 Subject: [PATCH 10/17] Force autoreformat all sources to conform 1.94 and policies --- demo/src/bin/otlp-bofh-emitter.rs | 42 +-- demo/src/bin/otlp-bofh-grpc-emitter.rs | 19 +- demo/src/bin/otlp-demo-collector.rs | 78 +--- demo/src/bin/otlp-wire-forwarder.rs | 28 +- demo/src/bin/replay-stall-client.rs | 47 +-- demo/src/bin/wire-hold-emitter.rs | 11 +- demo/src/lib.rs | 158 ++------ examples/read_file.rs | 8 +- examples/recover_file.rs | 12 +- examples/write_file.rs | 21 +- logjetd/src/config.rs | 42 +-- logjetd/src/config_utst.rs | 132 ++----- logjetd/src/daemon.rs | 501 ++++++------------------- logjetd/src/daemon_utst.rs | 36 +- logjetd/src/main.rs | 101 +---- logjetd/src/protocol.rs | 103 ++--- logjetd/src/protocol_utst.rs | 30 +- logjetd/src/replay.rs | 363 ++++-------------- logjetd/src/replay_utst.rs | 150 ++------ logjetd/src/spool.rs | 182 ++------- logjetd/src/spool_utst.rs | 267 +++---------- logjetd/src/tls.rs | 105 +----- logjetd/src/tls_utst.rs | 47 +-- logjetd/tests/bridge_flows.rs | 230 +++--------- logjetd/tests/common/mod.rs | 129 ++----- src/codec.rs | 3 +- src/error.rs | 47 +-- src/format.rs | 21 +- src/lib.rs | 5 +- src/reader.rs | 109 ++---- src/record.rs | 14 +- src/writer.rs | 95 ++--- tests/logjet.rs | 69 +--- 33 files changed, 642 insertions(+), 2563 deletions(-) diff --git a/demo/src/bin/otlp-bofh-emitter.rs b/demo/src/bin/otlp-bofh-emitter.rs index 56d7773..f20bac9 100644 --- a/demo/src/bin/otlp-bofh-emitter.rs +++ b/demo/src/bin/otlp-bofh-emitter.rs @@ -5,10 +5,7 @@ use std::time::Duration; use prost::Message; -use otlp_demo::{ - build_excuse_request, build_excuse_request_for_service_with_severity, - build_message_request_for_service, format_batch_plain, -}; +use otlp_demo::{build_excuse_request, build_excuse_request_for_service_with_severity, build_message_request_for_service, format_batch_plain}; fn main() -> Result<(), Box> { let mut args = env::args().skip(1); @@ -24,17 +21,10 @@ fn main() -> Result<(), Box> { while let Some(arg) = args.next() { match arg.as_str() { "--count" => { - count = Some( - args.next() - .ok_or("missing value for --count")? - .parse::()?, - ); + count = Some(args.next().ok_or("missing value for --count")?.parse::()?); } "--interval-ms" => { - interval_ms = args - .next() - .ok_or("missing value for --interval-ms")? - .parse::()?; + interval_ms = args.next().ok_or("missing value for --interval-ms")?.parse::()?; } "--once" => { count = Some(1); @@ -49,9 +39,7 @@ fn main() -> Result<(), Box> { severity = args.next().ok_or("missing value for --severity")?; } "--ca-file" => { - ca_file = Some(PathBuf::from( - args.next().ok_or("missing value for --ca-file")?, - )); + ca_file = Some(PathBuf::from(args.next().ok_or("missing value for --ca-file")?)); } "--server-name" => { server_name = Some(args.next().ok_or("missing value for --server-name")?); @@ -73,26 +61,12 @@ fn main() -> Result<(), Box> { let mut sequence = 1u64; loop { let request = match &once_message { - Some(message) => build_message_request_for_service( - sequence, - &service_name, - &severity, - message.clone(), - ), - None if service_name == "bofh-emitter" && severity == "warn" => { - build_excuse_request(sequence) - } - None => { - build_excuse_request_for_service_with_severity(sequence, &service_name, &severity) - } + Some(message) => build_message_request_for_service(sequence, &service_name, &severity, message.clone()), + None if service_name == "bofh-emitter" && severity == "warn" => build_excuse_request(sequence), + None => build_excuse_request_for_service_with_severity(sequence, &service_name, &severity), }; print!("{}", format_batch_plain(&request)); - match otlp_demo::post_raw_otlp_http( - &addr, - &request.encode_to_vec(), - ca_file.as_deref(), - server_name.as_deref(), - ) { + match otlp_demo::post_raw_otlp_http(&addr, &request.encode_to_vec(), ca_file.as_deref(), server_name.as_deref()) { Ok(()) => eprintln!("sent OTLP log batch #{sequence} to {display_target}"), Err(err) => eprintln!("send failed for batch #{sequence}: {err}"), } diff --git a/demo/src/bin/otlp-bofh-grpc-emitter.rs b/demo/src/bin/otlp-bofh-grpc-emitter.rs index 3e3b089..5aaff1a 100644 --- a/demo/src/bin/otlp-bofh-grpc-emitter.rs +++ b/demo/src/bin/otlp-bofh-grpc-emitter.rs @@ -1,23 +1,15 @@ use std::env; use std::time::Duration; -use opentelemetry_proto::tonic::collector::logs::v1::{ - ExportLogsServiceRequest, logs_service_client::LogsServiceClient, -}; +use opentelemetry_proto::tonic::collector::logs::v1::{ExportLogsServiceRequest, logs_service_client::LogsServiceClient}; use tonic::Request; use otlp_demo::{build_excuse_request, format_batch_plain}; #[tokio::main] async fn main() -> Result<(), Box> { - let addr = env::args() - .nth(1) - .unwrap_or_else(|| "127.0.0.1:4317".to_string()); - let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { - addr - } else { - format!("http://{addr}") - }; + let addr = env::args().nth(1).unwrap_or_else(|| "127.0.0.1:4317".to_string()); + let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { addr } else { format!("http://{addr}") }; eprintln!("otlp-bofh-grpc-emitter sending OTLP logs to {endpoint}"); @@ -35,10 +27,7 @@ async fn main() -> Result<(), Box> { } } -async fn send_batch( - endpoint: &str, - request: ExportLogsServiceRequest, -) -> Result<(), Box> { +async fn send_batch(endpoint: &str, request: ExportLogsServiceRequest) -> Result<(), Box> { let mut client = LogsServiceClient::connect(endpoint.to_string()).await?; client.export(Request::new(request)).await?; Ok(()) diff --git a/demo/src/bin/otlp-demo-collector.rs b/demo/src/bin/otlp-demo-collector.rs index 49a6178..056b6ee 100644 --- a/demo/src/bin/otlp-demo-collector.rs +++ b/demo/src/bin/otlp-demo-collector.rs @@ -24,20 +24,13 @@ fn main() -> Result<(), Box> { match arg.as_str() { "--tls" => tls = true, "--delay-ms" => { - delay_ms = args - .next() - .ok_or("missing value for --delay-ms")? - .parse::()?; + delay_ms = args.next().ok_or("missing value for --delay-ms")?.parse::()?; } "--cert-file" => { - cert_file = Some(PathBuf::from( - args.next().ok_or("missing value for --cert-file")?, - )); + cert_file = Some(PathBuf::from(args.next().ok_or("missing value for --cert-file")?)); } "--key-file" => { - key_file = Some(PathBuf::from( - args.next().ok_or("missing value for --key-file")?, - )); + key_file = Some(PathBuf::from(args.next().ok_or("missing value for --key-file")?)); } value => bind_addr = value.to_string(), } @@ -70,8 +63,7 @@ fn main() -> Result<(), Box> { request.respond(response)?; } Err(err) => { - let response = Response::from_string(format!("decode error: {err}")) - .with_status_code(StatusCode(400)); + let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); request.respond(response)?; } } @@ -81,10 +73,7 @@ fn main() -> Result<(), Box> { } fn run_tls( - bind_addr: &str, - cert_file: Option, - key_file: Option, - delay_ms: u64, + bind_addr: &str, cert_file: Option, key_file: Option, delay_ms: u64, ) -> Result<(), Box> { let cert_file = cert_file.ok_or("missing --cert-file for --tls")?; let key_file = key_file.ok_or("missing --key-file for --tls")?; @@ -106,9 +95,7 @@ fn run_tls( } fn handle_tls_client( - stream: std::net::TcpStream, - config: Arc, - delay_ms: u64, + stream: std::net::TcpStream, config: Arc, delay_ms: u64, ) -> Result<(), Box> { let conn = ServerConnection::new(config)?; let mut transport = StreamOwned::new(conn, stream); @@ -119,8 +106,7 @@ fn handle_tls_client( } fn handle_tls_http_request( - transport: &mut StreamOwned, - delay_ms: u64, + transport: &mut StreamOwned, delay_ms: u64, ) -> Result<(), Box> { let request = read_http_request(transport)?; if request.method != "POST" || request.path != "/v1/logs" { @@ -161,51 +147,31 @@ fn read_http_request(transport: &mut T) -> io::Result 16 * 1024 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "http header too large", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "http header too large")); } } - let header = std::str::from_utf8(&buffer[..buffer.len() - 4]) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid http header"))?; + let header = std::str::from_utf8(&buffer[..buffer.len() - 4]).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid http header"))?; let mut lines = header.lines(); - let request_line = lines - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing request line"))?; + let request_line = lines.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing request line"))?; let mut parts = request_line.split_whitespace(); - let method = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing method"))? - .to_string(); - let path = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing path"))? - .to_string(); + let method = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing method"))?.to_string(); + let path = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing path"))?.to_string(); let mut content_length = None; for line in lines { if let Some((name, value)) = line.split_once(':') && name.eq_ignore_ascii_case("content-length") { - content_length = Some(value.trim().parse::().map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "invalid content-length") - })?); + content_length = Some(value.trim().parse::().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid content-length"))?); } } - let content_length = content_length - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; + let content_length = content_length.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; let mut body = Vec::new(); - transport - .take(content_length as u64) - .read_to_end(&mut body)?; + transport.take(content_length as u64).read_to_end(&mut body)?; if body.len() != content_length { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "short http body", - )); + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "short http body")); } Ok(ParsedHttpRequest { method, path, body }) @@ -218,17 +184,9 @@ fn write_http_response(transport: &mut T, status: u16, body: &str) -> 404 => "Not Found", _ => "Error", }; - write!( - transport, - "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - status, - status_text, - body.len(), - body - )?; + write!(transport, "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", status, status_text, body.len(), body)?; transport.flush() } fn content_type_header() -> Header { - Header::from_bytes(&b"Content-Type"[..], &b"application/x-protobuf"[..]) - .expect("static content-type header is valid") + Header::from_bytes(&b"Content-Type"[..], &b"application/x-protobuf"[..]).expect("static content-type header is valid") } diff --git a/demo/src/bin/otlp-wire-forwarder.rs b/demo/src/bin/otlp-wire-forwarder.rs index 3937e00..2e61907 100644 --- a/demo/src/bin/otlp-wire-forwarder.rs +++ b/demo/src/bin/otlp-wire-forwarder.rs @@ -23,10 +23,7 @@ fn main() -> Result<(), Box> { if record.record_type == RecordType::Logs { post_raw_otlp_http(&dest, &record.payload, None, None)?; forwarded += 1; - eprintln!( - "forwarded record seq={} to http://{dest}/v1/logs", - record.seq - ); + eprintln!("forwarded record seq={} to http://{dest}/v1/logs", record.seq); } if let Some(limit) = max_records { @@ -52,19 +49,13 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result<()> { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != *b"LJRPH001" { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid replay hello magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay hello magic")); } let mut header = [0u8; 32]; stream.read_exact(&mut header)?; if header[0] != 1 { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported replay hello version: {}", header[0]), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported replay hello version: {}", header[0]))); } Ok(()) @@ -85,26 +76,19 @@ fn read_wire_record(reader: &mut R) -> io::Result> { } if magic != *b"LJNETV01" { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid wire protocol magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid wire protocol magic")); } let mut header = [0u8; 24]; reader.read_exact(&mut header)?; - let record_type = RecordType::from_u8(header[1]) - .map_err(|err| io::Error::new(ErrorKind::InvalidData, err.to_string()))?; + let record_type = RecordType::from_u8(header[1]).map_err(|err| io::Error::new(ErrorKind::InvalidData, err.to_string()))?; let payload_len = u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; let mut payload = vec![0u8; payload_len]; reader.read_exact(&mut payload)?; Ok(Some(WireRecord { record_type, - seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], - header[11], - ]), + seq: u64::from_le_bytes([header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11]]), payload, })) } diff --git a/demo/src/bin/replay-stall-client.rs b/demo/src/bin/replay-stall-client.rs index e89371f..9359232 100644 --- a/demo/src/bin/replay-stall-client.rs +++ b/demo/src/bin/replay-stall-client.rs @@ -14,10 +14,7 @@ fn main() -> Result<(), Box> { let mut stream = TcpStream::connect(&source)?; let hello = read_replay_hello(&mut stream)?; - eprintln!( - "stall client connected to {source}; stream_id={} first_seq={} last_seq={}", - hello.stream_id, hello.first_seq, hello.last_seq - ); + eprintln!("stall client connected to {source}; stream_id={} first_seq={} last_seq={}", hello.stream_id, hello.first_seq, hello.last_seq); write_replay_request(&mut stream, 0, true)?; let Some(record) = read_wire_record(&mut stream)? else { @@ -25,10 +22,7 @@ fn main() -> Result<(), Box> { return Ok(()); }; - eprintln!( - "stall client received seq={} and will now stop acknowledging for {} ms", - record.seq, stall_ms - ); + eprintln!("stall client received seq={} and will now stop acknowledging for {} ms", record.seq, stall_ms); thread::sleep(Duration::from_millis(stall_ms)); let mut extra = [0u8; 1]; @@ -60,34 +54,19 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != *b"LJRPH001" { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid replay hello magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay hello magic")); } let mut header = [0u8; 32]; stream.read_exact(&mut header)?; if header[0] != 1 { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported replay hello version: {}", header[0]), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported replay hello version: {}", header[0]))); } Ok(ReplayHello { - stream_id: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], - header[15], - ]), - first_seq: u64::from_le_bytes([ - header[16], header[17], header[18], header[19], header[20], header[21], header[22], - header[23], - ]), - last_seq: u64::from_le_bytes([ - header[24], header[25], header[26], header[27], header[28], header[29], header[30], - header[31], - ]), + stream_id: u64::from_le_bytes([header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15]]), + first_seq: u64::from_le_bytes([header[16], header[17], header[18], header[19], header[20], header[21], header[22], header[23]]), + last_seq: u64::from_le_bytes([header[24], header[25], header[26], header[27], header[28], header[29], header[30], header[31]]), }) } @@ -104,10 +83,7 @@ fn read_wire_record(reader: &mut R) -> io::Result> { } if magic != *b"LJNETV01" { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid wire protocol magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid wire protocol magic")); } let mut header = [0u8; 24]; @@ -116,10 +92,5 @@ fn read_wire_record(reader: &mut R) -> io::Result> { let mut payload = vec![0u8; payload_len]; reader.read_exact(&mut payload)?; - Ok(Some(WireRecord { - seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], - header[11], - ]), - })) + Ok(Some(WireRecord { seq: u64::from_le_bytes([header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11]]) })) } diff --git a/demo/src/bin/wire-hold-emitter.rs b/demo/src/bin/wire-hold-emitter.rs index a352873..86da2d7 100644 --- a/demo/src/bin/wire-hold-emitter.rs +++ b/demo/src/bin/wire-hold-emitter.rs @@ -18,10 +18,7 @@ fn main() -> Result<(), Box> { while let Some(arg) = args.next() { match arg.as_str() { "--hold-ms" => { - hold_ms = args - .next() - .ok_or("missing value for --hold-ms")? - .parse::()?; + hold_ms = args.next().ok_or("missing value for --hold-ms")?.parse::()?; } "--service-name" => { service_name = args.next().ok_or("missing value for --service-name")?; @@ -56,11 +53,7 @@ fn main() -> Result<(), Box> { } } -fn write_wire_record( - writer: &mut TcpStream, - seq: u64, - payload: &[u8], -) -> Result<(), Box> { +fn write_wire_record(writer: &mut TcpStream, seq: u64, payload: &[u8]) -> Result<(), Box> { let payload_len = u32::try_from(payload.len())?; writer.write_all(&WIRE_MAGIC)?; writer.write_all(&[WIRE_VERSION, RecordType::Logs as u8])?; diff --git a/demo/src/lib.rs b/demo/src/lib.rs index cbb9a07..949d206 100644 --- a/demo/src/lib.rs +++ b/demo/src/lib.rs @@ -34,26 +34,16 @@ pub fn build_excuse_request(sequence: u64) -> ExportLogsServiceRequest { build_excuse_request_for_service_with_severity(sequence, "bofh-emitter", "warn") } -pub fn build_excuse_request_for_service( - sequence: u64, - service_name: &str, -) -> ExportLogsServiceRequest { +pub fn build_excuse_request_for_service(sequence: u64, service_name: &str) -> ExportLogsServiceRequest { build_excuse_request_for_service_with_severity(sequence, service_name, "warn") } -pub fn build_excuse_request_for_service_with_severity( - sequence: u64, - service_name: &str, - severity: &str, -) -> ExportLogsServiceRequest { +pub fn build_excuse_request_for_service_with_severity(sequence: u64, service_name: &str, severity: &str) -> ExportLogsServiceRequest { build_message_request_for_service( sequence, service_name, severity, - format!( - "BOFH excuse #{sequence}: {}", - BOFH_EXCUSES[(sequence as usize) % BOFH_EXCUSES.len()] - ), + format!("BOFH excuse #{sequence}: {}", BOFH_EXCUSES[(sequence as usize) % BOFH_EXCUSES.len()]), ) } @@ -61,12 +51,7 @@ pub fn build_message_request(sequence: u64, body: String) -> ExportLogsServiceRe build_message_request_for_service(sequence, "bofh-emitter", "warn", body) } -pub fn build_message_request_for_service( - sequence: u64, - service_name: &str, - severity: &str, - body: String, -) -> ExportLogsServiceRequest { +pub fn build_message_request_for_service(sequence: u64, service_name: &str, severity: &str, body: String) -> ExportLogsServiceRequest { let nanos = unix_time_nanos(); let (severity_text, severity_number) = parse_demo_severity(severity); @@ -92,13 +77,7 @@ pub fn build_message_request_for_service( observed_time_unix_nano: nanos, severity_number, severity_text, - body: Some(AnyValue { - value: Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - body, - ), - ), - }), + body: Some(AnyValue { value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(body)) }), attributes: vec![ string_attr("demo.kind", "bofh"), string_attr("demo.component", "emitter"), @@ -134,28 +113,18 @@ pub fn post_otlp_http(addr: &str, request: &ExportLogsServiceRequest) -> io::Res post_raw_otlp_http(addr, &request.encode_to_vec(), None, None) } -pub fn post_raw_otlp_http( - addr: &str, - body: &[u8], - ca_file: Option<&Path>, - server_name: Option<&str>, -) -> io::Result<()> { +pub fn post_raw_otlp_http(addr: &str, body: &[u8], ca_file: Option<&Path>, server_name: Option<&str>) -> io::Result<()> { let endpoint = DemoEndpoint::parse(addr); let mut stream = TcpStream::connect(&endpoint.authority)?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; stream.set_read_timeout(Some(Duration::from_secs(5)))?; if endpoint.tls { - let ca_file = ca_file.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "https demo posting requires --ca-file or explicit CA path", - ) - })?; + let ca_file = + ca_file.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "https demo posting requires --ca-file or explicit CA path"))?; let client_config = load_demo_client_config(ca_file)?; let server_name = demo_server_name(&endpoint, server_name)?; - let conn = ClientConnection::new(client_config, server_name) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let conn = ClientConnection::new(client_config, server_name).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); return post_raw_otlp_http_transport(&endpoint, body, &mut transport); } @@ -173,11 +142,7 @@ pub fn load_demo_server_config(cert_path: &Path, key_path: &Path) -> io::Result< Ok(Arc::new(config)) } -pub fn load_demo_mtls_server_config( - ca_path: &Path, - cert_path: &Path, - key_path: &Path, -) -> io::Result> { +pub fn load_demo_mtls_server_config(ca_path: &Path, cert_path: &Path, key_path: &Path) -> io::Result> { let certs = load_certs(cert_path)?; let key = load_private_key(key_path)?; let roots = load_root_store(ca_path)?; @@ -191,11 +156,7 @@ pub fn load_demo_mtls_server_config( Ok(Arc::new(config)) } -fn post_raw_otlp_http_transport( - endpoint: &DemoEndpoint, - body: &[u8], - transport: &mut T, -) -> io::Result<()> { +fn post_raw_otlp_http_transport(endpoint: &DemoEndpoint, body: &[u8], transport: &mut T) -> io::Result<()> { write!( transport, "POST {} HTTP/1.1\r\nHost: {}\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", @@ -209,10 +170,7 @@ fn post_raw_otlp_http_transport( let mut response = String::new(); std::io::Read::read_to_string(transport, &mut response)?; if !response.starts_with("HTTP/1.1 200") && !response.starts_with("HTTP/1.0 200") { - return Err(io::Error::other(format!( - "collector returned non-200 response: {}", - response.lines().next().unwrap_or("unknown response") - ))); + return Err(io::Error::other(format!("collector returned non-200 response: {}", response.lines().next().unwrap_or("unknown response")))); } Ok(()) @@ -220,20 +178,12 @@ fn post_raw_otlp_http_transport( fn load_demo_client_config(ca_path: &Path) -> io::Result> { let roots = load_root_store(ca_path)?; - Ok(Arc::new( - ClientConfig::builder() - .with_root_certificates(roots) - .with_no_client_auth(), - )) + Ok(Arc::new(ClientConfig::builder().with_root_certificates(roots).with_no_client_auth())) } -fn demo_server_name( - endpoint: &DemoEndpoint, - override_name: Option<&str>, -) -> io::Result> { +fn demo_server_name(endpoint: &DemoEndpoint, override_name: Option<&str>) -> io::Result> { let name = override_name.unwrap_or_else(|| endpoint.server_name()); - ServerName::try_from(name.to_string()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) + ServerName::try_from(name.to_string()).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) } fn load_root_store(path: &Path) -> io::Result { @@ -245,9 +195,7 @@ fn load_root_store(path: &Path) -> io::Result { fn load_certs(path: &Path) -> io::Result>> { let mut reader = std::io::BufReader::new(fs::File::open(path)?); - rustls_pemfile::certs(&mut reader) - .collect::, _>>() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) + rustls_pemfile::certs(&mut reader).collect::, _>>().map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) } fn load_private_key(path: &Path) -> io::Result> { @@ -267,34 +215,19 @@ impl DemoEndpoint { fn parse(input: &str) -> Self { if let Some(rest) = input.strip_prefix("https://") { let (authority, path) = split_authority_and_path(rest); - return Self { - authority: authority.to_string(), - path: normalise_path(path), - tls: true, - }; + return Self { authority: authority.to_string(), path: normalise_path(path), tls: true }; } if let Some(rest) = input.strip_prefix("http://") { let (authority, path) = split_authority_and_path(rest); - return Self { - authority: authority.to_string(), - path: normalise_path(path), - tls: false, - }; + return Self { authority: authority.to_string(), path: normalise_path(path), tls: false }; } - Self { - authority: input.to_string(), - path: "/v1/logs".to_string(), - tls: false, - } + Self { authority: input.to_string(), path: "/v1/logs".to_string(), tls: false } } fn server_name(&self) -> &str { - self.authority - .rsplit_once(':') - .map(|(host, _)| host) - .unwrap_or(&self.authority) + self.authority.rsplit_once(':').map(|(host, _)| host).unwrap_or(&self.authority) } } @@ -331,21 +264,12 @@ fn format_batch(batch: &ExportLogsServiceRequest, coloured: bool) -> String { .resource .as_ref() .and_then(|resource| { - resource - .attributes - .iter() - .find(|attr| attr.key == "service.name") - .and_then(|attr| attr.value.as_ref()) - .and_then(any_value_to_string) + resource.attributes.iter().find(|attr| attr.key == "service.name").and_then(|attr| attr.value.as_ref()).and_then(any_value_to_string) }) .unwrap_or("unknown-service"); for scope_logs in &resource_logs.scope_logs { - let scope_name = scope_logs - .scope - .as_ref() - .map(|scope| scope.name.as_str()) - .unwrap_or("unknown-scope"); + let scope_name = scope_logs.scope.as_ref().map(|scope| scope.name.as_str()).unwrap_or("unknown-scope"); for log in &scope_logs.log_records { if coloured { @@ -363,11 +287,7 @@ fn format_batch(batch: &ExportLogsServiceRequest, coloured: bool) -> String { fn write_plain_record(out: &mut String, service_name: &str, scope_name: &str, log: &LogRecord) { let sev = severity_text(log); let body = log_body(log); - let _ = writeln!( - out, - "service={} scope={} severity={} ts={}", - service_name, scope_name, sev, log.time_unix_nano - ); + let _ = writeln!(out, "service={} scope={} severity={} ts={}", service_name, scope_name, sev, log.time_unix_nano); let _ = writeln!(out, "message: {body}"); let _ = writeln!(out); } @@ -392,54 +312,34 @@ fn write_coloured_record(out: &mut String, service_name: &str, scope_name: &str, } fn severity_text(log: &LogRecord) -> &str { - if log.severity_text.is_empty() { - "UNSPECIFIED" - } else { - log.severity_text.as_str() - } + if log.severity_text.is_empty() { "UNSPECIFIED" } else { log.severity_text.as_str() } } fn log_body(log: &LogRecord) -> &str { - log.body - .as_ref() - .and_then(any_value_to_string) - .unwrap_or("") + log.body.as_ref().and_then(any_value_to_string).unwrap_or("") } fn string_attr(key: &str, value: &str) -> KeyValue { KeyValue { key: key.to_string(), - value: Some(AnyValue { - value: Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - value.to_string(), - ), - ), - }), + value: Some(AnyValue { value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value.to_string())) }), } } fn int_attr(key: &str, value: i64) -> KeyValue { KeyValue { key: key.to_string(), - value: Some(AnyValue { - value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::IntValue(value)), - }), + value: Some(AnyValue { value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::IntValue(value)) }), } } fn any_value_to_string(value: &AnyValue) -> Option<&str> { match value.value.as_ref()? { - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => { - Some(value.as_str()) - } + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value.as_str()), _ => None, } } fn unix_time_nanos() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() as u64 + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as u64 } diff --git a/examples/read_file.rs b/examples/read_file.rs index ff11794..51adda8 100644 --- a/examples/read_file.rs +++ b/examples/read_file.rs @@ -8,13 +8,7 @@ fn main() -> Result<(), Box> { let mut reader = LogjetReader::new(BufReader::new(file)); while let Some(record) = reader.next_record()? { - println!( - "type={:?} seq={} ts={} payload_len={}", - record.record_type, - record.seq, - record.ts_unix_ns, - record.payload.len() - ); + println!("type={:?} seq={} ts={} payload_len={}", record.record_type, record.seq, record.ts_unix_ns, record.payload.len()); } println!("stats={:?}", reader.stats()); diff --git a/examples/recover_file.rs b/examples/recover_file.rs index d146954..ec9a652 100644 --- a/examples/recover_file.rs +++ b/examples/recover_file.rs @@ -8,18 +8,10 @@ fn main() -> Result<(), Box> { let mut reader = LogjetReader::new(BufReader::new(file)); while let Some(record) = reader.next_record()? { - println!( - "recovered {:?} seq={} payload_len={}", - record.record_type, - record.seq, - record.payload.len() - ); + println!("recovered {:?} seq={} payload_len={}", record.record_type, record.seq, record.payload.len()); } let stats = reader.stats(); - println!( - "blocks_ok={} blocks_bad={} bytes_skipped={} records_ok={}", - stats.blocks_ok, stats.blocks_bad, stats.bytes_skipped, stats.records_ok - ); + println!("blocks_ok={} blocks_bad={} bytes_skipped={} records_ok={}", stats.blocks_ok, stats.blocks_bad, stats.bytes_skipped, stats.records_ok); Ok(()) } diff --git a/examples/write_file.rs b/examples/write_file.rs index 6f5051c..e6c8dff 100644 --- a/examples/write_file.rs +++ b/examples/write_file.rs @@ -8,24 +8,9 @@ fn main() -> Result<(), Box> { let writer = BufWriter::new(file); let mut log_writer = LogjetWriter::new(writer); - log_writer.push( - RecordType::Logs, - 1, - 1_700_000_000_000_000_000, - b"fake-otlp-logs", - )?; - log_writer.push( - RecordType::Metrics, - 2, - 1_700_000_000_000_000_100, - b"fake-otlp-metrics", - )?; - log_writer.push( - RecordType::Traces, - 3, - 1_700_000_000_000_000_200, - b"fake-otlp-traces", - )?; + log_writer.push(RecordType::Logs, 1, 1_700_000_000_000_000_000, b"fake-otlp-logs")?; + log_writer.push(RecordType::Metrics, 2, 1_700_000_000_000_000_100, b"fake-otlp-metrics")?; + log_writer.push(RecordType::Traces, 3, 1_700_000_000_000_000_200, b"fake-otlp-traces")?; let mut writer = log_writer.into_inner()?; use std::io::Write; diff --git a/logjetd/src/config.rs b/logjetd/src/config.rs index 8cc5ae4..72b0f5b 100644 --- a/logjetd/src/config.rs +++ b/logjetd/src/config.rs @@ -274,9 +274,7 @@ impl Config { }; let output = raw.output.unwrap_or_else(|| "buffer".to_string()); - let ingest_addr = raw - .ingest_addr - .unwrap_or_else(|| "127.0.0.1:7001".to_string()); + let ingest_addr = raw.ingest_addr.unwrap_or_else(|| "127.0.0.1:7001".to_string()); let ingest_protocol = match raw.ingest_protocol.as_deref().unwrap_or("wire") { "wire" => IngestProtocol::Wire, "otlp-http" => IngestProtocol::OtlpHttp, @@ -290,17 +288,11 @@ impl Config { key_file: raw.ingest_key_file, require_client_cert: raw.ingest_require_client_cert.unwrap_or(false), }; - let ingest_limits = IngestLimits { - max_batch_bytes: raw.ingest_max_batch_bytes.unwrap_or(1024 * 1024), - max_clients: raw.ingest_max_clients.unwrap_or(32), - }; + let ingest_limits = + IngestLimits { max_batch_bytes: raw.ingest_max_batch_bytes.unwrap_or(1024 * 1024), max_clients: raw.ingest_max_clients.unwrap_or(32) }; let ingest_overload = IngestOverloadConfig { max_batches_per_second: raw.ingest_max_batches_per_second.unwrap_or(0), - priority_severity_floor: parse_severity_floor( - raw.ingest_priority_severity_floor - .as_deref() - .unwrap_or("error"), - )?, + priority_severity_floor: parse_severity_floor(raw.ingest_priority_severity_floor.as_deref().unwrap_or("error"))?, report_every_ms: raw.ingest_overload_report_ms.unwrap_or(5_000), }; if ingest_limits.max_batch_bytes == 0 { @@ -309,9 +301,7 @@ impl Config { if ingest_limits.max_clients == 0 { return Err("ingest.max-clients must be greater than zero".into()); } - let replay_addr = raw - .replay_addr - .unwrap_or_else(|| "0.0.0.0:7002".to_string()); + let replay_addr = raw.replay_addr.unwrap_or_else(|| "0.0.0.0:7002".to_string()); let replay_max_clients = raw.replay_max_clients.unwrap_or(32); if replay_max_clients == 0 { return Err("replay.max-clients must be greater than zero".into()); @@ -321,9 +311,7 @@ impl Config { return Err("replay.client-timeout-ms must be greater than zero".into()); } let collector = CollectorConfig { - url: raw - .collector_url - .unwrap_or_else(|| "http://127.0.0.1:4318/v1/logs".to_string()), + url: raw.collector_url.unwrap_or_else(|| "http://127.0.0.1:4318/v1/logs".to_string()), timeout_ms: raw.collector_timeout_ms.unwrap_or(10_000), ca_file: raw.collector_ca_file, cert_file: raw.collector_cert_file, @@ -365,18 +353,13 @@ impl Config { let keep_messages = raw.buffer_keep.unwrap_or(0); let storage = match output.as_str() { - "buffer" => StorageConfig::Buffer(BufferConfig { - limit: parse_buffer_limit(raw.buffer_size_kb, raw.buffer_messages)?, - keep_messages, - }), + "buffer" => StorageConfig::Buffer(BufferConfig { limit: parse_buffer_limit(raw.buffer_size_kb, raw.buffer_messages)?, keep_messages }), "file" => { let name = raw.file_name.unwrap_or_else(|| "bar.logjet".to_string()); StorageConfig::File(FileConfig { dir: raw.file_path.unwrap_or_else(|| PathBuf::from(".")), name, - segment_size_bytes: u64::try_from(kib_to_bytes( - raw.file_size_kb.unwrap_or(100), - )?)?, + segment_size_bytes: u64::try_from(kib_to_bytes(raw.file_size_kb.unwrap_or(100))?)?, }) } other => return Err(format!("invalid output mode: {other}").into()), @@ -413,16 +396,11 @@ fn parse_severity_floor(value: &str) -> Result Result> { - let bytes = value - .checked_mul(1024) - .ok_or("size overflow while converting KiB to bytes")?; + let bytes = value.checked_mul(1024).ok_or("size overflow while converting KiB to bytes")?; Ok(usize::try_from(bytes)?) } -fn parse_buffer_limit( - size_kib: Option, - messages: Option, -) -> Result> { +fn parse_buffer_limit(size_kib: Option, messages: Option) -> Result> { match (size_kib, messages) { (Some(_), Some(_)) => Err("buffer.size and buffer.messages conflict; set only one".into()), (Some(size_kib), None) => Ok(BufferLimit::Bytes(kib_to_bytes(size_kib)?)), diff --git a/logjetd/src/config_utst.rs b/logjetd/src/config_utst.rs index fac2735..d9bb7ec 100644 --- a/logjetd/src/config_utst.rs +++ b/logjetd/src/config_utst.rs @@ -1,7 +1,4 @@ -use super::{ - BackpressureMode, BufferLimit, Config, IngestProtocol, SeverityFloor, StorageConfig, - UpstreamMode, -}; +use super::{BackpressureMode, BufferLimit, Config, IngestProtocol, SeverityFloor, StorageConfig, UpstreamMode}; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -21,10 +18,7 @@ fn empty_config_file_uses_defaults() { assert_eq!(config.ingest_limits.max_batch_bytes, 1024 * 1024); assert_eq!(config.ingest_limits.max_clients, 32); assert_eq!(config.ingest_overload.max_batches_per_second, 0); - assert_eq!( - config.ingest_overload.priority_severity_floor, - SeverityFloor::Error - ); + assert_eq!(config.ingest_overload.priority_severity_floor, SeverityFloor::Error); assert_eq!(config.ingest_overload.report_every_ms, 5_000); assert_eq!(config.replay_addr, "0.0.0.0:7002"); assert_eq!(config.replay_max_clients, 32); @@ -60,10 +54,7 @@ fn empty_config_file_uses_defaults() { #[test] fn buffer_size_and_messages_conflict() { - let path = write_temp_config( - "buffer-conflict", - "output: buffer\nbuffer.size: 10\nbuffer.messages: 5\n", - ); + let path = write_temp_config("buffer-conflict", "output: buffer\nbuffer.size: 10\nbuffer.messages: 5\n"); let err = Config::load(&path).unwrap_err().to_string(); assert!(err.contains("buffer.size and buffer.messages conflict")); fs::remove_file(path).unwrap(); @@ -79,73 +70,37 @@ fn file_mode_and_collector_settings_parse() { assert_eq!(config.ingest_protocol, IngestProtocol::OtlpGrpc); assert_eq!(config.collector.timeout_ms, 3210); - assert_eq!( - config.upstream.replay_addr.as_deref(), - Some("10.0.0.15:7002") - ); + assert_eq!(config.upstream.replay_addr.as_deref(), Some("10.0.0.15:7002")); assert_eq!(config.upstream.mode, UpstreamMode::Keep); assert!(config.upstream.state_file.is_none()); assert_eq!(config.upstream.retry_ms, 222); assert_eq!(config.upstream.connect_timeout_ms, 333); assert!(config.ingest_tls.enable); - assert_eq!( - config.ingest_tls.ca_file.as_deref(), - Some(Path::new("./ingest-ca.pem")) - ); - assert_eq!( - config.ingest_tls.cert_file.as_deref(), - Some(Path::new("./ingest.pem")) - ); - assert_eq!( - config.ingest_tls.key_file.as_deref(), - Some(Path::new("./ingest.key")) - ); + assert_eq!(config.ingest_tls.ca_file.as_deref(), Some(Path::new("./ingest-ca.pem"))); + assert_eq!(config.ingest_tls.cert_file.as_deref(), Some(Path::new("./ingest.pem"))); + assert_eq!(config.ingest_tls.key_file.as_deref(), Some(Path::new("./ingest.key"))); assert!(config.ingest_tls.require_client_cert); assert_eq!(config.ingest_limits.max_batch_bytes, 262_144); assert_eq!(config.ingest_limits.max_clients, 7); assert_eq!(config.ingest_overload.max_batches_per_second, 12); - assert_eq!( - config.ingest_overload.priority_severity_floor, - SeverityFloor::Fatal - ); + assert_eq!(config.ingest_overload.priority_severity_floor, SeverityFloor::Fatal); assert_eq!(config.ingest_overload.report_every_ms, 900); assert_eq!(config.replay_max_clients, 9); assert_eq!(config.replay_client_timeout_ms, 444); assert!(config.tls.enable); assert_eq!(config.tls.ca_file.as_deref(), Some(Path::new("./ca.pem"))); - assert_eq!( - config.tls.cert_file.as_deref(), - Some(Path::new("./node.pem")) - ); - assert_eq!( - config.tls.key_file.as_deref(), - Some(Path::new("./node.key")) - ); + assert_eq!(config.tls.cert_file.as_deref(), Some(Path::new("./node.pem"))); + assert_eq!(config.tls.key_file.as_deref(), Some(Path::new("./node.key"))); assert!(config.tls.require_client_cert); - assert_eq!( - config.tls.server_name.as_deref(), - Some("appliance.internal") - ); + assert_eq!(config.tls.server_name.as_deref(), Some("appliance.internal")); assert_eq!(config.collector.url, "https://127.0.0.1:4320/custom"); assert!(config.backpressure.enabled); assert_eq!(config.backpressure.mode, BackpressureMode::Block); assert_eq!(config.backpressure.max_buffered_records, 23); - assert_eq!( - config.collector.ca_file.as_deref(), - Some(Path::new("./collector-ca.pem")) - ); - assert_eq!( - config.collector.cert_file.as_deref(), - Some(Path::new("./collector.pem")) - ); - assert_eq!( - config.collector.key_file.as_deref(), - Some(Path::new("./collector.key")) - ); - assert_eq!( - config.collector.server_name.as_deref(), - Some("collector.internal") - ); + assert_eq!(config.collector.ca_file.as_deref(), Some(Path::new("./collector-ca.pem"))); + assert_eq!(config.collector.cert_file.as_deref(), Some(Path::new("./collector.pem"))); + assert_eq!(config.collector.key_file.as_deref(), Some(Path::new("./collector.key"))); + assert_eq!(config.collector.server_name.as_deref(), Some("collector.internal")); match config.storage { StorageConfig::File(file) => { @@ -169,10 +124,7 @@ fn invalid_ingest_protocol_is_rejected() { #[test] fn invalid_ingest_priority_floor_is_rejected() { - let path = write_temp_config( - "bad-ingest-priority-floor", - "ingest.priority-severity-at-least: nope\n", - ); + let path = write_temp_config("bad-ingest-priority-floor", "ingest.priority-severity-at-least: nope\n"); let err = Config::load(&path).unwrap_err().to_string(); assert!(err.contains("invalid ingest.priority-severity-at-least")); fs::remove_file(path).unwrap(); @@ -185,33 +137,18 @@ fn https_collector_fields_parse_without_file_mode() { "collector.url: https://collector.example:443/v1/logs\ncollector.ca-file: ./ca.pem\ncollector.server-name: collector.example\n", ); let config = Config::load(&path).unwrap(); - assert_eq!( - config.collector.url, - "https://collector.example:443/v1/logs" - ); - assert_eq!( - config.collector.ca_file.as_deref(), - Some(Path::new("./ca.pem")) - ); - assert_eq!( - config.collector.server_name.as_deref(), - Some("collector.example") - ); + assert_eq!(config.collector.url, "https://collector.example:443/v1/logs"); + assert_eq!(config.collector.ca_file.as_deref(), Some(Path::new("./ca.pem"))); + assert_eq!(config.collector.server_name.as_deref(), Some("collector.example")); fs::remove_file(path).unwrap(); } #[test] fn upstream_mode_drain_parses() { - let path = write_temp_config( - "upstream-drain", - "upstream.mode: drain\nupstream.replay: 127.0.0.1:7002\nupstream.state-file: ./bridge.state\n", - ); + let path = write_temp_config("upstream-drain", "upstream.mode: drain\nupstream.replay: 127.0.0.1:7002\nupstream.state-file: ./bridge.state\n"); let config = Config::load(&path).unwrap(); assert_eq!(config.upstream.mode, UpstreamMode::Drain); - assert_eq!( - config.upstream.state_file.as_deref(), - Some(Path::new("./bridge.state")) - ); + assert_eq!(config.upstream.state_file.as_deref(), Some(Path::new("./bridge.state"))); fs::remove_file(path).unwrap(); } @@ -233,10 +170,7 @@ fn invalid_backpressure_mode_is_rejected() { #[test] fn backpressure_mode_block_parses() { - let path = write_temp_config( - "backpressure-block", - "backpressure.enabled: true\nbackpressure.mode: block\n", - ); + let path = write_temp_config("backpressure-block", "backpressure.enabled: true\nbackpressure.mode: block\n"); let config = Config::load(&path).unwrap(); assert!(config.backpressure.enabled); assert_eq!(config.backpressure.mode, BackpressureMode::Block); @@ -273,19 +207,13 @@ fn invalid_ingest_limit_values_are_rejected() { assert!(replay_err.contains("replay.max-clients")); fs::remove_file(replay_path).unwrap(); - let replay_timeout_path = - write_temp_config("bad-replay-timeout", "replay.client-timeout-ms: 0\n"); + let replay_timeout_path = write_temp_config("bad-replay-timeout", "replay.client-timeout-ms: 0\n"); let replay_timeout_err = Config::load(&replay_timeout_path).unwrap_err().to_string(); assert!(replay_timeout_err.contains("replay.client-timeout-ms")); fs::remove_file(replay_timeout_path).unwrap(); - let backpressure_buffer_path = write_temp_config( - "bad-backpressure-buffer", - "backpressure.max-buffered-records: 0\n", - ); - let backpressure_buffer_err = Config::load(&backpressure_buffer_path) - .unwrap_err() - .to_string(); + let backpressure_buffer_path = write_temp_config("bad-backpressure-buffer", "backpressure.max-buffered-records: 0\n"); + let backpressure_buffer_err = Config::load(&backpressure_buffer_path).unwrap_err().to_string(); assert!(backpressure_buffer_err.contains("backpressure.max-buffered-records")); fs::remove_file(backpressure_buffer_path).unwrap(); } @@ -297,12 +225,6 @@ fn write_temp_config(label: &str, body: &str) -> PathBuf { } fn unique_temp_path(label: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - std::env::temp_dir().join(format!( - "logjetd-{label}-{nanos}-{}.yaml", - std::process::id() - )) + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.yaml", std::process::id())) } diff --git a/logjetd/src/daemon.rs b/logjetd/src/daemon.rs index 438ef5a..39fcf4b 100644 --- a/logjetd/src/daemon.rs +++ b/logjetd/src/daemon.rs @@ -18,14 +18,9 @@ use tiny_http::{Method, Response, Server, StatusCode}; use tonic::transport::{Certificate, Identity, ServerTlsConfig}; use tonic::{Request, Response as GrpcResponse, Status}; -use crate::config::{ - Config, IngestLimits, IngestOverloadConfig, IngestProtocol, IngestTlsConfig, SeverityFloor, -}; +use crate::config::{Config, IngestLimits, IngestOverloadConfig, IngestProtocol, IngestTlsConfig, SeverityFloor}; use crate::protocol::WireRecord; -use crate::protocol::{ - ReplayHello, read_record_with_limit, read_replay_ack, read_replay_request, write_record, - write_replay_hello, -}; +use crate::protocol::{ReplayHello, read_record_with_limit, read_replay_ack, read_replay_request, write_record, write_replay_hello}; use crate::spool::Spool; use crate::tls::{load_ingest_server_config, load_server_config}; @@ -83,41 +78,26 @@ enum IngestDecision { impl SharedSpool { fn new(spool: Spool) -> Self { - Self { - spool: Mutex::new(spool), - wake_state: Mutex::new(0), - wake_cv: Condvar::new(), - } + Self { spool: Mutex::new(spool), wake_state: Mutex::new(0), wake_cv: Condvar::new() } } fn notify_change(&self) -> io::Result<()> { - let mut generation = self - .wake_state - .lock() - .map_err(|_| io::Error::other("wake-state mutex poisoned"))?; + let mut generation = self.wake_state.lock().map_err(|_| io::Error::other("wake-state mutex poisoned"))?; *generation = generation.saturating_add(1); self.wake_cv.notify_all(); Ok(()) } fn wait_for_change(&self, seen_generation: &mut u64) -> io::Result<()> { - let generation = self - .wake_state - .lock() - .map_err(|_| io::Error::other("wake-state mutex poisoned"))?; - let generation = self - .wake_cv - .wait_while(generation, |current| *current == *seen_generation) - .map_err(|_| io::Error::other("wake-state mutex poisoned"))?; + let generation = self.wake_state.lock().map_err(|_| io::Error::other("wake-state mutex poisoned"))?; + let generation = + self.wake_cv.wait_while(generation, |current| *current == *seen_generation).map_err(|_| io::Error::other("wake-state mutex poisoned"))?; *seen_generation = *generation; Ok(()) } fn current_generation(&self) -> io::Result { - let generation = self - .wake_state - .lock() - .map_err(|_| io::Error::other("wake-state mutex poisoned"))?; + let generation = self.wake_state.lock().map_err(|_| io::Error::other("wake-state mutex poisoned"))?; Ok(*generation) } } @@ -137,19 +117,14 @@ impl SharedIngestPolicy { } fn decide(&self, priority: BatchPriority) -> io::Result { - let mut state = self - .inner - .lock() - .map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; + let mut state = self.inner.lock().map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; let now = Instant::now(); if now.duration_since(state.window_start) >= Duration::from_secs(1) { state.window_start = now; state.accepted_in_window = 0; } - let decision = if self.config.max_batches_per_second == 0 - || state.accepted_in_window < self.config.max_batches_per_second - { + let decision = if self.config.max_batches_per_second == 0 || state.accepted_in_window < self.config.max_batches_per_second { state.accepted_in_window = state.accepted_in_window.saturating_add(1); state.stats.accepted = state.stats.accepted.saturating_add(1); IngestDecision::Accept @@ -161,10 +136,7 @@ impl SharedIngestPolicy { IngestDecision::RejectRateLimited }; - if matches!( - decision, - IngestDecision::AcceptPriorityBypass | IngestDecision::RejectRateLimited - ) { + if matches!(decision, IngestDecision::AcceptPriorityBypass | IngestDecision::RejectRateLimited) { maybe_report_overload(&self.config, &mut state); } @@ -172,20 +144,14 @@ impl SharedIngestPolicy { } fn note_oversize(&self) -> io::Result<()> { - let mut state = self - .inner - .lock() - .map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; + let mut state = self.inner.lock().map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; state.stats.oversize_rejected = state.stats.oversize_rejected.saturating_add(1); maybe_report_overload(&self.config, &mut state); Ok(()) } fn note_client_cap(&self) -> io::Result<()> { - let mut state = self - .inner - .lock() - .map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; + let mut state = self.inner.lock().map_err(|_| io::Error::other("ingest policy mutex poisoned"))?; state.stats.client_cap_rejected = state.stats.client_cap_rejected.saturating_add(1); maybe_report_overload(&self.config, &mut state); Ok(()) @@ -202,11 +168,7 @@ fn maybe_report_overload(config: &IngestOverloadConfig, state: &mut IngestPolicy } eprintln!( "logjetd ingest overload stats accepted={} priority-bypass={} rate-limited={} oversize-rejected={} client-cap-rejected={}", - state.stats.accepted, - state.stats.priority_bypass, - state.stats.rate_limited, - state.stats.oversize_rejected, - state.stats.client_cap_rejected + state.stats.accepted, state.stats.priority_bypass, state.stats.rate_limited, state.stats.oversize_rejected, state.stats.client_cap_rejected ); state.next_report_at = now + Duration::from_millis(config.report_every_ms.max(1)); } @@ -237,15 +199,7 @@ pub fn serve(config: DaemonConfig) -> io::Result<()> { let replay_thread = thread::Builder::new() .name("logjetd-replay".to_string()) - .spawn(move || { - replay_loop( - replay_addr, - replay_spool, - replay_max_clients, - replay_client_timeout_ms, - tls, - ) - })?; + .spawn(move || replay_loop(replay_addr, replay_spool, replay_max_clients, replay_client_timeout_ms, tls))?; eprintln!("logjetd using config {}", config.config_path.display()); ingest_loop( @@ -257,19 +211,12 @@ pub fn serve(config: DaemonConfig) -> io::Result<()> { spool, next_seq, )?; - replay_thread - .join() - .map_err(|_| io::Error::other("replay listener thread panicked"))? + replay_thread.join().map_err(|_| io::Error::other("replay listener thread panicked"))? } fn ingest_loop( - bind_addr: String, - protocol: IngestProtocol, - ingest_tls: IngestTlsConfig, - ingest_limits: IngestLimits, - ingest_policy: Arc, - spool: Arc, - next_seq: Arc, + bind_addr: String, protocol: IngestProtocol, ingest_tls: IngestTlsConfig, ingest_limits: IngestLimits, ingest_policy: Arc, + spool: Arc, next_seq: Arc, ) -> io::Result<()> { let limiter = Arc::new(ConnectionLimiter::new(ingest_limits.max_clients)); match protocol { @@ -286,105 +233,64 @@ fn ingest_loop( let ingest_policy = Arc::clone(&ingest_policy); let limiter = Arc::clone(&limiter); let max_batch_bytes = ingest_limits.max_batch_bytes; - thread::Builder::new() - .name("logjetd-ingest-client".to_string()) - .spawn(move || { - if let Err(err) = handle_ingest_client( - stream, - spool, - ingest_policy, - limiter, - max_batch_bytes, - ) { - eprintln!("logjetd ingest client error: {err}"); - } - })?; + thread::Builder::new().name("logjetd-ingest-client".to_string()).spawn(move || { + if let Err(err) = handle_ingest_client(stream, spool, ingest_policy, limiter, max_batch_bytes) { + eprintln!("logjetd ingest client error: {err}"); + } + })?; } } IngestProtocol::OtlpHttp => { if ingest_tls.enable { - return otlp_http_tls_loop( - bind_addr, - ingest_tls, - ingest_limits, - ingest_policy, - spool, - next_seq, - limiter, - ); + return otlp_http_tls_loop(bind_addr, ingest_tls, ingest_limits, ingest_policy, spool, next_seq, limiter); } - let server = - Server::http(&bind_addr).map_err(|err| io::Error::other(err.to_string()))?; - eprintln!( - "logjetd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", - ingest_limits.max_batch_bytes - ); + let server = Server::http(&bind_addr).map_err(|err| io::Error::other(err.to_string()))?; + eprintln!("logjetd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes); for mut request in server.incoming_requests() { if request.method() != &Method::Post || request.url() != "/v1/logs" { - let response = - Response::from_string("not found").with_status_code(StatusCode(404)); + let response = Response::from_string("not found").with_status_code(StatusCode(404)); let _ = request.respond(response); continue; } let mut body = Vec::with_capacity(ingest_limits.max_batch_bytes.min(8192)); - request - .as_reader() - .take((ingest_limits.max_batch_bytes + 1) as u64) - .read_to_end(&mut body)?; + request.as_reader().take((ingest_limits.max_batch_bytes + 1) as u64).read_to_end(&mut body)?; if body.len() > ingest_limits.max_batch_bytes { ingest_policy.note_oversize()?; - let response = Response::from_string("payload too large") - .with_status_code(StatusCode(413)); - request - .respond(response) - .map_err(|err| io::Error::other(err.to_string()))?; + let response = Response::from_string("payload too large").with_status_code(StatusCode(413)); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; continue; } match ExportLogsServiceRequest::decode(body.as_slice()) { Ok(batch) => { - let decision = - ingest_policy.decide(classify_otlp_batch_priority(&batch))?; + let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; if matches!(decision, IngestDecision::RejectRateLimited) { - let response = Response::from_string("rate limit exceeded") - .with_status_code(StatusCode(429)); - request - .respond(response) - .map_err(|err| io::Error::other(err.to_string()))?; + let response = Response::from_string("rate limit exceeded").with_status_code(StatusCode(429)); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; continue; } let record = WireRecord { record_type: logjet::RecordType::Logs, seq: next_seq.fetch_add(1, Ordering::Relaxed), - ts_unix_ns: extract_batch_timestamp(&batch) - .unwrap_or_else(unix_time_nanos), + ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), payload: body, }; append_batch_record(&spool, record)?; let response = Response::empty(200); - request - .respond(response) - .map_err(|err| io::Error::other(err.to_string()))?; + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; } Err(err) => { - let response = Response::from_string(format!("decode error: {err}")) - .with_status_code(StatusCode(400)); - request - .respond(response) - .map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); + request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; } } } } IngestProtocol::OtlpGrpc => { - let addr: SocketAddr = bind_addr.parse().map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("invalid gRPC bind addr: {err}"), - ) - })?; + let addr: SocketAddr = + bind_addr.parse().map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, format!("invalid gRPC bind addr: {err}")))?; eprintln!( "logjetd ingest listening on {}://{bind_addr} using otlp-grpc max-batch-bytes={} max-clients={}", if ingest_tls.enable { "grpcs" } else { "grpc" }, @@ -392,37 +298,18 @@ fn ingest_loop( ingest_limits.max_clients ); - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .map_err(|err| io::Error::other(err.to_string()))?; - let service = OtlpGrpcLogsService { - spool, - next_seq, - ingest_policy, - }; - let grpc_tls = if ingest_tls.enable { - Some(build_grpc_server_tls_config(&ingest_tls)?) - } else { - None - }; + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().map_err(|err| io::Error::other(err.to_string()))?; + let service = OtlpGrpcLogsService { spool, next_seq, ingest_policy }; + let grpc_tls = if ingest_tls.enable { Some(build_grpc_server_tls_config(&ingest_tls)?) } else { None }; runtime.block_on(async move { let builder = tonic::transport::Server::builder(); - let builder = if let Some(tls) = grpc_tls { - builder - .tls_config(tls) - .map_err(|err| io::Error::other(err.to_string()))? - } else { - builder - }; + let builder = + if let Some(tls) = grpc_tls { builder.tls_config(tls).map_err(|err| io::Error::other(err.to_string()))? } else { builder }; builder .concurrency_limit_per_connection(ingest_limits.max_clients) - .add_service( - LogsServiceServer::new(service) - .max_decoding_message_size(ingest_limits.max_batch_bytes), - ) + .add_service(LogsServiceServer::new(service).max_decoding_message_size(ingest_limits.max_batch_bytes)) .serve(addr) .await .map_err(|err| io::Error::other(err.to_string())) @@ -434,13 +321,8 @@ fn ingest_loop( } fn otlp_http_tls_loop( - bind_addr: String, - ingest_tls: IngestTlsConfig, - ingest_limits: IngestLimits, - ingest_policy: Arc, - spool: Arc, - next_seq: Arc, - limiter: Arc, + bind_addr: String, ingest_tls: IngestTlsConfig, ingest_limits: IngestLimits, ingest_policy: Arc, spool: Arc, + next_seq: Arc, limiter: Arc, ) -> io::Result<()> { let listener = TcpListener::bind(&bind_addr)?; let tls_server = load_ingest_server_config(&ingest_tls)?; @@ -457,68 +339,39 @@ fn otlp_http_tls_loop( let tls_server = tls_server.clone(); let limiter = Arc::clone(&limiter); let max_batch_bytes = ingest_limits.max_batch_bytes; - thread::Builder::new() - .name("logjetd-otlp-http-tls-client".to_string()) - .spawn(move || { - if let Err(err) = handle_otlp_http_tls_client( - stream, - tls_server, - spool, - ingest_policy, - next_seq, - limiter, - max_batch_bytes, - ) { - eprintln!("logjetd otlp-http tls client error: {err}"); - } - })?; + thread::Builder::new().name("logjetd-otlp-http-tls-client".to_string()).spawn(move || { + if let Err(err) = handle_otlp_http_tls_client(stream, tls_server, spool, ingest_policy, next_seq, limiter, max_batch_bytes) { + eprintln!("logjetd otlp-http tls client error: {err}"); + } + })?; } Ok(()) } fn handle_otlp_http_tls_client( - stream: TcpStream, - tls_server: Arc, - spool: Arc, - ingest_policy: Arc, - next_seq: Arc, - limiter: Arc, - max_batch_bytes: usize, + stream: TcpStream, tls_server: Arc, spool: Arc, ingest_policy: Arc, next_seq: Arc, + limiter: Arc, max_batch_bytes: usize, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { eprintln!("logjetd ingest refused TLS client: ingest.max-clients reached"); ingest_policy.note_client_cap()?; return Ok(()); }; - let conn = ServerConnection::new(tls_server) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let conn = ServerConnection::new(tls_server).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); - let result = handle_otlp_http_transport( - &mut transport, - spool, - ingest_policy, - next_seq, - max_batch_bytes, - ); + let result = handle_otlp_http_transport(&mut transport, spool, ingest_policy, next_seq, max_batch_bytes); transport.conn.send_close_notify(); let _ = transport.flush(); result } fn handle_otlp_http_transport( - transport: &mut T, - spool: Arc, - ingest_policy: Arc, - next_seq: Arc, - max_batch_bytes: usize, + transport: &mut T, spool: Arc, ingest_policy: Arc, next_seq: Arc, max_batch_bytes: usize, ) -> io::Result<()> { let request = match read_http_request(transport, max_batch_bytes) { Ok(request) => request, - Err(err) - if err.kind() == io::ErrorKind::InvalidData - && err.to_string() == "payload too large" => - { + Err(err) if err.kind() == io::ErrorKind::InvalidData && err.to_string() == "payload too large" => { ingest_policy.note_oversize()?; write_http_response(transport, 413, "payload too large")?; return Ok(()); @@ -556,10 +409,7 @@ fn handle_otlp_http_transport( fn append_batch_record(spool: &Arc, record: WireRecord) -> io::Result<()> { { - let mut inner = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; + let mut inner = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; inner.append(record)?; } spool.notify_change() @@ -571,10 +421,7 @@ struct ParsedHttpRequest { body: Vec, } -fn read_http_request( - transport: &mut T, - max_batch_bytes: usize, -) -> io::Result { +fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io::Result { const MAX_HEADER_BYTES: usize = 16 * 1024; let mut buffer = Vec::new(); let mut byte = [0u8; 1]; @@ -583,10 +430,7 @@ fn read_http_request( transport.read_exact(&mut byte)?; buffer.push(byte[0]); if buffer.len() > MAX_HEADER_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "http header too large", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "http header too large")); } if buffer.ends_with(b"\r\n\r\n") { break; @@ -594,51 +438,31 @@ fn read_http_request( } let header_end = buffer.len(); - let header_text = std::str::from_utf8(&buffer[..header_end - 4]).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "http header is not valid utf-8") - })?; + let header_text = + std::str::from_utf8(&buffer[..header_end - 4]).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "http header is not valid utf-8"))?; let mut lines = header_text.lines(); - let request_line = lines - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http request line"))?; + let request_line = lines.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http request line"))?; let mut parts = request_line.split_whitespace(); - let method = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http method"))? - .to_string(); - let path = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http path"))? - .to_string(); + let method = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http method"))?.to_string(); + let path = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http path"))?.to_string(); let mut content_length = None; for line in lines { if let Some((name, value)) = line.split_once(':') && name.eq_ignore_ascii_case("content-length") { - content_length = Some(value.trim().parse::().map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "invalid content-length") - })?); + content_length = Some(value.trim().parse::().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid content-length"))?); } } - let content_length = content_length - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; + let content_length = content_length.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; if content_length > max_batch_bytes { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "payload too large", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "payload too large")); } let mut body = Vec::with_capacity(content_length); - transport - .take(content_length as u64) - .read_to_end(&mut body)?; + transport.take(content_length as u64).read_to_end(&mut body)?; if body.len() != content_length { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "short http body", - )); + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "short http body")); } Ok(ParsedHttpRequest { method, path, body }) @@ -654,40 +478,27 @@ fn write_http_response(transport: &mut T, status: u16, body: &str) 503 => "Service Unavailable", _ => "Error", }; - write!( - transport, - "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - status, - status_text, - body.len(), - body - )?; + write!(transport, "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", status, status_text, body.len(), body)?; transport.flush() } fn build_grpc_server_tls_config(ingest_tls: &IngestTlsConfig) -> io::Result { - let cert_file = ingest_tls.cert_file.as_deref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "ingest.cert-file is required when ingest.tls-enable is true", - ) - })?; - let key_file = ingest_tls.key_file.as_deref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "ingest.key-file is required when ingest.tls-enable is true", - ) - })?; + let cert_file = ingest_tls + .cert_file + .as_deref() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ingest.cert-file is required when ingest.tls-enable is true"))?; + let key_file = ingest_tls + .key_file + .as_deref() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ingest.key-file is required when ingest.tls-enable is true"))?; let identity = Identity::from_pem(fs::read(cert_file)?, fs::read(key_file)?); let mut tls = ServerTlsConfig::new().identity(identity); if ingest_tls.require_client_cert { - let ca_file = ingest_tls.ca_file.as_deref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "ingest.ca-file is required when ingest.require-client-cert is true", - ) - })?; + let ca_file = ingest_tls + .ca_file + .as_deref() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ingest.ca-file is required when ingest.require-client-cert is true"))?; tls = tls.client_ca_root(Certificate::from_pem(fs::read(ca_file)?)); } Ok(tls) @@ -698,11 +509,7 @@ fn build_grpc_server_tls_config(ingest_tls: &IngestTlsConfig) -> io::Result, - max_clients: usize, - client_timeout_ms: u64, - tls: crate::config::TlsConfig, + bind_addr: String, spool: Arc, max_clients: usize, client_timeout_ms: u64, tls: crate::config::TlsConfig, ) -> io::Result<()> { let listener = TcpListener::bind(&bind_addr)?; let limiter = Arc::new(ConnectionLimiter::new(max_clients)); @@ -712,9 +519,7 @@ fn replay_loop( } else { None }; - eprintln!( - "logjetd replay listening on {bind_addr} max-clients={max_clients} client-timeout-ms={client_timeout_ms}" - ); + eprintln!("logjetd replay listening on {bind_addr} max-clients={max_clients} client-timeout-ms={client_timeout_ms}"); for stream in listener.incoming() { let stream = stream?; @@ -724,24 +529,18 @@ fn replay_loop( let spool = Arc::clone(&spool); let tls_server = tls_server.clone(); let limiter = Arc::clone(&limiter); - thread::Builder::new() - .name("logjetd-replay-client".to_string()) - .spawn(move || { - if let Err(err) = handle_replay_client(stream, spool, tls_server, limiter) { - eprintln!("logjetd replay client error: {err}"); - } - })?; + thread::Builder::new().name("logjetd-replay-client".to_string()).spawn(move || { + if let Err(err) = handle_replay_client(stream, spool, tls_server, limiter) { + eprintln!("logjetd replay client error: {err}"); + } + })?; } Ok(()) } fn handle_ingest_client( - stream: TcpStream, - spool: Arc, - ingest_policy: Arc, - limiter: Arc, - max_batch_bytes: usize, + stream: TcpStream, spool: Arc, ingest_policy: Arc, limiter: Arc, max_batch_bytes: usize, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { eprintln!("logjetd ingest refused wire client: ingest.max-clients reached"); @@ -752,14 +551,8 @@ fn handle_ingest_client( let mut reader = BufReader::new(stream); while let Some(record) = read_record_with_limit(&mut reader, max_batch_bytes)? { - if matches!( - ingest_policy.decide(BatchPriority::Unknown)?, - IngestDecision::RejectRateLimited - ) { - eprintln!( - "logjetd ingest dropped wire record seq={} because ingest rate limit was exceeded", - record.seq - ); + if matches!(ingest_policy.decide(BatchPriority::Unknown)?, IngestDecision::RejectRateLimited) { + eprintln!("logjetd ingest dropped wire record seq={} because ingest rate limit was exceeded", record.seq); continue; } append_batch_record(&spool, record)?; @@ -778,10 +571,7 @@ struct ConnectionLimiter { impl ConnectionLimiter { fn new(max_clients: usize) -> Self { - Self { - max_clients, - active_clients: std::sync::atomic::AtomicUsize::new(0), - } + Self { max_clients, active_clients: std::sync::atomic::AtomicUsize::new(0) } } fn try_acquire(self: &Arc) -> Option { @@ -790,14 +580,8 @@ impl ConnectionLimiter { if current >= self.max_clients { return None; } - if self - .active_clients - .compare_exchange(current, current + 1, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - return Some(ConnectionPermit { - limiter: Arc::clone(self), - }); + if self.active_clients.compare_exchange(current, current + 1, Ordering::AcqRel, Ordering::Relaxed).is_ok() { + return Some(ConnectionPermit { limiter: Arc::clone(self) }); } } } @@ -814,18 +598,14 @@ impl Drop for ConnectionPermit { } fn handle_replay_client( - stream: TcpStream, - spool: Arc, - tls_server: Option>, - limiter: Arc, + stream: TcpStream, spool: Arc, tls_server: Option>, limiter: Arc, ) -> io::Result<()> { let Some(_permit) = limiter.try_acquire() else { eprintln!("logjetd replay refused client: replay.max-clients reached"); return Ok(()); }; if let Some(server_config) = tls_server { - let conn = ServerConnection::new(server_config) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let conn = ServerConnection::new(server_config).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); return handle_replay_transport(&mut transport, spool); } @@ -834,42 +614,22 @@ fn handle_replay_client( handle_replay_transport(&mut transport, spool) } -fn handle_replay_transport( - transport: &mut T, - spool: Arc, -) -> io::Result<()> { +fn handle_replay_transport(transport: &mut T, spool: Arc) -> io::Result<()> { let hello = { - let spool = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; + let spool = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; let (first_seq, last_seq) = spool.sequence_bounds()?.unwrap_or((0, 0)); - ReplayHello { - stream_id: spool.stream_id(), - first_seq, - last_seq, - } + ReplayHello { stream_id: spool.stream_id(), first_seq, last_seq } }; write_replay_hello(transport, &hello)?; transport.flush()?; let request = read_replay_request(transport)?; let (mut cursor, mut seen_generation) = { - let spool_guard = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; - ( - spool_guard.replay_cursor_after(request.from_seq)?, - spool.current_generation()?, - ) + let spool_guard = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; + (spool_guard.replay_cursor_after(request.from_seq)?, spool.current_generation()?) }; - eprintln!( - "logjetd replay client requested records after seq={} mode={}", - request.from_seq, - if request.consume { "drain" } else { "keep" } - ); + eprintln!("logjetd replay client requested records after seq={} mode={}", request.from_seq, if request.consume { "drain" } else { "keep" }); if request.consume { return handle_replay_transport_drain(transport, spool, &mut cursor, &mut seen_generation); @@ -879,10 +639,7 @@ fn handle_replay_transport( let mut sent_any = false; loop { let next_record = { - let spool = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; + let spool = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; spool.next_for_cursor(&mut cursor)? }; @@ -904,17 +661,11 @@ fn handle_replay_transport( } fn handle_replay_transport_drain( - transport: &mut T, - spool: Arc, - cursor: &mut crate::spool::ReplayCursor, - seen_generation: &mut u64, + transport: &mut T, spool: Arc, cursor: &mut crate::spool::ReplayCursor, seen_generation: &mut u64, ) -> io::Result<()> { loop { let next_record = { - let spool = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; + let spool = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; spool.next_for_cursor(cursor)? }; @@ -928,20 +679,11 @@ fn handle_replay_transport_drain( let ack = read_replay_ack(transport)?; if ack.ack_seq != record.seq { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "replay ack out of order: expected {} got {}", - record.seq, ack.ack_seq - ), - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, format!("replay ack out of order: expected {} got {}", record.seq, ack.ack_seq))); } { - let mut spool = spool - .spool - .lock() - .map_err(|_| io::Error::other("spool mutex poisoned"))?; + let mut spool = spool.spool.lock().map_err(|_| io::Error::other("spool mutex poisoned"))?; spool.consume_through(ack.ack_seq)?; } } @@ -988,10 +730,7 @@ fn priority_from_severity_number(severity_number: i32) -> BatchPriority { } fn unix_time_nanos() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() as u64 + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as u64 } #[derive(Clone)] @@ -1003,16 +742,9 @@ struct OtlpGrpcLogsService { #[tonic::async_trait] impl LogsService for OtlpGrpcLogsService { - async fn export( - &self, - request: Request, - ) -> Result, Status> { + async fn export(&self, request: Request) -> Result, Status> { let batch = request.into_inner(); - match self - .ingest_policy - .decide(classify_otlp_batch_priority(&batch)) - .map_err(|err| Status::internal(err.to_string()))? - { + match self.ingest_policy.decide(classify_otlp_batch_priority(&batch)).map_err(|err| Status::internal(err.to_string()))? { IngestDecision::Accept | IngestDecision::AcceptPriorityBypass => {} IngestDecision::RejectRateLimited => { return Err(Status::resource_exhausted("ingest rate limit exceeded")); @@ -1026,11 +758,8 @@ impl LogsService for OtlpGrpcLogsService { payload, }; - append_batch_record(&self.spool, record) - .map_err(|err| Status::internal(err.to_string()))?; + append_batch_record(&self.spool, record).map_err(|err| Status::internal(err.to_string()))?; - Ok(GrpcResponse::new(ExportLogsServiceResponse { - partial_success: None, - })) + Ok(GrpcResponse::new(ExportLogsServiceResponse { partial_success: None })) } } diff --git a/logjetd/src/daemon_utst.rs b/logjetd/src/daemon_utst.rs index 0330d46..ec61b4f 100644 --- a/logjetd/src/daemon_utst.rs +++ b/logjetd/src/daemon_utst.rs @@ -1,6 +1,5 @@ use super::{ - BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, - classify_otlp_batch_priority, read_http_request, write_http_response, + BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, classify_otlp_batch_priority, read_http_request, write_http_response, }; use crate::config::{IngestOverloadConfig, SeverityFloor}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; @@ -117,14 +116,8 @@ fn ingest_policy_rate_limits_low_priority_batches() { priority_severity_floor: SeverityFloor::Error, report_every_ms: 0, }); - assert_eq!( - policy.decide(BatchPriority::Warn).unwrap(), - IngestDecision::Accept - ); - assert_eq!( - policy.decide(BatchPriority::Warn).unwrap(), - IngestDecision::RejectRateLimited - ); + assert_eq!(policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::Accept); + assert_eq!(policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::RejectRateLimited); } #[test] @@ -134,24 +127,15 @@ fn ingest_policy_allows_priority_bypass_above_threshold() { priority_severity_floor: SeverityFloor::Error, report_every_ms: 0, }); - assert_eq!( - policy.decide(BatchPriority::Warn).unwrap(), - IngestDecision::Accept - ); - assert_eq!( - policy.decide(BatchPriority::Error).unwrap(), - IngestDecision::AcceptPriorityBypass - ); + assert_eq!(policy.decide(BatchPriority::Warn).unwrap(), IngestDecision::Accept); + assert_eq!(policy.decide(BatchPriority::Error).unwrap(), IngestDecision::AcceptPriorityBypass); } #[test] fn classify_otlp_batch_priority_uses_highest_log_severity() { let batch = ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: Vec::new(), - dropped_attributes_count: 0, - }), + resource: Some(Resource { attributes: Vec::new(), dropped_attributes_count: 0 }), scope_logs: vec![ScopeLogs { scope: Some(InstrumentationScope { name: "test".to_string(), @@ -164,9 +148,7 @@ fn classify_otlp_batch_priority_uses_highest_log_severity() { severity_number: 13, severity_text: "WARN".to_string(), body: Some(AnyValue { - value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - "warn".to_string(), - )), + value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue("warn".to_string())), }), ..Default::default() }, @@ -174,9 +156,7 @@ fn classify_otlp_batch_priority_uses_highest_log_severity() { severity_number: 17, severity_text: "ERROR".to_string(), body: Some(AnyValue { - value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - "error".to_string(), - )), + value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue("error".to_string())), }), ..Default::default() }, diff --git a/logjetd/src/main.rs b/logjetd/src/main.rs index 0b3a5f8..5b09320 100644 --- a/logjetd/src/main.rs +++ b/logjetd/src/main.rs @@ -64,17 +64,9 @@ fn run() -> Result<(), Box> { command = Some("replay"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => { - replay_path = Some(PathBuf::from( - args.next().ok_or("missing value for --path")?, - )) - } - "--name" => { - replay_name = Some(args.next().ok_or("missing value for --name")?) - } - "--dest" => { - replay_dest = Some(args.next().ok_or("missing value for --dest")?) - } + "--path" => replay_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), + "--name" => replay_name = Some(args.next().ok_or("missing value for --name")?), + "--dest" => replay_dest = Some(args.next().ok_or("missing value for --dest")?), _ => return Err(format!("unknown replay argument: {flag}").into()), } } @@ -84,14 +76,8 @@ fn run() -> Result<(), Box> { command = Some("segments"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => { - segment_path = Some(PathBuf::from( - args.next().ok_or("missing value for --path")?, - )) - } - "--name" => { - segment_name = Some(args.next().ok_or("missing value for --name")?) - } + "--path" => segment_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), + "--name" => segment_name = Some(args.next().ok_or("missing value for --name")?), _ => return Err(format!("unknown segments argument: {flag}").into()), } } @@ -101,27 +87,13 @@ fn run() -> Result<(), Box> { command = Some("prune"); while let Some(flag) = args.next() { match flag.as_str() { - "--path" => { - prune_path = Some(PathBuf::from( - args.next().ok_or("missing value for --path")?, - )) - } - "--name" => { - prune_name = Some(args.next().ok_or("missing value for --name")?) - } + "--path" => prune_path = Some(PathBuf::from(args.next().ok_or("missing value for --path")?)), + "--name" => prune_name = Some(args.next().ok_or("missing value for --name")?), "--keep-files" => { - prune_keep_files = Some( - args.next() - .ok_or("missing value for --keep-files")? - .parse::()?, - ); + prune_keep_files = Some(args.next().ok_or("missing value for --keep-files")?.parse::()?); } "--keep-bytes" => { - prune_keep_bytes = Some( - args.next() - .ok_or("missing value for --keep-bytes")? - .parse::()?, - ); + prune_keep_bytes = Some(args.next().ok_or("missing value for --keep-bytes")?.parse::()?); } "--dry-run" => prune_dry_run = true, _ => return Err(format!("unknown prune argument: {flag}").into()), @@ -133,9 +105,7 @@ fn run() -> Result<(), Box> { command = Some("bridge"); while let Some(flag) = args.next() { match flag.as_str() { - "--source" => { - bridge_source = Some(args.next().ok_or("missing value for --source")?) - } + "--source" => bridge_source = Some(args.next().ok_or("missing value for --source")?), _ => return Err(format!("unknown bridge argument: {flag}").into()), } } @@ -148,10 +118,7 @@ fn run() -> Result<(), Box> { match command { Some("serve") | None => { let config = Config::load(&config_path)?; - serve(DaemonConfig { - config, - config_path, - })?; + serve(DaemonConfig { config, config_path })?; } Some("inspect") => { let path = PathBuf::from(command_arg.ok_or("missing path")?); @@ -172,20 +139,10 @@ fn run() -> Result<(), Box> { } let files = validate_replay_path(&path, &name)?; if files.is_empty() { - return Err(format!( - "no replay files found for name '{}' in {}", - name, - path.display() - ) - .into()); + return Err(format!("no replay files found for name '{}' in {}", name, path.display()).into()); } - eprintln!( - "replaying {} file(s) from {} to {}", - files.len(), - path.display(), - collector.url - ); + eprintln!("replaying {} file(s) from {} to {}", files.len(), path.display(), collector.url); let sent = replay_path_to_otlp_http(&path, &name, &collector)?; eprintln!("replayed {sent} OTLP log batch record(s)"); } @@ -197,24 +154,12 @@ fn run() -> Result<(), Box> { }; eprintln!("bridging from {} to {}", source, config.collector.url); - bridge_wire_to_otlp_http( - &source, - &config.collector, - &config.backpressure, - &config.upstream, - &config.tls, - )?; + bridge_wire_to_otlp_http(&source, &config.collector, &config.backpressure, &config.upstream, &config.tls)?; } Some("prune") => { let path = prune_path.ok_or("missing --path")?; let name = prune_name.ok_or("missing --name")?; - let removed = prune_named_segments( - &path, - &name, - prune_keep_files, - prune_keep_bytes, - prune_dry_run, - )?; + let removed = prune_named_segments(&path, &name, prune_keep_files, prune_keep_bytes, prune_dry_run)?; for entry in &removed { if prune_dry_run { println!("would_remove={}", entry.display()); @@ -237,25 +182,17 @@ fn print_usage() { println!(" logjetd [serve] [-c|--config ]"); println!(" logjetd inspect "); println!(" logjetd segments --path --name "); - println!( - " logjetd replay [-c|--config ] --path --name [--dest ]" - ); - println!( - " logjetd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]" - ); + println!(" logjetd replay [-c|--config ] --path --name [--dest ]"); + println!(" logjetd prune --path --name [--keep-files | --keep-bytes ] [--dry-run]"); println!(" logjetd bridge [-c|--config ] [--source ]"); println!(); println!("Commands:"); println!(" serve Run ingest and replay listeners using YAML configuration"); println!(" inspect Print stored record metadata from a .logjet file or spool directory"); println!(" segments Print ordered file-segment metadata for one rotated .logjet spool"); - println!( - " replay Read ordered .logjet files and blast OTLP log batches to an OTLP/HTTP collector" - ); + println!(" replay Read ordered .logjet files and blast OTLP log batches to an OTLP/HTTP collector"); println!(" prune Remove oldest rotated file segments by count or total bytes"); - println!( - " bridge Connect to a replay listener, drain backlog, stay attached, and forward OTLP logs to the configured collector" - ); + println!(" bridge Connect to a replay listener, drain backlog, stay attached, and forward OTLP logs to the configured collector"); println!(); println!("Config path defaults to /etc/logjet.conf."); } diff --git a/logjetd/src/protocol.rs b/logjetd/src/protocol.rs index 5286e6d..f6d430d 100644 --- a/logjetd/src/protocol.rs +++ b/logjetd/src/protocol.rs @@ -41,10 +41,7 @@ pub fn read_record(reader: &mut R) -> io::Result> { read_record_with_limit(reader, usize::MAX) } -pub fn read_record_with_limit( - reader: &mut R, - max_payload_len: usize, -) -> io::Result> { +pub fn read_record_with_limit(reader: &mut R, max_payload_len: usize) -> io::Result> { let mut magic = [0u8; 8]; match reader.read_exact(&mut magic) { Ok(()) => {} @@ -53,10 +50,7 @@ pub fn read_record_with_limit( } if magic != WIRE_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid wire protocol magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid wire protocol magic")); } let mut header = [0u8; 24]; @@ -64,24 +58,14 @@ pub fn read_record_with_limit( let version = header[0]; if version != WIRE_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported wire protocol version: {version}"), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported wire protocol version: {version}"))); } - let record_type = RecordType::from_u8(header[1]).map_err(|err| { - io::Error::new( - ErrorKind::InvalidData, - format!("invalid wire record type: {err}"), - ) - })?; + let record_type = + RecordType::from_u8(header[1]).map_err(|err| io::Error::new(ErrorKind::InvalidData, format!("invalid wire record type: {err}")))?; let payload_len = u32::from_le_bytes([header[20], header[21], header[22], header[23]]) as usize; if payload_len > max_payload_len { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("wire payload too large: {payload_len} > {max_payload_len}"), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("wire payload too large: {payload_len} > {max_payload_len}"))); } let mut payload = vec![0u8; payload_len]; @@ -89,25 +73,15 @@ pub fn read_record_with_limit( Ok(Some(WireRecord { record_type, - seq: u64::from_le_bytes([ - header[4], header[5], header[6], header[7], header[8], header[9], header[10], - header[11], - ]), - ts_unix_ns: u64::from_le_bytes([ - header[12], header[13], header[14], header[15], header[16], header[17], header[18], - header[19], - ]), + seq: u64::from_le_bytes([header[4], header[5], header[6], header[7], header[8], header[9], header[10], header[11]]), + ts_unix_ns: u64::from_le_bytes([header[12], header[13], header[14], header[15], header[16], header[17], header[18], header[19]]), payload, })) } pub fn write_record(writer: &mut W, record: &WireRecord) -> io::Result<()> { - let payload_len = u32::try_from(record.payload.len()).map_err(|_| { - io::Error::new( - ErrorKind::InvalidInput, - "payload too large for wire protocol", - ) - })?; + let payload_len = + u32::try_from(record.payload.len()).map_err(|_| io::Error::new(ErrorKind::InvalidInput, "payload too large for wire protocol"))?; writer.write_all(&WIRE_MAGIC)?; writer.write_all(&[WIRE_VERSION, record.record_type as u8])?; @@ -123,28 +97,19 @@ pub fn read_replay_request(reader: &mut R) -> io::Result let mut magic = [0u8; 8]; reader.read_exact(&mut magic)?; if magic != REPLAY_REQUEST_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid replay request magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay request magic")); } let mut header = [0u8; 16]; reader.read_exact(&mut header)?; let version = header[0]; if version != REPLAY_REQUEST_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported replay request version: {version}"), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported replay request version: {version}"))); } Ok(ReplayRequest { consume: header[1] & 0x01 != 0, - from_seq: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], - header[15], - ]), + from_seq: u64::from_le_bytes([header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15]]), }) } @@ -161,35 +126,20 @@ pub fn read_replay_hello(reader: &mut R) -> io::Result { let mut magic = [0u8; 8]; reader.read_exact(&mut magic)?; if magic != REPLAY_HELLO_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid replay hello magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay hello magic")); } let mut header = [0u8; 32]; reader.read_exact(&mut header)?; let version = header[0]; if version != REPLAY_HELLO_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported replay hello version: {version}"), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported replay hello version: {version}"))); } Ok(ReplayHello { - stream_id: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], - header[15], - ]), - first_seq: u64::from_le_bytes([ - header[16], header[17], header[18], header[19], header[20], header[21], header[22], - header[23], - ]), - last_seq: u64::from_le_bytes([ - header[24], header[25], header[26], header[27], header[28], header[29], header[30], - header[31], - ]), + stream_id: u64::from_le_bytes([header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15]]), + first_seq: u64::from_le_bytes([header[16], header[17], header[18], header[19], header[20], header[21], header[22], header[23]]), + last_seq: u64::from_le_bytes([header[24], header[25], header[26], header[27], header[28], header[29], header[30], header[31]]), }) } @@ -207,28 +157,17 @@ pub fn read_replay_ack(reader: &mut R) -> io::Result { let mut magic = [0u8; 8]; reader.read_exact(&mut magic)?; if magic != REPLAY_ACK_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid replay ack magic", - )); + return Err(io::Error::new(ErrorKind::InvalidData, "invalid replay ack magic")); } let mut header = [0u8; 16]; reader.read_exact(&mut header)?; let version = header[0]; if version != REPLAY_ACK_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported replay ack version: {version}"), - )); + return Err(io::Error::new(ErrorKind::InvalidData, format!("unsupported replay ack version: {version}"))); } - Ok(ReplayAck { - ack_seq: u64::from_le_bytes([ - header[8], header[9], header[10], header[11], header[12], header[13], header[14], - header[15], - ]), - }) + Ok(ReplayAck { ack_seq: u64::from_le_bytes([header[8], header[9], header[10], header[11], header[12], header[13], header[14], header[15]]) }) } pub fn write_replay_ack(writer: &mut W, ack: &ReplayAck) -> io::Result<()> { diff --git a/logjetd/src/protocol_utst.rs b/logjetd/src/protocol_utst.rs index 8ee44c5..b1f3656 100644 --- a/logjetd/src/protocol_utst.rs +++ b/logjetd/src/protocol_utst.rs @@ -1,18 +1,12 @@ use super::{ - ReplayAck, ReplayHello, ReplayRequest, WireRecord, read_record, read_record_with_limit, - read_replay_ack, read_replay_hello, read_replay_request, write_record, write_replay_ack, - write_replay_hello, write_replay_request, + ReplayAck, ReplayHello, ReplayRequest, WireRecord, read_record, read_record_with_limit, read_replay_ack, read_replay_hello, read_replay_request, + write_record, write_replay_ack, write_replay_hello, write_replay_request, }; use logjet::RecordType; #[test] fn round_trip_record() { - let record = WireRecord { - record_type: RecordType::Logs, - seq: 42, - ts_unix_ns: 77, - payload: b"abc".to_vec(), - }; + let record = WireRecord { record_type: RecordType::Logs, seq: 42, ts_unix_ns: 77, payload: b"abc".to_vec() }; let mut bytes = Vec::new(); write_record(&mut bytes, &record).unwrap(); let decoded = read_record(&mut bytes.as_slice()).unwrap().unwrap(); @@ -21,10 +15,7 @@ fn round_trip_record() { #[test] fn replay_request_round_trip() { - let request = ReplayRequest { - from_seq: 1234, - consume: true, - }; + let request = ReplayRequest { from_seq: 1234, consume: true }; let mut bytes = Vec::new(); write_replay_request(&mut bytes, &request).unwrap(); let decoded = read_replay_request(&mut bytes.as_slice()).unwrap(); @@ -42,11 +33,7 @@ fn replay_ack_round_trip() { #[test] fn replay_hello_round_trip() { - let hello = ReplayHello { - stream_id: 77, - first_seq: 10, - last_seq: 99, - }; + let hello = ReplayHello { stream_id: 77, first_seq: 10, last_seq: 99 }; let mut bytes = Vec::new(); write_replay_hello(&mut bytes, &hello).unwrap(); let decoded = read_replay_hello(&mut bytes.as_slice()).unwrap(); @@ -79,12 +66,7 @@ fn unsupported_record_version_is_rejected() { #[test] fn record_payload_over_limit_is_rejected() { - let record = WireRecord { - record_type: RecordType::Logs, - seq: 42, - ts_unix_ns: 77, - payload: b"abcdef".to_vec(), - }; + let record = WireRecord { record_type: RecordType::Logs, seq: 42, ts_unix_ns: 77, payload: b"abcdef".to_vec() }; let mut bytes = Vec::new(); write_record(&mut bytes, &record).unwrap(); let err = read_record_with_limit(&mut bytes.as_slice(), 5).unwrap_err(); diff --git a/logjetd/src/replay.rs b/logjetd/src/replay.rs index 2e92eb1..1289b4a 100644 --- a/logjetd/src/replay.rs +++ b/logjetd/src/replay.rs @@ -9,24 +9,12 @@ use std::time::Duration; use logjet::{LogjetReader, ReaderConfig, RecordType}; use rustls::{ClientConfig, ClientConnection, StreamOwned}; -use crate::config::{ - BackpressureConfig, BackpressureMode, CollectorConfig, TlsConfig, UpstreamConfig, UpstreamMode, -}; -use crate::protocol::{ - ReplayAck, ReplayHello, ReplayRequest, read_record, read_replay_hello, write_replay_ack, - write_replay_request, -}; +use crate::config::{BackpressureConfig, BackpressureMode, CollectorConfig, TlsConfig, UpstreamConfig, UpstreamMode}; +use crate::protocol::{ReplayAck, ReplayHello, ReplayRequest, read_record, read_replay_hello, write_replay_ack, write_replay_request}; use crate::spool::list_named_segments; -use crate::tls::{ - load_client_config, load_collector_client_config, parse_collector_server_name, - parse_server_name, -}; - -pub fn replay_path_to_otlp_http( - path: &Path, - name: &str, - collector: &CollectorConfig, -) -> io::Result { +use crate::tls::{load_client_config, load_collector_client_config, parse_collector_server_name, parse_server_name}; + +pub fn replay_path_to_otlp_http(path: &Path, name: &str, collector: &CollectorConfig) -> io::Result { let mut sent = 0u64; let endpoint = CollectorEndpoint::parse(&collector.url)?; let transport = CollectorTransport { @@ -34,11 +22,7 @@ pub fn replay_path_to_otlp_http( backpressure_enabled: false, backpressure_mode: BackpressureMode::Disconnect, max_buffered_records: 1, - tls_client: if endpoint.tls { - Some(load_collector_client_config(collector)?) - } else { - None - }, + tls_client: if endpoint.tls { Some(load_collector_client_config(collector)?) } else { None }, endpoint, collector: collector.clone(), upstream_mode: UpstreamMode::Keep, @@ -67,30 +51,18 @@ pub fn validate_replay_path(path: &Path, name: &str) -> io::Result> } pub fn bridge_wire_to_otlp_http( - source: &str, - collector: &CollectorConfig, - backpressure: &BackpressureConfig, - upstream: &UpstreamConfig, - tls: &TlsConfig, + source: &str, collector: &CollectorConfig, backpressure: &BackpressureConfig, upstream: &UpstreamConfig, tls: &TlsConfig, ) -> io::Result<()> { let endpoint = CollectorEndpoint::parse(&collector.url)?; let connect_timeout = Duration::from_millis(upstream.connect_timeout_ms); let retry_delay = Duration::from_millis(upstream.retry_ms); - let tls_client = if tls.enable { - Some(load_client_config(tls)?) - } else { - None - }; + let tls_client = if tls.enable { Some(load_client_config(tls)?) } else { None }; let collector_transport = CollectorTransport { timeout: Duration::from_millis(collector.timeout_ms), backpressure_enabled: backpressure.enabled, backpressure_mode: backpressure.mode, max_buffered_records: backpressure.max_buffered_records, - tls_client: if endpoint.tls { - Some(load_collector_client_config(collector)?) - } else { - None - }, + tls_client: if endpoint.tls { Some(load_collector_client_config(collector)?) } else { None }, endpoint, collector: collector.clone(), upstream_mode: upstream.mode, @@ -101,34 +73,17 @@ pub fn bridge_wire_to_otlp_http( "bridge resume state file {} loaded seq={} stream-id={}", path.display(), state.last_seq, - state - .stream_id - .map(|value| value.to_string()) - .unwrap_or_else(|| "unset".to_string()) + state.stream_id.map(|value| value.to_string()).unwrap_or_else(|| "unset".to_string()) ); } loop { - match bridge_once( - source, - connect_timeout, - &mut state, - upstream.state_file.as_deref(), - tls, - tls_client.clone(), - &collector_transport, - ) { + match bridge_once(source, connect_timeout, &mut state, upstream.state_file.as_deref(), tls, tls_client.clone(), &collector_transport) { Ok(()) => { - eprintln!( - "bridge source {source} closed after seq={}; reconnecting in {} ms", - state.last_seq, upstream.retry_ms - ); + eprintln!("bridge source {source} closed after seq={}; reconnecting in {} ms", state.last_seq, upstream.retry_ms); } Err(err) => { - eprintln!( - "bridge source {source} error after seq={}: {err}; reconnecting in {} ms", - state.last_seq, upstream.retry_ms - ); + eprintln!("bridge source {source} error after seq={}: {err}; reconnecting in {} ms", state.last_seq, upstream.retry_ms); } } thread::sleep(retry_delay); @@ -136,13 +91,8 @@ pub fn bridge_wire_to_otlp_http( } fn bridge_once( - source: &str, - connect_timeout: Duration, - state: &mut BridgeState, - state_file: Option<&Path>, - tls: &TlsConfig, - tls_client: Option>, - collector_transport: &CollectorTransport, + source: &str, connect_timeout: Duration, state: &mut BridgeState, state_file: Option<&Path>, tls: &TlsConfig, + tls_client: Option>, collector_transport: &CollectorTransport, ) -> io::Result<()> { let stream = connect_with_timeout(source, connect_timeout)?; stream.set_read_timeout(None)?; @@ -150,47 +100,24 @@ fn bridge_once( if let Some(client_config) = tls_client { let server_name = parse_server_name(tls, source)?; - let conn = ClientConnection::new(client_config, server_name) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let conn = ClientConnection::new(client_config, server_name).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut transport = StreamOwned::new(conn, stream); - return bridge_transport( - source, - state, - state_file, - &mut transport, - collector_transport, - ); + return bridge_transport(source, state, state_file, &mut transport, collector_transport); } let mut transport = stream; - bridge_transport( - source, - state, - state_file, - &mut transport, - collector_transport, - ) + bridge_transport(source, state, state_file, &mut transport, collector_transport) } fn bridge_transport( - source: &str, - state: &mut BridgeState, - state_file: Option<&Path>, - transport: &mut T, - collector_transport: &CollectorTransport, + source: &str, state: &mut BridgeState, state_file: Option<&Path>, transport: &mut T, collector_transport: &CollectorTransport, ) -> io::Result<()> { let hello = read_replay_hello(transport)?; reconcile_bridge_state(source, state, &hello)?; write_bridge_state(state_file, state)?; let consume = collector_transport.upstream_mode == UpstreamMode::Drain; - write_replay_request( - transport, - &ReplayRequest { - from_seq: state.last_seq, - consume, - }, - )?; + write_replay_request(transport, &ReplayRequest { from_seq: state.last_seq, consume })?; transport.flush()?; eprintln!( "bridge connected to {} and requested records after seq={} mode={} backpressure={}", @@ -217,15 +144,7 @@ fn bridge_transport( let mut pending = std::collections::VecDeque::new(); while let Some(record) = read_record(transport)? { - flush_ready_results( - transport, - state, - state_file, - consume, - &mut pending, - &result_rx, - false, - )?; + flush_ready_results(transport, state, state_file, consume, &mut pending, &result_rx, false)?; if record.record_type != RecordType::Logs { commit_record(transport, state, state_file, consume, record.seq)?; @@ -233,14 +152,7 @@ fn bridge_transport( } let seq = record.seq; - match enqueue_export_task( - &task_tx, - collector_transport, - ExportTask { - seq, - payload: record.payload, - }, - ) { + match enqueue_export_task(&task_tx, collector_transport, ExportTask { seq, payload: record.payload }) { Ok(EnqueueOutcome::Queued) => pending.push_back(PendingExport::Queued(seq)), Ok(EnqueueOutcome::DroppedNewest) => pending.push_back(PendingExport::Dropped(seq)), Err(err) => { @@ -252,15 +164,7 @@ fn bridge_transport( } drop(task_tx); - flush_ready_results( - transport, - state, - state_file, - consume, - &mut pending, - &result_rx, - true, - )?; + flush_ready_results(transport, state, state_file, consume, &mut pending, &result_rx, true)?; match exporter.join() { Ok(Ok(())) => Ok(()), Ok(Err(err)) => Err(err), @@ -269,63 +173,36 @@ fn bridge_transport( } fn enqueue_export_task( - task_tx: &mpsc::SyncSender, - collector_transport: &CollectorTransport, - task: ExportTask, + task_tx: &mpsc::SyncSender, collector_transport: &CollectorTransport, task: ExportTask, ) -> io::Result { match collector_transport.backpressure_mode { - BackpressureMode::Block => { - task_tx - .send(task) - .map(|()| EnqueueOutcome::Queued) - .map_err(|_| { - io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped") - }) - } + BackpressureMode::Block => task_tx + .send(task) + .map(|()| EnqueueOutcome::Queued) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")), BackpressureMode::Disconnect => match task_tx.try_send(task) { Ok(()) => Ok(EnqueueOutcome::Queued), - Err(mpsc::TrySendError::Full(_)) => Err(io::Error::new( - io::ErrorKind::TimedOut, - "collector export buffer is full; disconnecting bridge", - )), - Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "collector export worker stopped", - )), + Err(mpsc::TrySendError::Full(_)) => Err(io::Error::new(io::ErrorKind::TimedOut, "collector export buffer is full; disconnecting bridge")), + Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")), }, BackpressureMode::DropNewest => match task_tx.try_send(task) { Ok(()) => Ok(EnqueueOutcome::Queued), Err(mpsc::TrySendError::Full(task)) => { - eprintln!( - "bridge dropping seq={} because collector export buffer is full (mode=drop-newest)", - task.seq - ); + eprintln!("bridge dropping seq={} because collector export buffer is full (mode=drop-newest)", task.seq); Ok(EnqueueOutcome::DroppedNewest) } - Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "collector export worker stopped", - )), + Err(mpsc::TrySendError::Disconnected(_)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")), }, } } fn export_worker( - collector_transport: CollectorTransport, - task_rx: mpsc::Receiver, - result_tx: mpsc::Sender, + collector_transport: CollectorTransport, task_rx: mpsc::Receiver, result_tx: mpsc::Sender, ) -> io::Result<()> { while let Ok(task) = task_rx.recv() { - let outcome = post_raw_otlp_http(&collector_transport, &task.payload) - .map(|()| ExportOutcome::Delivered); + let outcome = post_raw_otlp_http(&collector_transport, &task.payload).map(|()| ExportOutcome::Delivered); let failed = outcome.is_err(); - if result_tx - .send(ExportResult { - seq: task.seq, - outcome, - }) - .is_err() - { + if result_tx.send(ExportResult { seq: task.seq, outcome }).is_err() { break; } if failed { @@ -336,13 +213,8 @@ fn export_worker( } fn flush_ready_results( - transport: &mut T, - state: &mut BridgeState, - state_file: Option<&Path>, - consume: bool, - pending: &mut std::collections::VecDeque, - result_rx: &mpsc::Receiver, - block: bool, + transport: &mut T, state: &mut BridgeState, state_file: Option<&Path>, consume: bool, pending: &mut std::collections::VecDeque, + result_rx: &mpsc::Receiver, block: bool, ) -> io::Result<()> { loop { let Some(front) = pending.front() else { @@ -350,34 +222,23 @@ fn flush_ready_results( }; let result = match front { - PendingExport::Dropped(seq) => ExportResult { - seq: *seq, - outcome: Ok(ExportOutcome::DroppedNewest), - }, + PendingExport::Dropped(seq) => ExportResult { seq: *seq, outcome: Ok(ExportOutcome::DroppedNewest) }, PendingExport::Queued(expected_seq) => { let result = if block { - result_rx.recv().map_err(|_| { - io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped") - })? + result_rx.recv().map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped"))? } else { match result_rx.try_recv() { Ok(result) => result, Err(mpsc::TryRecvError::Empty) => return Ok(()), Err(mpsc::TryRecvError::Disconnected) => { - return Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "collector export worker stopped", - )); + return Err(io::Error::new(io::ErrorKind::BrokenPipe, "collector export worker stopped")); } } }; if result.seq != *expected_seq { return Err(io::Error::new( io::ErrorKind::InvalidData, - format!( - "collector export worker returned seq={} out of order; expected {}", - result.seq, expected_seq - ), + format!("collector export worker returned seq={} out of order; expected {}", result.seq, expected_seq), )); } result @@ -386,9 +247,7 @@ fn flush_ready_results( pending.pop_front(); match result.outcome { - Ok(ExportOutcome::Delivered) => { - commit_record(transport, state, state_file, consume, result.seq)? - } + Ok(ExportOutcome::Delivered) => commit_record(transport, state, state_file, consume, result.seq)?, Ok(ExportOutcome::DroppedNewest) => { commit_record(transport, state, state_file, consume, result.seq)?; } @@ -398,11 +257,7 @@ fn flush_ready_results( } fn commit_record( - transport: &mut T, - state: &mut BridgeState, - state_file: Option<&Path>, - consume: bool, - seq: u64, + transport: &mut T, state: &mut BridgeState, state_file: Option<&Path>, consume: bool, seq: u64, ) -> io::Result<()> { state.last_seq = seq; write_bridge_state(state_file, state)?; @@ -414,10 +269,7 @@ fn commit_record( } fn post_raw_otlp_http(collector_transport: &CollectorTransport, payload: &[u8]) -> io::Result<()> { - let stream = connect_with_timeout( - &collector_transport.endpoint.authority, - collector_transport.timeout, - )?; + let stream = connect_with_timeout(&collector_transport.endpoint.authority, collector_transport.timeout)?; if !collector_transport.backpressure_enabled { stream.set_write_timeout(Some(collector_transport.timeout))?; stream.set_read_timeout(Some(collector_transport.timeout))?; @@ -435,18 +287,11 @@ fn post_raw_otlp_http(collector_transport: &CollectorTransport, payload: &[u8]) } if let Some(client_config) = &collector_transport.tls_client { - let server_name = parse_collector_server_name( - &collector_transport.collector, - &collector_transport.endpoint.authority, - )?; - let conn = ClientConnection::new(client_config.clone(), server_name) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let server_name = parse_collector_server_name(&collector_transport.collector, &collector_transport.endpoint.authority)?; + let conn = + ClientConnection::new(client_config.clone(), server_name).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; let mut tls_transport = StreamOwned::new(conn, stream); - return post_raw_otlp_http_transport( - &collector_transport.endpoint, - payload, - &mut tls_transport, - ); + return post_raw_otlp_http_transport(&collector_transport.endpoint, payload, &mut tls_transport); } let mut plain_transport = stream; @@ -462,12 +307,8 @@ fn connect_with_timeout(authority: &str, timeout: Duration) -> io::Result io::Error { @@ -482,16 +323,10 @@ struct BridgeState { fn read_bridge_state(path: Option<&Path>) -> io::Result { let Some(path) = path else { - return Ok(BridgeState { - stream_id: None, - last_seq: 0, - }); + return Ok(BridgeState { stream_id: None, last_seq: 0 }); }; if !path.exists() { - return Ok(BridgeState { - stream_id: None, - last_seq: 0, - }); + return Ok(BridgeState { stream_id: None, last_seq: 0 }); } let text = fs::read_to_string(path)?; @@ -509,23 +344,13 @@ fn write_bridge_state(path: Option<&Path>, state: &BridgeState) -> io::Result<() } fs::write( path, - format!( - "stream_id={}\nlast_seq={}\n", - state - .stream_id - .map(|value| value.to_string()) - .unwrap_or_else(|| "unset".to_string()), - state.last_seq - ), + format!("stream_id={}\nlast_seq={}\n", state.stream_id.map(|value| value.to_string()).unwrap_or_else(|| "unset".to_string()), state.last_seq), ) } fn parse_bridge_state(text: &str) -> io::Result { if let Ok(last_seq) = text.trim().parse::() { - return Ok(BridgeState { - stream_id: None, - last_seq, - }); + return Ok(BridgeState { stream_id: None, last_seq }); } let mut stream_id = None; @@ -540,43 +365,30 @@ fn parse_bridge_state(text: &str) -> io::Result { stream_id = if value == "unset" { Some(None) } else { - Some(Some(value.parse::().map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid bridge state stream_id: {err}"), - ) - })?)) + Some(Some( + value + .parse::() + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid bridge state stream_id: {err}")))?, + )) }; } "last_seq" => { - last_seq = Some(value.trim().parse::().map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid bridge state last_seq: {err}"), - ) - })?); + last_seq = Some( + value + .trim() + .parse::() + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid bridge state last_seq: {err}")))?, + ); } _ => {} } } - let last_seq = last_seq.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "invalid bridge state: missing last_seq", - ) - })?; - Ok(BridgeState { - stream_id: stream_id.unwrap_or(None), - last_seq, - }) + let last_seq = last_seq.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid bridge state: missing last_seq"))?; + Ok(BridgeState { stream_id: stream_id.unwrap_or(None), last_seq }) } -fn reconcile_bridge_state( - source: &str, - state: &mut BridgeState, - hello: &ReplayHello, -) -> io::Result<()> { +fn reconcile_bridge_state(source: &str, state: &mut BridgeState, hello: &ReplayHello) -> io::Result<()> { if let Some(saved_stream_id) = state.stream_id { if saved_stream_id != hello.stream_id { eprintln!( @@ -666,46 +478,24 @@ impl CollectorEndpoint { if let Some(rest) = input.strip_prefix("http://") { let (authority, path) = split_authority_and_path(rest); if authority.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "collector.url missing host:port", - )); + return Err(io::Error::new(io::ErrorKind::InvalidInput, "collector.url missing host:port")); } - return Ok(Self { - authority: authority.to_string(), - path: normalise_path(path), - tls: false, - }); + return Ok(Self { authority: authority.to_string(), path: normalise_path(path), tls: false }); } if let Some(rest) = input.strip_prefix("https://") { let (authority, path) = split_authority_and_path(rest); if authority.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "collector.url missing host:port", - )); + return Err(io::Error::new(io::ErrorKind::InvalidInput, "collector.url missing host:port")); } - return Ok(Self { - authority: authority.to_string(), - path: normalise_path(path), - tls: true, - }); + return Ok(Self { authority: authority.to_string(), path: normalise_path(path), tls: true }); } - Ok(Self { - authority: input.to_string(), - path: "/v1/logs".to_string(), - tls: false, - }) + Ok(Self { authority: input.to_string(), path: "/v1/logs".to_string(), tls: false }) } } -fn post_raw_otlp_http_transport( - endpoint: &CollectorEndpoint, - payload: &[u8], - transport: &mut T, -) -> io::Result<()> { +fn post_raw_otlp_http_transport(endpoint: &CollectorEndpoint, payload: &[u8], transport: &mut T) -> io::Result<()> { write!( transport, "POST {} HTTP/1.1\r\nHost: {}\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", @@ -719,10 +509,7 @@ fn post_raw_otlp_http_transport( let mut response = String::new(); std::io::Read::read_to_string(transport, &mut response)?; if !response.starts_with("HTTP/1.1 200") && !response.starts_with("HTTP/1.0 200") { - return Err(io::Error::other(format!( - "collector returned non-200 response: {}", - response.lines().next().unwrap_or("unknown response") - ))); + return Err(io::Error::other(format!("collector returned non-200 response: {}", response.lines().next().unwrap_or("unknown response")))); } Ok(()) diff --git a/logjetd/src/replay_utst.rs b/logjetd/src/replay_utst.rs index 5a998b8..91d9048 100644 --- a/logjetd/src/replay_utst.rs +++ b/logjetd/src/replay_utst.rs @@ -1,7 +1,6 @@ use super::{ - BridgeState, CollectorEndpoint, CollectorTransport, EnqueueOutcome, ExportTask, - enqueue_export_task, parse_bridge_state, read_bridge_state, reconcile_bridge_state, - write_bridge_state, + BridgeState, CollectorEndpoint, CollectorTransport, EnqueueOutcome, ExportTask, enqueue_export_task, parse_bridge_state, read_bridge_state, + reconcile_bridge_state, write_bridge_state, }; use crate::config::{BackpressureMode, CollectorConfig, UpstreamMode}; use crate::protocol::ReplayHello; @@ -40,38 +39,16 @@ fn https_url_is_supported() { #[test] fn missing_authority_is_rejected() { - let err = CollectorEndpoint::parse("http:///v1/logs") - .err() - .unwrap() - .to_string(); + let err = CollectorEndpoint::parse("http:///v1/logs").err().unwrap().to_string(); assert!(err.contains("missing host:port")); } #[test] fn bridge_state_round_trip() { let path = unique_temp_path("bridge-state"); - assert_eq!( - read_bridge_state(Some(&path)).unwrap(), - BridgeState { - stream_id: None, - last_seq: 0, - } - ); - write_bridge_state( - Some(&path), - &BridgeState { - stream_id: Some(42), - last_seq: 77, - }, - ) - .unwrap(); - assert_eq!( - read_bridge_state(Some(&path)).unwrap(), - BridgeState { - stream_id: Some(42), - last_seq: 77, - } - ); + assert_eq!(read_bridge_state(Some(&path)).unwrap(), BridgeState { stream_id: None, last_seq: 0 }); + write_bridge_state(Some(&path), &BridgeState { stream_id: Some(42), last_seq: 77 }).unwrap(); + assert_eq!(read_bridge_state(Some(&path)).unwrap(), BridgeState { stream_id: Some(42), last_seq: 77 }); fs::remove_file(path).unwrap(); } @@ -87,85 +64,30 @@ fn invalid_bridge_state_is_rejected() { #[test] fn legacy_numeric_bridge_state_still_parses() { let state = parse_bridge_state("123\n").unwrap(); - assert_eq!( - state, - BridgeState { - stream_id: None, - last_seq: 123, - } - ); + assert_eq!(state, BridgeState { stream_id: None, last_seq: 123 }); } #[test] fn bridge_state_resets_on_stream_id_change() { - let mut state = BridgeState { - stream_id: Some(11), - last_seq: 99, - }; - reconcile_bridge_state( - "127.0.0.1:7002", - &mut state, - &ReplayHello { - stream_id: 22, - first_seq: 1, - last_seq: 5, - }, - ) - .unwrap(); - assert_eq!( - state, - BridgeState { - stream_id: Some(22), - last_seq: 0, - } - ); + let mut state = BridgeState { stream_id: Some(11), last_seq: 99 }; + reconcile_bridge_state("127.0.0.1:7002", &mut state, &ReplayHello { stream_id: 22, first_seq: 1, last_seq: 5 }).unwrap(); + assert_eq!(state, BridgeState { stream_id: Some(22), last_seq: 0 }); } #[test] fn bridge_state_resets_when_legacy_saved_seq_is_above_upstream_last_seq() { - let mut state = BridgeState { - stream_id: None, - last_seq: 99, - }; - reconcile_bridge_state( - "127.0.0.1:7002", - &mut state, - &ReplayHello { - stream_id: 55, - first_seq: 1, - last_seq: 5, - }, - ) - .unwrap(); - assert_eq!( - state, - BridgeState { - stream_id: Some(55), - last_seq: 0, - } - ); + let mut state = BridgeState { stream_id: None, last_seq: 99 }; + reconcile_bridge_state("127.0.0.1:7002", &mut state, &ReplayHello { stream_id: 55, first_seq: 1, last_seq: 5 }).unwrap(); + assert_eq!(state, BridgeState { stream_id: Some(55), last_seq: 0 }); } #[test] fn disconnect_mode_errors_when_export_queue_is_full() { let transport = test_collector_transport(BackpressureMode::Disconnect, 1); let (task_tx, _task_rx) = mpsc::sync_channel(1); - task_tx - .send(ExportTask { - seq: 1, - payload: vec![1], - }) - .unwrap(); - - let err = enqueue_export_task( - &task_tx, - &transport, - ExportTask { - seq: 2, - payload: vec![2], - }, - ) - .unwrap_err(); + task_tx.send(ExportTask { seq: 1, payload: vec![1] }).unwrap(); + + let err = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, payload: vec![2] }).unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); } @@ -173,35 +95,15 @@ fn disconnect_mode_errors_when_export_queue_is_full() { fn drop_newest_mode_reports_drop_when_export_queue_is_full() { let transport = test_collector_transport(BackpressureMode::DropNewest, 1); let (task_tx, _task_rx) = mpsc::sync_channel(1); - task_tx - .send(ExportTask { - seq: 1, - payload: vec![1], - }) - .unwrap(); - - let outcome = enqueue_export_task( - &task_tx, - &transport, - ExportTask { - seq: 2, - payload: vec![2], - }, - ) - .unwrap(); + task_tx.send(ExportTask { seq: 1, payload: vec![1] }).unwrap(); + + let outcome = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, payload: vec![2] }).unwrap(); assert_eq!(outcome, EnqueueOutcome::DroppedNewest); } -fn test_collector_transport( - mode: BackpressureMode, - max_buffered_records: usize, -) -> CollectorTransport { +fn test_collector_transport(mode: BackpressureMode, max_buffered_records: usize) -> CollectorTransport { CollectorTransport { - endpoint: CollectorEndpoint { - authority: "127.0.0.1:4318".to_string(), - path: "/v1/logs".to_string(), - tls: false, - }, + endpoint: CollectorEndpoint { authority: "127.0.0.1:4318".to_string(), path: "/v1/logs".to_string(), tls: false }, timeout: std::time::Duration::from_millis(1000), backpressure_enabled: true, backpressure_mode: mode, @@ -220,12 +122,6 @@ fn test_collector_transport( } fn unique_temp_path(label: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - std::env::temp_dir().join(format!( - "logjetd-{label}-{nanos}-{}.state", - std::process::id() - )) + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}.state", std::process::id())) } diff --git a/logjetd/src/spool.rs b/logjetd/src/spool.rs index 650fd0b..8bd735e 100644 --- a/logjetd/src/spool.rs +++ b/logjetd/src/spool.rs @@ -98,15 +98,10 @@ impl Spool { pub fn next_for_cursor(&self, cursor: &mut ReplayCursor) -> io::Result> { match (self, cursor) { - (Self::Buffer(spool), ReplayCursor::Buffer(cursor)) => { - Ok(spool.next_for_cursor(cursor)) - } + (Self::Buffer(spool), ReplayCursor::Buffer(cursor)) => Ok(spool.next_for_cursor(cursor)), (Self::File(spool), ReplayCursor::File(cursor)) => spool.next_for_cursor(cursor), (Self::Buffer(_), ReplayCursor::File(_)) | (Self::File(_), ReplayCursor::Buffer(_)) => { - Err(io::Error::new( - ErrorKind::InvalidInput, - "replay cursor type does not match spool type", - )) + Err(io::Error::new(ErrorKind::InvalidInput, "replay cursor type does not match spool type")) } } } @@ -140,21 +135,13 @@ impl Spool { Some((_first_seq, last_seq)) => last_seq, None => 0, }; - last_seq - .checked_add(1) - .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "sequence seed overflow")) + last_seq.checked_add(1).ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "sequence seed overflow")) } } impl BufferSpool { fn new(config: BufferConfig) -> Self { - Self { - stream_id: generate_stream_id(), - limit: config.limit, - keep_messages: config.keep_messages, - records: VecDeque::new(), - tail_bytes: 0, - } + Self { stream_id: generate_stream_id(), limit: config.limit, keep_messages: config.keep_messages, records: VecDeque::new(), tail_bytes: 0 } } fn append(&mut self, record: WireRecord) { @@ -168,25 +155,14 @@ impl BufferSpool { } fn replay_cursor_after(&self, last_seq: u64) -> BufferReplayCursor { - let next_index = self - .records - .iter() - .position(|record| record.seq > last_seq) - .unwrap_or(self.records.len()); - BufferReplayCursor { - next_index, - last_seq, - } + let next_index = self.records.iter().position(|record| record.seq > last_seq).unwrap_or(self.records.len()); + BufferReplayCursor { next_index, last_seq } } fn next_for_cursor(&self, cursor: &mut BufferReplayCursor) -> Option { let index_still_aligned = match cursor.next_index { 0 => true, - value => self - .records - .get(value.saturating_sub(1)) - .map(|record| record.seq <= cursor.last_seq) - .unwrap_or(false), + value => self.records.get(value.saturating_sub(1)).map(|record| record.seq <= cursor.last_seq).unwrap_or(false), }; if index_still_aligned @@ -198,10 +174,7 @@ impl BufferSpool { return Some(record.clone()); } - let next_index = self - .records - .iter() - .position(|record| record.seq > cursor.last_seq)?; + let next_index = self.records.iter().position(|record| record.seq > cursor.last_seq)?; let record = self.records.get(next_index)?.clone(); cursor.next_index = next_index + 1; cursor.last_seq = record.seq; @@ -237,12 +210,7 @@ impl BufferSpool { } fn recalculate_tail_bytes(&mut self) { - self.tail_bytes = self - .records - .iter() - .skip(self.keep_messages) - .map(record_size) - .sum(); + self.tail_bytes = self.records.iter().skip(self.keep_messages).map(record_size).sum(); } fn sequence_bounds(&self) -> Option<(u64, u64)> { @@ -269,19 +237,14 @@ impl FileSpool { if size < config.segment_size_bytes { (segment.id, segment.path.clone(), size) } else { - let next_id = segment.id.checked_add(1).ok_or_else(|| { - io::Error::new(ErrorKind::InvalidData, "segment id overflow") - })?; + let next_id = segment.id.checked_add(1).ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; (next_id, segment_path(&config.dir, &base_stem, next_id), 0) } } None => (0, segment_path(&config.dir, &base_stem, 0), 0), }; - let file = OpenOptions::new() - .create(true) - .append(true) - .open(&active_segment_path)?; + let file = OpenOptions::new().create(true).append(true).open(&active_segment_path)?; let mut spool = Self { dir: config.dir, @@ -304,14 +267,7 @@ impl FileSpool { self.rotate()?; } - self.active_writer - .push( - record.record_type, - record.seq, - record.ts_unix_ns, - &record.payload, - ) - .map_err(to_io_error)?; + self.active_writer.push(record.record_type, record.seq, record.ts_unix_ns, &record.payload).map_err(to_io_error)?; self.active_writer.flush_block().map_err(to_io_error)?; self.refresh_active_size()?; @@ -323,10 +279,7 @@ impl FileSpool { } fn replay_cursor_after(&self, last_seq: u64) -> FileReplayCursor { - FileReplayCursor { - next_segment_id_hint: 0, - last_seq, - } + FileReplayCursor { next_segment_id_hint: 0, last_seq } } fn next_for_cursor(&self, cursor: &mut FileReplayCursor) -> io::Result> { @@ -338,8 +291,7 @@ impl FileSpool { } let file = File::open(&segment.path)?; - let mut reader = - LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); + let mut reader = LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); while let Some(record) = reader.next_record().map_err(to_io_error)? { if record.seq <= floor_seq { @@ -374,15 +326,10 @@ impl FileSpool { fn rotate(&mut self) -> io::Result<()> { self.active_writer.flush_block().map_err(to_io_error)?; - self.active_segment_id = self - .active_segment_id - .checked_add(1) - .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; + self.active_segment_id = + self.active_segment_id.checked_add(1).ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; self.active_segment_path = segment_path(&self.dir, &self.base_stem, self.active_segment_id); - let file = OpenOptions::new() - .create(true) - .append(true) - .open(&self.active_segment_path)?; + let file = OpenOptions::new().create(true).append(true).open(&self.active_segment_path)?; self.active_writer = LogjetWriter::new(file); self.active_size_bytes = 0; Ok(()) @@ -418,15 +365,10 @@ impl FileSpool { fn advance_empty_active_segment(&mut self) -> io::Result<()> { self.active_writer.flush_block().map_err(to_io_error)?; let old_path = self.active_segment_path.clone(); - self.active_segment_id = self - .active_segment_id - .checked_add(1) - .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; + self.active_segment_id = + self.active_segment_id.checked_add(1).ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "segment id overflow"))?; self.active_segment_path = segment_path(&self.dir, &self.base_stem, self.active_segment_id); - let file = OpenOptions::new() - .create(true) - .append(true) - .open(&self.active_segment_path)?; + let file = OpenOptions::new().create(true).append(true).open(&self.active_segment_path)?; self.active_writer = LogjetWriter::new(file); self.active_size_bytes = 0; if old_path.exists() { @@ -441,8 +383,7 @@ impl FileSpool { for segment in list_segments(&self.dir, &self.base_stem)? { let file = File::open(&segment.path)?; - let mut reader = - LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); + let mut reader = LogjetReader::with_config(BufReader::new(file), ReaderConfig::default()); while let Some(record) = reader.next_record().map_err(to_io_error)? { if record.seq <= self.consumed_through_seq { @@ -466,9 +407,7 @@ pub fn inspect_path(path: &Path) -> io::Result<()> { if path.is_dir() { for entry in fs::read_dir(path)? { let entry = entry?; - if entry.file_type()?.is_file() - && entry.path().extension().and_then(|ext| ext.to_str()) == Some("logjet") - { + if entry.file_type()?.is_file() && entry.path().extension().and_then(|ext| ext.to_str()) == Some("logjet") { inspect_file(&entry.path())?; } } @@ -486,14 +425,8 @@ pub fn print_named_segments(dir: &Path, file_name: &str) -> io::Result<()> { summary.path.display(), summary.size_bytes, summary.record_count, - summary - .first_seq - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()), - summary - .last_seq - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()) + summary.first_seq.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string()), + summary.last_seq.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string()) ); } Ok(()) @@ -529,23 +462,13 @@ pub fn summarise_named_segments(dir: &Path, file_name: &str) -> io::Result, - keep_bytes: Option, - dry_run: bool, + dir: &Path, file_name: &str, keep_files: Option, keep_bytes: Option, dry_run: bool, ) -> io::Result> { if keep_files.is_none() && keep_bytes.is_none() { - return Err(io::Error::new( - ErrorKind::InvalidInput, - "set --keep-files or --keep-bytes", - )); + return Err(io::Error::new(ErrorKind::InvalidInput, "set --keep-files or --keep-bytes")); } if keep_files.is_some() && keep_bytes.is_some() { - return Err(io::Error::new( - ErrorKind::InvalidInput, - "use either --keep-files or --keep-bytes, not both", - )); + return Err(io::Error::new(ErrorKind::InvalidInput, "use either --keep-files or --keep-bytes, not both")); } let summaries = summarise_named_segments(dir, file_name)?; @@ -577,10 +500,7 @@ pub fn prune_named_segments( } } - let removed_paths = remove_ids - .into_iter() - .map(|index| summaries[index].path.clone()) - .collect::>(); + let removed_paths = remove_ids.into_iter().map(|index| summaries[index].path.clone()).collect::>(); if !dry_run { for path in &removed_paths { @@ -608,13 +528,7 @@ fn inspect_file(path: &Path) -> io::Result<()> { } fn print_record(record: &OwnedRecord) { - println!( - "type={:?} seq={} ts={} payload_len={}", - record.record_type, - record.seq, - record.ts_unix_ns, - record.payload.len() - ); + println!("type={:?} seq={} ts={} payload_len={}", record.record_type, record.seq, record.ts_unix_ns, record.payload.len()); } fn list_segments(dir: &Path, base_stem: &str) -> io::Result> { @@ -634,10 +548,7 @@ fn list_segments(dir: &Path, base_stem: &str) -> io::Result> { continue; }; - segments.push(SegmentInfo { - id, - path: entry.path(), - }); + segments.push(SegmentInfo { id, path: entry.path() }); } segments.sort_by_key(|segment| segment.id); @@ -649,10 +560,7 @@ pub fn list_named_segments(dir: &Path, file_name: &str) -> io::Result String { - file_name - .strip_suffix(".logjet") - .unwrap_or(file_name) - .to_string() + file_name.strip_suffix(".logjet").unwrap_or(file_name).to_string() } fn parse_segment_id(name: &str, base_stem: &str) -> Option { @@ -669,19 +577,11 @@ fn parse_segment_id(name: &str, base_stem: &str) -> Option { } fn segment_path(dir: &Path, base_stem: &str, id: u64) -> PathBuf { - if id == 0 { - dir.join(format!("{base_stem}.logjet")) - } else { - dir.join(format!("{base_stem}-{id}.logjet")) - } + if id == 0 { dir.join(format!("{base_stem}.logjet")) } else { dir.join(format!("{base_stem}-{id}.logjet")) } } fn record_size(record: &WireRecord) -> usize { - 1usize - .saturating_add(8) - .saturating_add(8) - .saturating_add(4) - .saturating_add(record.payload.len()) + 1usize.saturating_add(8).saturating_add(8).saturating_add(4).saturating_add(record.payload.len()) } fn to_io_error(err: logjet::Error) -> io::Error { @@ -694,12 +594,7 @@ fn read_consumed_state(path: &Path) -> io::Result { } let text = fs::read_to_string(path)?; - let seq = text.trim().parse::().map_err(|err| { - io::Error::new( - ErrorKind::InvalidData, - format!("invalid consumed state: {err}"), - ) - })?; + let seq = text.trim().parse::().map_err(|err| io::Error::new(ErrorKind::InvalidData, format!("invalid consumed state: {err}")))?; Ok(seq) } @@ -710,9 +605,7 @@ fn write_consumed_state(path: &Path, seq: u64) -> io::Result<()> { fn read_or_create_stream_id(path: &Path) -> io::Result { if path.exists() { let text = fs::read_to_string(path)?; - let stream_id = text.trim().parse::().map_err(|err| { - io::Error::new(ErrorKind::InvalidData, format!("invalid stream id: {err}")) - })?; + let stream_id = text.trim().parse::().map_err(|err| io::Error::new(ErrorKind::InvalidData, format!("invalid stream id: {err}")))?; return Ok(stream_id); } @@ -722,10 +615,7 @@ fn read_or_create_stream_id(path: &Path) -> io::Result { } fn generate_stream_id() -> u64 { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() as u64; + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos() as u64; nanos ^ ((std::process::id() as u64) << 32) } diff --git a/logjetd/src/spool_utst.rs b/logjetd/src/spool_utst.rs index 40da180..25d43e9 100644 --- a/logjetd/src/spool_utst.rs +++ b/logjetd/src/spool_utst.rs @@ -8,18 +8,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; #[test] fn preserve_prefix_survives_eviction() { - let mut spool = BufferSpool::new(BufferConfig { - limit: BufferLimit::Bytes(90), - keep_messages: 2, - }); + let mut spool = BufferSpool::new(BufferConfig { limit: BufferLimit::Bytes(90), keep_messages: 2 }); for seq in 1..=5 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![0u8; 20], - }); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![0u8; 20] }); } let kept: Vec = spool.records.iter().map(|record| record.seq).collect(); @@ -28,18 +20,10 @@ fn preserve_prefix_survives_eviction() { #[test] fn message_limit_rotates_tail_only() { - let mut spool = BufferSpool::new(BufferConfig { - limit: BufferLimit::Messages(4), - keep_messages: 2, - }); + let mut spool = BufferSpool::new(BufferConfig { limit: BufferLimit::Messages(4), keep_messages: 2 }); for seq in 1..=8 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![0u8; 8], - }); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![0u8; 8] }); } let kept: Vec = spool.records.iter().map(|record| record.seq).collect(); @@ -48,21 +32,10 @@ fn message_limit_rotates_tail_only() { #[test] fn replay_since_only_sends_newer_records() { - let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { - limit: BufferLimit::Messages(8), - keep_messages: 1, - })) - .unwrap(); + let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { limit: BufferLimit::Messages(8), keep_messages: 1 })).unwrap(); for seq in 1..=4 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8] }).unwrap(); } let mut bytes = Vec::new(); @@ -81,18 +54,10 @@ fn replay_since_only_sends_newer_records() { #[test] fn consume_through_removes_buffer_records() { - let mut spool = BufferSpool::new(BufferConfig { - limit: BufferLimit::Messages(8), - keep_messages: 2, - }); + let mut spool = BufferSpool::new(BufferConfig { limit: BufferLimit::Messages(8), keep_messages: 2 }); for seq in 1..=5 { - spool.append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8] }); } spool.consume_through(3); @@ -102,21 +67,10 @@ fn consume_through_removes_buffer_records() { #[test] fn buffer_replay_cursor_resyncs_after_front_records_are_consumed() { - let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { - limit: BufferLimit::Messages(8), - keep_messages: 1, - })) - .unwrap(); + let mut spool = Spool::open(StorageConfig::Buffer(BufferConfig { limit: BufferLimit::Messages(8), keep_messages: 1 })).unwrap(); for seq in 1..=5 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8] }).unwrap(); } let mut cursor = spool.replay_cursor_after(0).unwrap(); @@ -135,23 +89,12 @@ fn buffer_replay_cursor_resyncs_after_front_records_are_consumed() { #[test] fn file_spool_consume_state_survives_reopen() { let dir = unique_temp_dir("file-consume"); - let config = FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1024 * 1024, - }; + let config = FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1024 * 1024 }; { let mut spool = Spool::open(StorageConfig::File(config.clone())).unwrap(); for seq in 1..=3 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8] }).unwrap(); } spool.consume_through(2).unwrap(); } @@ -169,22 +112,11 @@ fn file_spool_consume_state_survives_reopen() { #[test] fn file_replay_cursor_skips_consumed_records_after_cleanup() { let dir = unique_temp_dir("file-cursor-consume"); - let config = FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1, - }; + let config = FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1 }; let mut spool = Spool::open(StorageConfig::File(config)).unwrap(); for seq in 1..=4 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8; 8] }).unwrap(); } let mut cursor = spool.replay_cursor_after(0).unwrap(); @@ -212,26 +144,8 @@ fn list_named_segments_orders_numeric_suffixes() { fs::write(dir.join("other.logjet"), b"x").unwrap(); let segments = super::list_named_segments(&dir, "bofh.logjet").unwrap(); - let names: Vec = segments - .iter() - .map(|segment| { - segment - .path - .file_name() - .unwrap() - .to_string_lossy() - .into_owned() - }) - .collect(); - assert_eq!( - names, - vec![ - "bofh.logjet", - "bofh-1.logjet", - "bofh-2.logjet", - "bofh-10.logjet" - ] - ); + let names: Vec = segments.iter().map(|segment| segment.path.file_name().unwrap().to_string_lossy().into_owned()).collect(); + assert_eq!(names, vec!["bofh.logjet", "bofh-1.logjet", "bofh-2.logjet", "bofh-10.logjet"]); fs::remove_dir_all(dir).unwrap(); } @@ -239,22 +153,11 @@ fn list_named_segments_orders_numeric_suffixes() { #[test] fn file_spool_rotates_when_segment_size_is_exceeded() { let dir = unique_temp_dir("file-rotate"); - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1, - })) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1 })).unwrap(); for seq in 1..=2 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![0u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![0u8; 8] }).unwrap(); } assert!(dir.join("bofh.logjet").exists()); @@ -266,37 +169,17 @@ fn file_spool_rotates_when_segment_size_is_exceeded() { fn file_spool_reuses_existing_non_full_segment() { let dir = unique_temp_dir("file-reuse"); { - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1024 * 1024, - })) - .unwrap(); - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq: 1, - ts_unix_ns: 1, - payload: vec![1u8; 8], - }) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1024 * 1024 })) + .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq: 1, ts_unix_ns: 1, payload: vec![1u8; 8] }).unwrap(); } { - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1024 * 1024, - })) - .unwrap(); - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq: 2, - ts_unix_ns: 2, - payload: vec![2u8; 8], - }) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1024 * 1024 })) + .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq: 2, ts_unix_ns: 2, payload: vec![2u8; 8] }).unwrap(); } assert!(dir.join("bofh.logjet").exists()); @@ -307,33 +190,15 @@ fn file_spool_reuses_existing_non_full_segment() { #[test] fn file_spool_preserves_stream_id_and_advances_sequence_seed_after_reopen() { let dir = unique_temp_dir("file-stream-id"); - let config = FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1024 * 1024, - }; + let config = FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1024 * 1024 }; let first_stream_id; { let mut spool = Spool::open(StorageConfig::File(config.clone())).unwrap(); first_stream_id = spool.stream_id(); assert_eq!(spool.next_sequence_seed().unwrap(), 1); - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq: 1, - ts_unix_ns: 1, - payload: vec![1u8; 8], - }) - .unwrap(); - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq: 2, - ts_unix_ns: 2, - payload: vec![2u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq: 1, ts_unix_ns: 1, payload: vec![1u8; 8] }).unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq: 2, ts_unix_ns: 2, payload: vec![2u8; 8] }).unwrap(); } { @@ -348,22 +213,11 @@ fn file_spool_preserves_stream_id_and_advances_sequence_seed_after_reopen() { #[test] fn summarise_named_segments_reports_sequence_ranges() { let dir = unique_temp_dir("segment-summary"); - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1, - })) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1 })).unwrap(); for seq in 1..=3 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8; 8] }).unwrap(); } let summaries = super::summarise_named_segments(&dir, "bofh.logjet").unwrap(); @@ -379,22 +233,11 @@ fn summarise_named_segments_reports_sequence_ranges() { #[test] fn prune_named_segments_by_file_count_keeps_newest_segment() { let dir = unique_temp_dir("segment-prune-count"); - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1, - })) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1 })).unwrap(); for seq in 1..=4 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8; 8] }).unwrap(); } let removed = super::prune_named_segments(&dir, "bofh.logjet", Some(2), None, false).unwrap(); @@ -403,14 +246,7 @@ fn prune_named_segments_by_file_count_keeps_newest_segment() { let names: Vec = super::list_named_segments(&dir, "bofh.logjet") .unwrap() .into_iter() - .map(|segment| { - segment - .path - .file_name() - .unwrap() - .to_string_lossy() - .into_owned() - }) + .map(|segment| segment.path.file_name().unwrap().to_string_lossy().into_owned()) .collect(); assert_eq!(names.len(), 2); assert_eq!(names, vec!["bofh-3.logjet", "bofh-4.logjet"]); @@ -421,41 +257,22 @@ fn prune_named_segments_by_file_count_keeps_newest_segment() { #[test] fn prune_named_segments_dry_run_does_not_remove_files() { let dir = unique_temp_dir("segment-prune-dry-run"); - let mut spool = Spool::open(StorageConfig::File(FileConfig { - dir: dir.clone(), - name: "bofh.logjet".to_string(), - segment_size_bytes: 1, - })) - .unwrap(); + let mut spool = + Spool::open(StorageConfig::File(FileConfig { dir: dir.clone(), name: "bofh.logjet".to_string(), segment_size_bytes: 1 })).unwrap(); for seq in 1..=3 { - spool - .append(WireRecord { - record_type: RecordType::Logs, - seq, - ts_unix_ns: seq, - payload: vec![seq as u8; 8], - }) - .unwrap(); + spool.append(WireRecord { record_type: RecordType::Logs, seq, ts_unix_ns: seq, payload: vec![seq as u8; 8] }).unwrap(); } let removed = super::prune_named_segments(&dir, "bofh.logjet", Some(1), None, true).unwrap(); assert_eq!(removed.len(), 3); - assert_eq!( - super::list_named_segments(&dir, "bofh.logjet") - .unwrap() - .len(), - 4 - ); + assert_eq!(super::list_named_segments(&dir, "bofh.logjet").unwrap().len(), 4); fs::remove_dir_all(dir).unwrap(); } fn unique_temp_dir(label: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); let dir = std::env::temp_dir().join(format!("logjetd-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&dir).unwrap(); dir diff --git a/logjetd/src/tls.rs b/logjetd/src/tls.rs index 33b6352..2ba2ee9 100644 --- a/logjetd/src/tls.rs +++ b/logjetd/src/tls.rs @@ -11,51 +11,26 @@ use rustls::{ClientConfig, RootCertStore, ServerConfig}; use crate::config::{CollectorConfig, IngestTlsConfig, TlsConfig}; pub fn load_server_config(tls: &TlsConfig) -> io::Result> { - load_server_config_from_parts( - tls.cert_file.as_deref(), - tls.key_file.as_deref(), - tls.ca_file.as_deref(), - tls.require_client_cert, - "tls", - ) + load_server_config_from_parts(tls.cert_file.as_deref(), tls.key_file.as_deref(), tls.ca_file.as_deref(), tls.require_client_cert, "tls") } pub fn load_ingest_server_config(tls: &IngestTlsConfig) -> io::Result> { - load_server_config_from_parts( - tls.cert_file.as_deref(), - tls.key_file.as_deref(), - tls.ca_file.as_deref(), - tls.require_client_cert, - "ingest", - ) + load_server_config_from_parts(tls.cert_file.as_deref(), tls.key_file.as_deref(), tls.ca_file.as_deref(), tls.require_client_cert, "ingest") } pub fn load_client_config(tls: &TlsConfig) -> io::Result> { - load_client_config_from_parts( - tls.ca_file.as_deref(), - tls.cert_file.as_deref(), - tls.key_file.as_deref(), - "tls", - ) + load_client_config_from_parts(tls.ca_file.as_deref(), tls.cert_file.as_deref(), tls.key_file.as_deref(), "tls") } pub fn load_collector_client_config(collector: &CollectorConfig) -> io::Result> { - load_client_config_from_parts( - collector.ca_file.as_deref(), - collector.cert_file.as_deref(), - collector.key_file.as_deref(), - "collector", - ) + load_client_config_from_parts(collector.ca_file.as_deref(), collector.cert_file.as_deref(), collector.key_file.as_deref(), "collector") } pub fn parse_server_name(tls: &TlsConfig, authority: &str) -> io::Result> { parse_server_name_override(tls.server_name.as_deref(), authority) } -pub fn parse_collector_server_name( - collector: &CollectorConfig, - authority: &str, -) -> io::Result> { +pub fn parse_collector_server_name(collector: &CollectorConfig, authority: &str) -> io::Result> { parse_server_name_override(collector.server_name.as_deref(), authority) } @@ -63,31 +38,20 @@ pub fn authority_host(authority: &str) -> &str { if let Some(rest) = authority.strip_prefix('[') { return rest.split(']').next().unwrap_or(authority); } - authority - .rsplit_once(':') - .map(|(host, _)| host) - .unwrap_or(authority) + authority.rsplit_once(':').map(|(host, _)| host).unwrap_or(authority) } -fn parse_server_name_override( - override_name: Option<&str>, - authority: &str, -) -> io::Result> { +fn parse_server_name_override(override_name: Option<&str>, authority: &str) -> io::Result> { let name = override_name.unwrap_or_else(|| authority_host(authority)); if let Ok(ip) = name.parse::() { return Ok(ServerName::IpAddress(ip.into())); } - ServerName::try_from(name.to_string()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) + ServerName::try_from(name.to_string()).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) } fn load_server_config_from_parts( - cert_file: Option<&Path>, - key_file: Option<&Path>, - ca_file: Option<&Path>, - require_client_cert: bool, - namespace: &str, + cert_file: Option<&Path>, key_file: Option<&Path>, ca_file: Option<&Path>, require_client_cert: bool, namespace: &str, ) -> io::Result> { let cert_file = cert_file.ok_or_else(|| { io::Error::new( @@ -107,41 +71,27 @@ fn load_server_config_from_parts( let builder = ServerConfig::builder(); let server_config = if require_client_cert { let ca_file = ca_file.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("{namespace}.ca-file is required when client certificates are required"), - ) + io::Error::new(io::ErrorKind::InvalidInput, format!("{namespace}.ca-file is required when client certificates are required")) })?; let roots = load_root_store(ca_file)?; - let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) - .build() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; + let verifier = + WebPkiClientVerifier::builder(Arc::new(roots)).build().map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; builder .with_client_cert_verifier(verifier) .with_single_cert(certs, key) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? } else { - builder - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? + builder.with_no_client_auth().with_single_cert(certs, key).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? }; Ok(Arc::new(server_config)) } fn load_client_config_from_parts( - ca_file: Option<&Path>, - cert_file: Option<&Path>, - key_file: Option<&Path>, - namespace: &str, + ca_file: Option<&Path>, cert_file: Option<&Path>, key_file: Option<&Path>, namespace: &str, ) -> io::Result> { - let ca_file = ca_file.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("{namespace}.ca-file is required for TLS client mode"), - ) - })?; + let ca_file = + ca_file.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, format!("{namespace}.ca-file is required for TLS client mode")))?; let roots = load_root_store(ca_file)?; let builder = ClientConfig::builder().with_root_certificates(roots); @@ -153,9 +103,7 @@ fn load_client_config_from_parts( _ => { return Err(io::Error::new( io::ErrorKind::InvalidInput, - format!( - "{namespace}.cert-file and {namespace}.key-file must either both be set or both be unset" - ), + format!("{namespace}.cert-file and {namespace}.key-file must either both be set or both be unset"), )); } }; @@ -168,34 +116,21 @@ fn load_root_store(path: &Path) -> io::Result { let mut roots = RootCertStore::empty(); let (_added, ignored) = roots.add_parsable_certificates(certs); if ignored > 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "failed to parse {ignored} CA certificate(s) from {}", - path.display() - ), - )); + return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("failed to parse {ignored} CA certificate(s) from {}", path.display()))); } Ok(roots) } fn load_certs(path: &Path) -> io::Result>> { let mut reader = BufReader::new(File::open(path)?); - rustls_pemfile::certs(&mut reader) - .collect::, _>>() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) + rustls_pemfile::certs(&mut reader).collect::, _>>().map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string())) } fn load_private_key(path: &Path) -> io::Result> { let mut reader = BufReader::new(File::open(path)?); rustls_pemfile::private_key(&mut reader) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("no private key found in {}", path.display()), - ) - }) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, format!("no private key found in {}", path.display()))) } #[cfg(test)] diff --git a/logjetd/src/tls_utst.rs b/logjetd/src/tls_utst.rs index d60f005..b8d52d0 100644 --- a/logjetd/src/tls_utst.rs +++ b/logjetd/src/tls_utst.rs @@ -1,17 +1,9 @@ -use super::{ - authority_host, load_client_config, load_ingest_server_config, load_server_config, - parse_server_name, -}; +use super::{authority_host, load_client_config, load_ingest_server_config, load_server_config, parse_server_name}; use crate::config::{IngestTlsConfig, TlsConfig}; use std::path::PathBuf; fn demo_cert_path(name: &str) -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("demo") - .join("remote-drain-tls") - .join("certs") - .join(name) + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..").join("demo").join("remote-drain-tls").join("certs").join(name) } #[test] @@ -41,28 +33,14 @@ fn parse_server_name_uses_override() { #[test] fn parse_server_name_accepts_ip_authority() { - let tls = TlsConfig { - enable: true, - ca_file: None, - cert_file: None, - key_file: None, - require_client_cert: false, - server_name: None, - }; + let tls = TlsConfig { enable: true, ca_file: None, cert_file: None, key_file: None, require_client_cert: false, server_name: None }; parse_server_name(&tls, "127.0.0.1:7002").unwrap(); } #[test] fn load_client_config_requires_ca_file() { - let tls = TlsConfig { - enable: true, - ca_file: None, - cert_file: None, - key_file: None, - require_client_cert: false, - server_name: None, - }; + let tls = TlsConfig { enable: true, ca_file: None, cert_file: None, key_file: None, require_client_cert: false, server_name: None }; let err = load_client_config(&tls).unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -70,14 +48,7 @@ fn load_client_config_requires_ca_file() { #[test] fn load_server_config_requires_cert_and_key() { - let tls = TlsConfig { - enable: true, - ca_file: None, - cert_file: None, - key_file: None, - require_client_cert: false, - server_name: None, - }; + let tls = TlsConfig { enable: true, ca_file: None, cert_file: None, key_file: None, require_client_cert: false, server_name: None }; let err = load_server_config(&tls).unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); @@ -100,13 +71,7 @@ fn load_server_config_requires_ca_when_client_certs_required() { #[test] fn load_ingest_server_config_requires_cert_and_key() { - let tls = IngestTlsConfig { - enable: true, - ca_file: None, - cert_file: None, - key_file: None, - require_client_cert: false, - }; + let tls = IngestTlsConfig { enable: true, ca_file: None, cert_file: None, key_file: None, require_client_cert: false }; let err = load_ingest_server_config(&tls).unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); diff --git a/logjetd/tests/bridge_flows.rs b/logjetd/tests/bridge_flows.rs index a765283..44ced52 100644 --- a/logjetd/tests/bridge_flows.rs +++ b/logjetd/tests/bridge_flows.rs @@ -7,8 +7,8 @@ use std::thread; use std::time::Duration; use common::{ - ChildGuard, MockCollector, TestDir, connect_replay_client, free_port, logjetd_command, - post_otlp_http, read_replay_message, replay_messages, wait_for_tcp, wait_until, + ChildGuard, MockCollector, TestDir, connect_replay_client, free_port, logjetd_command, post_otlp_http, read_replay_message, replay_messages, + wait_for_tcp, wait_until, }; #[test] @@ -26,9 +26,7 @@ fn bridge_keep_forwards_backlog_in_order() -> io::Result<()> { )?; let bridge_config = dir.write( "bridge.conf", - &format!( - "collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n" - ), + &format!("collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n"), )?; let _appliance = ChildGuard::spawn({ @@ -50,27 +48,11 @@ fn bridge_keep_forwards_backlog_in_order() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 3) - })?; - assert_eq!( - collector.messages(), - vec![ - "KEEP 001".to_string(), - "KEEP 002".to_string(), - "KEEP 003".to_string() - ] - ); + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; + assert_eq!(collector.messages(), vec!["KEEP 001".to_string(), "KEEP 002".to_string(), "KEEP 003".to_string()]); let retained = replay_messages(&format!("127.0.0.1:{replay_port}"), 0, 3)?; - assert_eq!( - retained, - vec![ - "KEEP 001".to_string(), - "KEEP 002".to_string(), - "KEEP 003".to_string() - ] - ); + assert_eq!(retained, vec!["KEEP 001".to_string(), "KEEP 002".to_string(), "KEEP 003".to_string()]); Ok(()) } @@ -90,9 +72,7 @@ fn bridge_drain_consumes_upstream_records() -> io::Result<()> { )?; let bridge_config = dir.write( "bridge.conf", - &format!( - "collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: drain\n" - ), + &format!("collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: drain\n"), )?; let _appliance = ChildGuard::spawn({ @@ -114,21 +94,10 @@ fn bridge_drain_consumes_upstream_records() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 3) - })?; - assert_eq!( - collector.messages(), - vec![ - "DRAIN 001".to_string(), - "DRAIN 002".to_string(), - "DRAIN 003".to_string() - ] - ); + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; + assert_eq!(collector.messages(), vec!["DRAIN 001".to_string(), "DRAIN 002".to_string(), "DRAIN 003".to_string()]); - wait_until(Duration::from_secs(5), || { - Ok(replay_messages(&format!("127.0.0.1:{replay_port}"), 0, 1)?.is_empty()) - })?; + wait_until(Duration::from_secs(5), || Ok(replay_messages(&format!("127.0.0.1:{replay_port}"), 0, 1)?.is_empty()))?; Ok(()) } @@ -166,11 +135,7 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; for message in ["RESUME 001", "RESUME 002", "RESUME 003"] { - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-resume", - message, - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-resume", message)?; } { @@ -179,17 +144,11 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 3) - })?; + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 3))?; } for message in ["RESUME 004", "RESUME 005", "RESUME 006"] { - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-resume", - message, - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-resume", message)?; } { @@ -198,9 +157,7 @@ fn bridge_resume_state_survives_restart() -> io::Result<()> { cmd.arg("--config").arg(&bridge_config).arg("bridge"); cmd })?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 6) - })?; + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 6))?; } assert_eq!( @@ -236,9 +193,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { )?; let bridge_config = dir.write( "bridge-file.conf", - &format!( - "collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n" - ), + &format!("collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n"), )?; let _appliance = ChildGuard::spawn({ @@ -261,9 +216,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { cmd })?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 5) - })?; + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 5))?; assert_eq!( collector.messages(), vec![ @@ -277,10 +230,7 @@ fn bridge_keep_works_with_file_rotation() -> io::Result<()> { let rotated_count = fs::read_dir(&spool_dir)? .filter_map(Result::ok) - .filter(|entry| { - entry.file_name().to_string_lossy().starts_with("rotation") - && entry.file_name().to_string_lossy().ends_with(".logjet") - }) + .filter(|entry| entry.file_name().to_string_lossy().starts_with("rotation") && entry.file_name().to_string_lossy().ends_with(".logjet")) .count(); assert!(rotated_count >= 2); @@ -331,24 +281,12 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { })?; wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5))?; wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-reset", - "ALPHA 001", - )?; - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-reset", - "ALPHA 002", - )?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 2) - })?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "ALPHA 001")?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "ALPHA 002")?; + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 2))?; } - wait_until(Duration::from_secs(5), || { - Ok(TcpStream::connect(format!("127.0.0.1:{replay_port}")).is_err()) - })?; + wait_until(Duration::from_secs(5), || Ok(TcpStream::connect(format!("127.0.0.1:{replay_port}")).is_err()))?; { let _appliance = ChildGuard::spawn({ @@ -358,30 +296,12 @@ fn bridge_resets_saved_state_when_upstream_stream_changes() -> io::Result<()> { })?; wait_for_tcp(&format!("127.0.0.1:{ingest_port}"), Duration::from_secs(5))?; wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-reset", - "BRAVO 001", - )?; - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-reset", - "BRAVO 002", - )?; - wait_until(Duration::from_secs(5), || { - Ok(collector.messages().len() >= 4) - })?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "BRAVO 001")?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-reset", "BRAVO 002")?; + wait_until(Duration::from_secs(5), || Ok(collector.messages().len() >= 4))?; } - assert_eq!( - collector.messages(), - vec![ - "ALPHA 001".to_string(), - "ALPHA 002".to_string(), - "BRAVO 001".to_string(), - "BRAVO 002".to_string(), - ] - ); + assert_eq!(collector.messages(), vec!["ALPHA 001".to_string(), "ALPHA 002".to_string(), "BRAVO 001".to_string(), "BRAVO 002".to_string(),]); Ok(()) } @@ -422,16 +342,10 @@ fn bridge_block_mode_handles_slow_collector_without_losing_order() -> io::Result })?; for index in 1..=6 { - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-slow", - &format!("SLOW {index:03}"), - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-slow", &format!("SLOW {index:03}"))?; } - wait_until(Duration::from_secs(10), || { - Ok(collector.messages().len() >= 6) - })?; + wait_until(Duration::from_secs(10), || Ok(collector.messages().len() >= 6))?; assert_eq!( collector.messages(), vec![ @@ -473,11 +387,7 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { for index in 1..=120 { let message = format!("RECOVER {index:03} {}", noisy_message(index)); - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "replay-corruption", - &message, - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "replay-corruption", &message)?; } } @@ -493,27 +403,15 @@ fn replay_recovers_after_middle_of_file_is_removed() -> io::Result<()> { let collector = MockCollector::start(collector_port)?; let status = { let mut cmd = logjetd_command(); - cmd.arg("replay") - .arg("--path") - .arg(&spool_dir) - .arg("--name") - .arg("recover.logjet") - .arg("--dest") - .arg(format!("127.0.0.1:{collector_port}")); + cmd.arg("replay").arg("--path").arg(&spool_dir).arg("--name").arg("recover.logjet").arg("--dest").arg(format!("127.0.0.1:{collector_port}")); cmd.status()? }; assert!(status.success()); - wait_until(Duration::from_secs(5), || { - Ok(!collector.messages().is_empty()) - })?; + wait_until(Duration::from_secs(5), || Ok(!collector.messages().is_empty()))?; let messages = collector.messages(); assert!(messages.len() < 120); - assert!( - messages - .iter() - .any(|message| message.starts_with("RECOVER 090 ")) - ); + assert!(messages.iter().any(|message| message.starts_with("RECOVER 090 "))); Ok(()) } @@ -533,9 +431,7 @@ fn bridge_forwards_large_payloads_end_to_end() -> io::Result<()> { )?; let bridge_config = dir.write( "bridge.conf", - &format!( - "collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n" - ), + &format!("collector.url: http://127.0.0.1:{collector_port}/v1/logs\nupstream.replay: 127.0.0.1:{replay_port}\nupstream.mode: keep\n"), )?; let _appliance = ChildGuard::spawn({ @@ -554,15 +450,9 @@ fn bridge_forwards_large_payloads_end_to_end() -> io::Result<()> { })?; let large_message = format!("LARGE 001 {}", noisy_message(10_001)); - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-large", - &large_message, - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-large", &large_message)?; - wait_until(Duration::from_secs(5), || { - Ok(!collector.messages().is_empty()) - })?; + wait_until(Duration::from_secs(5), || Ok(!collector.messages().is_empty()))?; assert_eq!(collector.messages(), vec![large_message]); Ok(()) @@ -590,11 +480,7 @@ fn multiple_replay_clients_receive_backlog_independently() -> io::Result<()> { wait_for_tcp(&format!("127.0.0.1:{replay_port}"), Duration::from_secs(5))?; for index in 1..=20 { - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-multi", - &format!("MULTI {index:03}"), - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-multi", &format!("MULTI {index:03}"))?; } let replay_addr = format!("127.0.0.1:{replay_port}"); @@ -602,16 +488,10 @@ fn multiple_replay_clients_receive_backlog_independently() -> io::Result<()> { let first = thread::spawn(move || replay_messages(&replay_addr, 0, 20)); let second = thread::spawn(move || replay_messages(&replay_addr_clone, 0, 20)); - let first_messages = first - .join() - .map_err(|_| io::Error::other("first replay thread panicked"))??; - let second_messages = second - .join() - .map_err(|_| io::Error::other("second replay thread panicked"))??; + let first_messages = first.join().map_err(|_| io::Error::other("first replay thread panicked"))??; + let second_messages = second.join().map_err(|_| io::Error::other("second replay thread panicked"))??; - let expected = (1..=20) - .map(|index| format!("MULTI {index:03}")) - .collect::>(); + let expected = (1..=20).map(|index| format!("MULTI {index:03}")).collect::>(); assert_eq!(first_messages, expected); assert_eq!(second_messages, expected); @@ -645,46 +525,26 @@ fn replay_client_receives_backlog_then_live_records_without_reconnect() -> io::R } let mut replay = connect_replay_client(&format!("127.0.0.1:{replay_port}"), 0, false)?; - assert_eq!( - read_replay_message(&mut replay)?, - Some("HANDOFF 001".to_string()) - ); - assert_eq!( - read_replay_message(&mut replay)?, - Some("HANDOFF 002".to_string()) - ); - assert_eq!( - read_replay_message(&mut replay)?, - Some("HANDOFF 003".to_string()) - ); + assert_eq!(read_replay_message(&mut replay)?, Some("HANDOFF 001".to_string())); + assert_eq!(read_replay_message(&mut replay)?, Some("HANDOFF 002".to_string())); + assert_eq!(read_replay_message(&mut replay)?, Some("HANDOFF 003".to_string())); thread::sleep(Duration::from_millis(200)); assert_eq!(read_replay_message(&mut replay)?, None); - post_otlp_http( - &format!("127.0.0.1:{ingest_port}"), - "bridge-live", - "HANDOFF 004", - )?; + post_otlp_http(&format!("127.0.0.1:{ingest_port}"), "bridge-live", "HANDOFF 004")?; replay.set_read_timeout(Some(Duration::from_secs(5)))?; - assert_eq!( - read_replay_message(&mut replay)?, - Some("HANDOFF 004".to_string()) - ); + assert_eq!(read_replay_message(&mut replay)?, Some("HANDOFF 004".to_string())); Ok(()) } fn noisy_message(seed: usize) -> String { - let mut value = (seed as u64) - .wrapping_mul(0x9E37_79B9_7F4A_7C15) - .wrapping_add(0xD1B5_4A32_D192_ED03); + let mut value = (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(0xD1B5_4A32_D192_ED03); let mut text = String::with_capacity(4096); for _ in 0..256 { - value = value - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); + value = value.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); text.push_str(&format!("{value:016x}")); } text diff --git a/logjetd/tests/common/mod.rs b/logjetd/tests/common/mod.rs index b51b528..e350b5a 100644 --- a/logjetd/tests/common/mod.rs +++ b/logjetd/tests/common/mod.rs @@ -23,12 +23,8 @@ pub struct TestDir { impl TestDir { pub fn new(label: &str) -> io::Result { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let path = - std::env::temp_dir().join(format!("logjetd-it-{label}-{nanos}-{}", std::process::id())); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let path = std::env::temp_dir().join(format!("logjetd-it-{label}-{nanos}-{}", std::process::id())); fs::create_dir_all(&path)?; Ok(Self { path }) } @@ -56,10 +52,7 @@ pub struct ChildGuard { impl ChildGuard { pub fn spawn(mut command: Command) -> io::Result { - let child = command - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; + let child = command.stdout(Stdio::null()).stderr(Stdio::null()).spawn()?; Ok(Self { child }) } } @@ -114,10 +107,7 @@ where return Ok(()); } if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "timed out waiting for condition", - )); + return Err(io::Error::new(io::ErrorKind::TimedOut, "timed out waiting for condition")); } thread::sleep(Duration::from_millis(25)); } @@ -150,34 +140,22 @@ impl MockCollector { }); let _ = addr; - Ok(Self { - received, - _thread: thread, - }) + Ok(Self { received, _thread: thread }) } pub fn messages(&self) -> Vec { - self.received - .lock() - .unwrap() - .iter() - .flat_map(extract_messages) - .collect() + self.received.lock().unwrap().iter().flat_map(extract_messages).collect() } } -fn handle_http_request( - stream: &mut TcpStream, - received: &Arc>>, - delay: Duration, -) -> io::Result<()> { +fn handle_http_request(stream: &mut TcpStream, received: &Arc>>, delay: Duration) -> io::Result<()> { let request = read_http_request(stream)?; if request.method != "POST" || request.path != "/v1/logs" { write_http_response(stream, 404, "not found")?; return Ok(()); } - let batch = ExportLogsServiceRequest::decode(request.body.as_slice()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + let batch = + ExportLogsServiceRequest::decode(request.body.as_slice()).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; received.lock().unwrap().push(batch); if !delay.is_zero() { thread::sleep(delay); @@ -201,40 +179,25 @@ fn read_http_request(stream: &mut TcpStream) -> io::Result { break; } if header.len() > 16 * 1024 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "header too large", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "header too large")); } } - let header_text = std::str::from_utf8(&header[..header.len() - 4]) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid header"))?; + let header_text = std::str::from_utf8(&header[..header.len() - 4]).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid header"))?; let mut lines = header_text.lines(); - let request_line = lines - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing request line"))?; + let request_line = lines.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing request line"))?; let mut parts = request_line.split_whitespace(); - let method = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing method"))? - .to_string(); - let path = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing path"))? - .to_string(); + let method = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing method"))?.to_string(); + let path = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing path"))?.to_string(); let mut content_length = None; for line in lines { if let Some((name, value)) = line.split_once(':') && name.eq_ignore_ascii_case("content-length") { - content_length = Some(value.trim().parse::().map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "invalid content-length") - })?); + content_length = Some(value.trim().parse::().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid content-length"))?); } } - let content_length = content_length - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; + let content_length = content_length.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing content-length"))?; let mut body = vec![0u8; content_length]; stream.read_exact(&mut body)?; Ok(HttpRequest { method, path, body }) @@ -246,14 +209,7 @@ fn write_http_response(stream: &mut TcpStream, status: u16, body: &str) -> io::R 404 => "Not Found", _ => "Error", }; - write!( - stream, - "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - status, - status_text, - body.len(), - body - )?; + write!(stream, "HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", status, status_text, body.len(), body)?; stream.flush() } @@ -272,10 +228,7 @@ pub fn post_otlp_http(addr: &str, service_name: &str, message: &str) -> io::Resu let mut response = String::new(); stream.read_to_string(&mut response)?; if !response.starts_with("HTTP/1.1 200") { - return Err(io::Error::other(format!( - "non-200 response: {}", - response.lines().next().unwrap_or("unknown") - ))); + return Err(io::Error::other(format!("non-200 response: {}", response.lines().next().unwrap_or("unknown")))); } Ok(()) } @@ -287,8 +240,8 @@ pub fn replay_messages(addr: &str, from_seq: u64, limit: usize) -> io::Result { - let batch = ExportLogsServiceRequest::decode(record.as_slice()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + let batch = + ExportLogsServiceRequest::decode(record.as_slice()).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; messages.extend(extract_messages(&batch)); } None => break, @@ -314,8 +267,7 @@ pub fn read_replay_message(stream: &mut TcpStream) -> io::Result> let Some(payload) = read_replay_payload(stream)? else { return Ok(None); }; - let batch = ExportLogsServiceRequest::decode(payload.as_slice()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + let batch = ExportLogsServiceRequest::decode(payload.as_slice()).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; Ok(extract_messages(&batch).into_iter().next()) } @@ -323,10 +275,7 @@ fn read_replay_hello(stream: &mut TcpStream) -> io::Result<()> { let mut magic = [0u8; 8]; stream.read_exact(&mut magic)?; if magic != REPLAY_HELLO_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "invalid replay hello magic", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid replay hello magic")); } let mut header = [0u8; 32]; stream.read_exact(&mut header)?; @@ -347,19 +296,14 @@ fn read_wire_record(stream: &mut TcpStream) -> io::Result>> { match stream.read_exact(&mut magic) { Ok(()) => {} Err(err) - if err.kind() == io::ErrorKind::UnexpectedEof - || err.kind() == io::ErrorKind::TimedOut - || err.kind() == io::ErrorKind::WouldBlock => + if err.kind() == io::ErrorKind::UnexpectedEof || err.kind() == io::ErrorKind::TimedOut || err.kind() == io::ErrorKind::WouldBlock => { return Ok(None); } Err(err) => return Err(err), } if magic != WIRE_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "invalid wire magic", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid wire magic")); } let mut header = [0u8; 24]; stream.read_exact(&mut header)?; @@ -370,21 +314,14 @@ fn read_wire_record(stream: &mut TcpStream) -> io::Result>> { } fn build_logs_request(service_name: &str, message: &str) -> ExportLogsServiceRequest { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as u64; ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { resource: Some(Resource { attributes: vec![KeyValue { key: "service.name".to_string(), value: Some(AnyValue { - value: Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - service_name.to_string(), - ), - ), + value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(service_name.to_string())), }), }], dropped_attributes_count: 0, @@ -401,13 +338,7 @@ fn build_logs_request(service_name: &str, message: &str) -> ExportLogsServiceReq observed_time_unix_nano: nanos, severity_number: SeverityNumber::Info as i32, severity_text: "INFO".to_string(), - body: Some(AnyValue { - value: Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - message.to_string(), - ), - ), - }), + body: Some(AnyValue { value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(message.to_string())) }), ..Default::default() }], schema_url: String::new(), @@ -423,11 +354,7 @@ fn extract_messages(batch: &ExportLogsServiceRequest) -> Vec { for scope_logs in &resource_logs.scope_logs { for record in &scope_logs.log_records { if let Some(body) = &record.body - && let Some( - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( - value, - ), - ) = &body.value + && let Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value)) = &body.value { messages.push(value.clone()); } diff --git a/src/codec.rs b/src/codec.rs index addafca..aeda447 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -35,8 +35,7 @@ impl Codec { output.extend_from_slice(input); } Self::Lz4 => { - let decoded = lz4_flex::block::decompress(input, expected_len) - .map_err(|err| Error::Codec(err.to_string()))?; + let decoded = lz4_flex::block::decompress(input, expected_len).map_err(|err| Error::Codec(err.to_string()))?; output.extend_from_slice(&decoded); } } diff --git a/src/error.rs b/src/error.rs index 167fd35..2053dfe 100644 --- a/src/error.rs +++ b/src/error.rs @@ -8,27 +8,14 @@ pub enum Error { UnknownCodec(u8), UnknownVersion(u8), HeaderTooShort(u16), - HeaderCrcMismatch { - expected: u32, - actual: u32, - }, - BlockCrcMismatch { - expected: u32, - actual: u32, - }, - LengthTooLarge { - field: &'static str, - value: u64, - limit: usize, - }, + HeaderCrcMismatch { expected: u32, actual: u32 }, + BlockCrcMismatch { expected: u32, actual: u32 }, + LengthTooLarge { field: &'static str, value: u64, limit: usize }, InvalidHeader(&'static str), Truncated(&'static str), VarintTooLong, NumericOverflow(&'static str), - RecordTooLarge { - encoded_len: usize, - block_target_size: usize, - }, + RecordTooLarge { encoded_len: usize, block_target_size: usize }, Codec(String), } @@ -43,35 +30,21 @@ impl Display for Error { Self::UnknownVersion(value) => write!(f, "unknown version: {value}"), Self::HeaderTooShort(value) => write!(f, "header too short: {value}"), Self::HeaderCrcMismatch { expected, actual } => { - write!( - f, - "header crc mismatch: expected {expected:#010x}, got {actual:#010x}" - ) + write!(f, "header crc mismatch: expected {expected:#010x}, got {actual:#010x}") } Self::BlockCrcMismatch { expected, actual } => { - write!( - f, - "block crc mismatch: expected {expected:#010x}, got {actual:#010x}" - ) + write!(f, "block crc mismatch: expected {expected:#010x}, got {actual:#010x}") } - Self::LengthTooLarge { - field, - value, - limit, - } => { + Self::LengthTooLarge { field, value, limit } => { write!(f, "{field} too large: {value} > {limit}") } Self::InvalidHeader(msg) => write!(f, "invalid header: {msg}"), Self::Truncated(msg) => write!(f, "truncated data: {msg}"), Self::VarintTooLong => write!(f, "varint too long"), Self::NumericOverflow(field) => write!(f, "numeric overflow: {field}"), - Self::RecordTooLarge { - encoded_len, - block_target_size, - } => write!( - f, - "record too large for block target: {encoded_len} > {block_target_size}" - ), + Self::RecordTooLarge { encoded_len, block_target_size } => { + write!(f, "record too large for block target: {encoded_len} > {block_target_size}") + } Self::Codec(msg) => write!(f, "codec error: {msg}"), } } diff --git a/src/format.rs b/src/format.rs index 34762c7..ab308c3 100644 --- a/src/format.rs +++ b/src/format.rs @@ -86,17 +86,7 @@ impl BlockHeader { let record_count = u32::from_le_bytes([bytes[16], bytes[17], bytes[18], bytes[19]]); let header_crc32c = u32::from_le_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]); - Ok(Self { - version, - codec, - flags, - header_len, - reserved, - uncompressed_len, - compressed_len, - record_count, - header_crc32c, - }) + Ok(Self { version, codec, flags, header_len, reserved, uncompressed_len, compressed_len, record_count, header_crc32c }) } pub fn compute_header_crc(&self, ext_bytes: &[u8]) -> u32 { @@ -119,13 +109,8 @@ impl BlockHeaderExt { } Ok(Self { - base_seq: u64::from_le_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]), - base_ts_unix_ns: u64::from_le_bytes([ - bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], - bytes[15], - ]), + base_seq: u64::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]]), + base_ts_unix_ns: u64::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]]), }) } } diff --git a/src/lib.rs b/src/lib.rs index d17eaf4..25b50de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,9 +14,8 @@ pub mod writer; pub use codec::Codec; pub use error::{Error, Result}; pub use format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, - BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, - FORMAT_VERSION, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, + DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, FORMAT_VERSION, }; pub use reader::{LogjetReader, ReaderConfig, ReaderStats}; pub use record::{OwnedRecord, Record, RecordType}; diff --git a/src/reader.rs b/src/reader.rs index 57b79bf..7e1f908 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -4,8 +4,7 @@ use std::io::{ErrorKind, Read, Seek, SeekFrom}; use crate::crc::crc32c; use crate::error::{Error, Result}; use crate::format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, - BlockHeaderExt, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, BlockHeaderExt, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYNC_MARKER, }; use crate::record::{OwnedRecord, RecordType}; @@ -18,11 +17,7 @@ pub struct ReaderConfig { impl Default for ReaderConfig { fn default() -> Self { - Self { - sync_marker: DEFAULT_SYNC_MARKER, - max_compressed_len: DEFAULT_MAX_BLOCK_SIZE, - max_uncompressed_len: DEFAULT_MAX_BLOCK_SIZE, - } + Self { sync_marker: DEFAULT_SYNC_MARKER, max_compressed_len: DEFAULT_MAX_BLOCK_SIZE, max_uncompressed_len: DEFAULT_MAX_BLOCK_SIZE } } } @@ -52,22 +47,13 @@ impl LogjetReader { } pub fn with_config(inner: R, config: ReaderConfig) -> Self { - Self { - inner, - config, - stats: ReaderStats::default(), - pending: VecDeque::new(), - } + Self { inner, config, stats: ReaderStats::default(), pending: VecDeque::new() } } pub fn next_record(&mut self) -> Result> { loop { if let Some(record) = self.pending.pop_front() { - self.stats.records_ok = self - .stats - .records_ok - .checked_add(1) - .ok_or(Error::NumericOverflow("records_ok"))?; + self.stats.records_ok = self.stats.records_ok.checked_add(1).ok_or(Error::NumericOverflow("records_ok"))?; return Ok(Some(record)); } @@ -83,22 +69,15 @@ impl LogjetReader { loop { let Some(sync_start) = self.find_next_sync()? else { let eof = self.inner.seek(SeekFrom::End(0))?; - self.stats.bytes_skipped = self - .stats - .bytes_skipped - .checked_add(eof.saturating_sub(scan_origin)) - .ok_or(Error::NumericOverflow("bytes_skipped"))?; + self.stats.bytes_skipped = + self.stats.bytes_skipped.checked_add(eof.saturating_sub(scan_origin)).ok_or(Error::NumericOverflow("bytes_skipped"))?; self.inner.seek(SeekFrom::Start(eof))?; return Ok(false); }; match self.try_read_block_at(sync_start) { Ok(records) => { - self.stats.blocks_ok = self - .stats - .blocks_ok - .checked_add(1) - .ok_or(Error::NumericOverflow("blocks_ok"))?; + self.stats.blocks_ok = self.stats.blocks_ok.checked_add(1).ok_or(Error::NumericOverflow("blocks_ok"))?; self.stats.bytes_skipped = self .stats .bytes_skipped @@ -108,20 +87,12 @@ impl LogjetReader { return Ok(true); } Err(Error::Io(err)) if err.kind() == ErrorKind::UnexpectedEof => { - self.stats.blocks_bad = self - .stats - .blocks_bad - .checked_add(1) - .ok_or(Error::NumericOverflow("blocks_bad"))?; + self.stats.blocks_bad = self.stats.blocks_bad.checked_add(1).ok_or(Error::NumericOverflow("blocks_bad"))?; self.inner.seek(SeekFrom::Start(sync_start + 1))?; } Err(Error::Io(err)) => return Err(Error::Io(err)), Err(_) => { - self.stats.blocks_bad = self - .stats - .blocks_bad - .checked_add(1) - .ok_or(Error::NumericOverflow("blocks_bad"))?; + self.stats.blocks_bad = self.stats.blocks_bad.checked_add(1).ok_or(Error::NumericOverflow("blocks_bad"))?; self.inner.seek(SeekFrom::Start(sync_start + 1))?; } } @@ -176,32 +147,19 @@ impl LogjetReader { return Err(Error::HeaderTooShort(header.header_len)); } let header_len = usize::from(header.header_len); - let ext_len = header_len - .checked_sub(BLOCK_HEADER_FIXED_LEN) - .ok_or(Error::InvalidHeader("header length underflow"))?; + let ext_len = header_len.checked_sub(BLOCK_HEADER_FIXED_LEN).ok_or(Error::InvalidHeader("header length underflow"))?; if ext_len < BLOCK_HEADER_EXT_LEN { return Err(Error::InvalidHeader("missing required header extension")); } - validate_length( - "compressed_len", - header.compressed_len as usize, - self.config.max_compressed_len, - )?; - validate_length( - "uncompressed_len", - header.uncompressed_len as usize, - self.config.max_uncompressed_len, - )?; + validate_length("compressed_len", header.compressed_len as usize, self.config.max_compressed_len)?; + validate_length("uncompressed_len", header.uncompressed_len as usize, self.config.max_uncompressed_len)?; let mut ext_bytes = vec![0u8; ext_len]; self.inner.read_exact(&mut ext_bytes)?; let expected_header_crc = header.compute_header_crc(&ext_bytes); if expected_header_crc != header.header_crc32c { - return Err(Error::HeaderCrcMismatch { - expected: header.header_crc32c, - actual: expected_header_crc, - }); + return Err(Error::HeaderCrcMismatch { expected: header.header_crc32c, actual: expected_header_crc }); } let ext = BlockHeaderExt::decode(&ext_bytes[..BLOCK_HEADER_EXT_LEN])?; @@ -212,23 +170,17 @@ impl LogjetReader { self.inner.read_exact(&mut block_crc_bytes)?; let expected_block_crc = u32::from_le_bytes(block_crc_bytes); - let mut crc_bytes = - Vec::with_capacity(BLOCK_HEADER_FIXED_LEN + ext_bytes.len() + compressed.len()); + let mut crc_bytes = Vec::with_capacity(BLOCK_HEADER_FIXED_LEN + ext_bytes.len() + compressed.len()); crc_bytes.extend_from_slice(&fixed); crc_bytes.extend_from_slice(&ext_bytes); crc_bytes.extend_from_slice(&compressed); let actual_block_crc = crc32c(&crc_bytes); if expected_block_crc != actual_block_crc { - return Err(Error::BlockCrcMismatch { - expected: expected_block_crc, - actual: actual_block_crc, - }); + return Err(Error::BlockCrcMismatch { expected: expected_block_crc, actual: actual_block_crc }); } let mut payload = Vec::with_capacity(header.uncompressed_len as usize); - header - .codec - .decompress(&compressed, header.uncompressed_len as usize, &mut payload)?; + header.codec.decompress(&compressed, header.uncompressed_len as usize, &mut payload)?; let records = parse_records(&payload, header.record_count, ext)?; Ok(records) @@ -237,20 +189,12 @@ impl LogjetReader { fn validate_length(field: &'static str, value: usize, limit: usize) -> Result<()> { if value > limit { - return Err(Error::LengthTooLarge { - field, - value: value as u64, - limit, - }); + return Err(Error::LengthTooLarge { field, value: value as u64, limit }); } Ok(()) } -fn parse_records( - payload: &[u8], - record_count: u32, - ext: BlockHeaderExt, -) -> Result> { +fn parse_records(payload: &[u8], record_count: u32, ext: BlockHeaderExt) -> Result> { let mut records = VecDeque::with_capacity(record_count as usize); let mut cursor = 0usize; @@ -264,26 +208,17 @@ fn parse_records( let seq_delta = decode_varint(payload, &mut cursor)?; let ts_delta = decode_varint(payload, &mut cursor)?; let payload_len = decode_varint(payload, &mut cursor)?; - let payload_len = - usize::try_from(payload_len).map_err(|_| Error::NumericOverflow("payload_len"))?; + let payload_len = usize::try_from(payload_len).map_err(|_| Error::NumericOverflow("payload_len"))?; - let end = cursor - .checked_add(payload_len) - .ok_or(Error::NumericOverflow("record payload end"))?; + let end = cursor.checked_add(payload_len).ok_or(Error::NumericOverflow("record payload end"))?; if end > payload.len() { return Err(Error::Truncated("record payload")); } records.push_back(OwnedRecord { record_type, - seq: ext - .base_seq - .checked_add(seq_delta) - .ok_or(Error::NumericOverflow("record seq"))?, - ts_unix_ns: ext - .base_ts_unix_ns - .checked_add(ts_delta) - .ok_or(Error::NumericOverflow("record ts"))?, + seq: ext.base_seq.checked_add(seq_delta).ok_or(Error::NumericOverflow("record seq"))?, + ts_unix_ns: ext.base_ts_unix_ns.checked_add(ts_delta).ok_or(Error::NumericOverflow("record ts"))?, payload: payload[cursor..end].to_vec(), }); cursor = end; diff --git a/src/record.rs b/src/record.rs index 5191057..e78d1b3 100644 --- a/src/record.rs +++ b/src/record.rs @@ -37,22 +37,12 @@ pub struct OwnedRecord { impl OwnedRecord { pub fn as_record(&self) -> Record<'_> { - Record { - record_type: self.record_type, - seq: self.seq, - ts_unix_ns: self.ts_unix_ns, - payload: &self.payload, - } + Record { record_type: self.record_type, seq: self.seq, ts_unix_ns: self.ts_unix_ns, payload: &self.payload } } } impl<'a> From> for OwnedRecord { fn from(value: Record<'a>) -> Self { - Self { - record_type: value.record_type, - seq: value.seq, - ts_unix_ns: value.ts_unix_ns, - payload: value.payload.to_vec(), - } + Self { record_type: value.record_type, seq: value.seq, ts_unix_ns: value.ts_unix_ns, payload: value.payload.to_vec() } } } diff --git a/src/writer.rs b/src/writer.rs index 8106e22..af1a9fe 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -4,8 +4,8 @@ use crate::codec::Codec; use crate::crc::crc32c; use crate::error::{Error, Result}; use crate::format::{ - BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, - BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, DEFAULT_SYNC_MARKER, FORMAT_VERSION, + BLOCK_HEADER_EXT_LEN, BLOCK_HEADER_FIXED_LEN, BLOCK_HEADER_TOTAL_LEN, BlockHeader, BlockHeaderExt, DEFAULT_BLOCK_TARGET_SIZE, + DEFAULT_SYNC_MARKER, FORMAT_VERSION, }; use crate::record::RecordType; @@ -18,11 +18,7 @@ pub struct WriterConfig { impl Default for WriterConfig { fn default() -> Self { - Self { - block_target_size: DEFAULT_BLOCK_TARGET_SIZE, - codec: Codec::Lz4, - sync_marker: DEFAULT_SYNC_MARKER, - } + Self { block_target_size: DEFAULT_BLOCK_TARGET_SIZE, codec: Codec::Lz4, sync_marker: DEFAULT_SYNC_MARKER } } } @@ -58,13 +54,7 @@ impl LogjetWriter { } } - pub fn push( - &mut self, - record_type: RecordType, - seq: u64, - ts_unix_ns: u64, - payload: &[u8], - ) -> Result<()> { + pub fn push(&mut self, record_type: RecordType, seq: u64, ts_unix_ns: u64, payload: &[u8]) -> Result<()> { self.encoded_record_buf.clear(); let (base_seq, base_ts) = match (self.block_base_seq, self.block_base_ts_unix_ns) { @@ -76,27 +66,16 @@ impl LogjetWriter { } }; - let seq_delta = seq.checked_sub(base_seq).ok_or(Error::InvalidHeader( - "sequence must be monotonic within block", - ))?; - let ts_delta = ts_unix_ns.checked_sub(base_ts).ok_or(Error::InvalidHeader( - "timestamp must be monotonic within block", - ))?; + let seq_delta = seq.checked_sub(base_seq).ok_or(Error::InvalidHeader("sequence must be monotonic within block"))?; + let ts_delta = ts_unix_ns.checked_sub(base_ts).ok_or(Error::InvalidHeader("timestamp must be monotonic within block"))?; self.encoded_record_buf.push(record_type as u8); encode_varint(seq_delta, &mut self.encoded_record_buf)?; encode_varint(ts_delta, &mut self.encoded_record_buf)?; - encode_varint( - u64::try_from(payload.len()).map_err(|_| Error::NumericOverflow("payload len"))?, - &mut self.encoded_record_buf, - )?; + encode_varint(u64::try_from(payload.len()).map_err(|_| Error::NumericOverflow("payload len"))?, &mut self.encoded_record_buf)?; self.encoded_record_buf.extend_from_slice(payload); - let projected = self - .payload_buf - .len() - .checked_add(self.encoded_record_buf.len()) - .ok_or(Error::NumericOverflow("payload buf growth"))?; + let projected = self.payload_buf.len().checked_add(self.encoded_record_buf.len()).ok_or(Error::NumericOverflow("payload buf growth"))?; if !self.payload_buf.is_empty() && projected > self.config.block_target_size { self.flush_block()?; self.block_base_seq = Some(seq); @@ -105,18 +84,12 @@ impl LogjetWriter { self.encoded_record_buf.push(record_type as u8); encode_varint(0, &mut self.encoded_record_buf)?; encode_varint(0, &mut self.encoded_record_buf)?; - encode_varint( - u64::try_from(payload.len()).map_err(|_| Error::NumericOverflow("payload len"))?, - &mut self.encoded_record_buf, - )?; + encode_varint(u64::try_from(payload.len()).map_err(|_| Error::NumericOverflow("payload len"))?, &mut self.encoded_record_buf)?; self.encoded_record_buf.extend_from_slice(payload); } self.payload_buf.extend_from_slice(&self.encoded_record_buf); - self.record_count = self - .record_count - .checked_add(1) - .ok_or(Error::NumericOverflow("record_count"))?; + self.record_count = self.record_count.checked_add(1).ok_or(Error::NumericOverflow("record_count"))?; if self.payload_buf.len() >= self.config.block_target_size { self.flush_block()?; @@ -130,34 +103,23 @@ impl LogjetWriter { return Ok(()); } - let base_seq = self - .block_base_seq - .ok_or(Error::InvalidHeader("missing block base seq"))?; - let base_ts = self - .block_base_ts_unix_ns - .ok_or(Error::InvalidHeader("missing block base ts"))?; - - self.config - .codec - .compress(&self.payload_buf, &mut self.compressed_buf)?; - - let uncompressed_len = - u32::try_from(self.payload_buf.len()).map_err(|_| Error::LengthTooLarge { - field: "uncompressed_len", - value: self.payload_buf.len() as u64, - limit: u32::MAX as usize, - })?; - let compressed_len = - u32::try_from(self.compressed_buf.len()).map_err(|_| Error::LengthTooLarge { - field: "compressed_len", - value: self.compressed_buf.len() as u64, - limit: u32::MAX as usize, - })?; - - let header_ext = BlockHeaderExt { - base_seq, - base_ts_unix_ns: base_ts, - }; + let base_seq = self.block_base_seq.ok_or(Error::InvalidHeader("missing block base seq"))?; + let base_ts = self.block_base_ts_unix_ns.ok_or(Error::InvalidHeader("missing block base ts"))?; + + self.config.codec.compress(&self.payload_buf, &mut self.compressed_buf)?; + + let uncompressed_len = u32::try_from(self.payload_buf.len()).map_err(|_| Error::LengthTooLarge { + field: "uncompressed_len", + value: self.payload_buf.len() as u64, + limit: u32::MAX as usize, + })?; + let compressed_len = u32::try_from(self.compressed_buf.len()).map_err(|_| Error::LengthTooLarge { + field: "compressed_len", + value: self.compressed_buf.len() as u64, + limit: u32::MAX as usize, + })?; + + let header_ext = BlockHeaderExt { base_seq, base_ts_unix_ns: base_ts }; let mut ext_bytes = Vec::with_capacity(BLOCK_HEADER_EXT_LEN); header_ext.encode(&mut ext_bytes); @@ -165,8 +127,7 @@ impl LogjetWriter { version: FORMAT_VERSION, codec: self.config.codec, flags: 0, - header_len: u16::try_from(BLOCK_HEADER_TOTAL_LEN) - .map_err(|_| Error::NumericOverflow("header_len"))?, + header_len: u16::try_from(BLOCK_HEADER_TOTAL_LEN).map_err(|_| Error::NumericOverflow("header_len"))?, reserved: 0, uncompressed_len, compressed_len, diff --git a/tests/logjet.rs b/tests/logjet.rs index 3a4cfc6..3b6cd0a 100644 --- a/tests/logjet.rs +++ b/tests/logjet.rs @@ -32,26 +32,14 @@ fn read_all(bytes: Vec, config: ReaderConfig) -> ReadAllOutput { let mut reader = LogjetReader::with_config(Cursor::new(bytes), config); let mut out = Vec::new(); while let Some(record) = reader.next_record().unwrap() { - out.push(( - record.record_type, - record.seq, - record.ts_unix_ns, - record.payload, - )); + out.push((record.record_type, record.seq, record.ts_unix_ns, record.payload)); } (out, reader.stats()) } #[test] fn write_one_block_read_back() { - let bytes = write_records( - WriterConfig { - block_target_size: 1024, - codec: Codec::Lz4, - sync_marker: DEFAULT_SYNC_MARKER, - }, - 3, - ); + let bytes = write_records(WriterConfig { block_target_size: 1024, codec: Codec::Lz4, sync_marker: DEFAULT_SYNC_MARKER }, 3); let (records, stats) = read_all(bytes, ReaderConfig::default()); assert_eq!(records.len(), 3); @@ -64,14 +52,7 @@ fn write_one_block_read_back() { #[test] fn write_many_blocks_read_all() { - let bytes = write_records( - WriterConfig { - block_target_size: 48, - codec: Codec::None, - sync_marker: DEFAULT_SYNC_MARKER, - }, - 20, - ); + let bytes = write_records(WriterConfig { block_target_size: 48, codec: Codec::None, sync_marker: DEFAULT_SYNC_MARKER }, 20); let (records, stats) = read_all(bytes, ReaderConfig::default()); assert_eq!(records.len(), 20); @@ -82,20 +63,10 @@ fn write_many_blocks_read_all() { #[test] fn corrupt_middle_block_and_recover() { - let mut bytes = write_records( - WriterConfig { - block_target_size: 48, - codec: Codec::None, - sync_marker: DEFAULT_SYNC_MARKER, - }, - 12, - ); + let mut bytes = write_records(WriterConfig { block_target_size: 48, codec: Codec::None, sync_marker: DEFAULT_SYNC_MARKER }, 12); - let sync_positions: Vec = bytes - .windows(DEFAULT_SYNC_MARKER.len()) - .enumerate() - .filter_map(|(idx, window)| (window == DEFAULT_SYNC_MARKER).then_some(idx)) - .collect(); + let sync_positions: Vec = + bytes.windows(DEFAULT_SYNC_MARKER.len()).enumerate().filter_map(|(idx, window)| (window == DEFAULT_SYNC_MARKER).then_some(idx)).collect(); assert!(sync_positions.len() >= 3); let second = sync_positions[1]; @@ -132,14 +103,7 @@ fn oversized_lengths_rejected_safely() { let mut bytes = write_records(WriterConfig::default(), 1); let offset = DEFAULT_SYNC_MARKER.len() + 12; bytes[offset..offset + 4].copy_from_slice(&(32_u32 * 1024 * 1024).to_le_bytes()); - let (_, stats) = read_all( - bytes, - ReaderConfig { - max_compressed_len: 1024 * 1024, - max_uncompressed_len: 1024 * 1024, - ..ReaderConfig::default() - }, - ); + let (_, stats) = read_all(bytes, ReaderConfig { max_compressed_len: 1024 * 1024, max_uncompressed_len: 1024 * 1024, ..ReaderConfig::default() }); assert_eq!(stats.blocks_ok, 0); assert_eq!(stats.blocks_bad, 1); @@ -150,22 +114,15 @@ fn single_huge_record_stored_in_own_block() { let payload = vec![7u8; 8192]; let mut writer = LogjetWriter::with_config( Cursor::new(Vec::new()), - WriterConfig { - block_target_size: 128, - codec: Codec::Lz4, - sync_marker: DEFAULT_SYNC_MARKER, - }, + WriterConfig { block_target_size: 128, codec: Codec::Lz4, sync_marker: DEFAULT_SYNC_MARKER }, ); writer.push(RecordType::Logs, 1, 10, b"small").unwrap(); writer.push(RecordType::Logs, 2, 20, &payload).unwrap(); writer.push(RecordType::Logs, 3, 30, b"tail").unwrap(); let bytes = writer.into_inner().unwrap().into_inner(); - let sync_positions: Vec = bytes - .windows(DEFAULT_SYNC_MARKER.len()) - .enumerate() - .filter_map(|(idx, window)| (window == DEFAULT_SYNC_MARKER).then_some(idx)) - .collect(); + let sync_positions: Vec = + bytes.windows(DEFAULT_SYNC_MARKER.len()).enumerate().filter_map(|(idx, window)| (window == DEFAULT_SYNC_MARKER).then_some(idx)).collect(); assert_eq!(sync_positions.len(), 3); let (records, _) = read_all(bytes, ReaderConfig::default()); @@ -177,11 +134,7 @@ fn single_huge_record_stored_in_own_block() { fn seq_and_timestamp_deltas_round_trip() { let mut writer = LogjetWriter::with_config( Cursor::new(Vec::new()), - WriterConfig { - block_target_size: 1024, - codec: Codec::None, - sync_marker: DEFAULT_SYNC_MARKER, - }, + WriterConfig { block_target_size: 1024, codec: Codec::None, sync_marker: DEFAULT_SYNC_MARKER }, ); writer.push(RecordType::Logs, 42, 1_000, b"a").unwrap(); writer.push(RecordType::Metrics, 50, 1_250, b"b").unwrap(); From 1d052bdafb2550e3720bbc4df7e8e8f39951b50d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 16:04:09 +0100 Subject: [PATCH 11/17] Enforce clippy with 1.94 --- rust-toolchain.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 76a06e6..95e32ea 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,3 @@ [toolchain] channel = "1.94.0" +components = ["clippy"] From a39560cbd09e4b5f17e47fef727161b420b1c701 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:46:06 +0100 Subject: [PATCH 12/17] Add TUI viewer, kill the cat. :-) --- ljx/Cargo.toml | 6 + ljx/src/cli.rs | 20 +- ljx/src/commands/cat.rs | 8 - ljx/src/commands/mod.rs | 2 +- ljx/src/commands/view.rs | 1683 ++++++++++++++++++++++++++++++++++++++ ljx/src/main.rs | 2 +- ljx/src/predicate.rs | 79 +- 7 files changed, 1783 insertions(+), 17 deletions(-) delete mode 100644 ljx/src/commands/cat.rs create mode 100644 ljx/src/commands/view.rs diff --git a/ljx/Cargo.toml b/ljx/Cargo.toml index 3fb51fc..1dacc7c 100644 --- a/ljx/Cargo.toml +++ b/ljx/Cargo.toml @@ -6,6 +6,12 @@ license = "Apache-2.0" [dependencies] clap = { version = "4.5", features = ["derive", "wrap_help"] } +chrono = { version = "0.4", default-features = false, features = ["std"] } colored = "3" +crossterm = "0.28" logjet = { path = ".." } +opentelemetry-proto = { version = "0.28", features = ["gen-tonic", "logs"] } +prost = "0.13" +ratatui = "0.29" regex = "1.12" +shlex = "1.3" diff --git a/ljx/src/cli.rs b/ljx/src/cli.rs index c8e7418..50f86ea 100644 --- a/ljx/src/cli.rs +++ b/ljx/src/cli.rs @@ -37,18 +37,28 @@ pub fn build_cli() -> clap::Command { "{appname} [OPTIONS] [ARGS]" )) .after_help( - "Examples:\n ljx count telemetry.logjet -F error -i\n ljx filter telemetry.logjet -o only-logs.logjet -e 'java\\..*\\.bs'", + "Examples:\n ljx count telemetry.logjet -F error -i\n ljx filter telemetry.logjet -o only-logs.logjet -e 'java\\..*\\.bs'\n ljx view telemetry.logjet", ) } #[derive(Debug, Subcommand)] pub enum Command { + #[command(about = "Split one .logjet input into multiple outputs")] Split(SplitArgs), + #[command(about = "Join multiple .logjet inputs into one ordered output")] Join(JoinArgs), + #[command(about = "Filter records into a new .logjet stream")] Filter(FilterArgs), + #[command(about = "Count records matching a predicate")] Count(CountArgs), + #[command(about = "Compute summary statistics for one .logjet file")] Stats(StatsArgs), - Cat(CatArgs), + #[command( + name = "view", + alias = "cat", + about = "Interactively browse filtered records in a terminal UI" + )] + View(ViewArgs), } #[derive(Debug, Clone, Args)] @@ -127,11 +137,11 @@ pub struct StatsArgs { } #[derive(Debug, Clone, Args)] -pub struct CatArgs { - #[arg(value_name = "INPUT")] +pub struct ViewArgs { + #[arg(value_name = "INPUT", help = "Input .logjet file or - for stdin")] pub input: PathBuf, - #[arg(long, default_value_t = false)] + #[arg(long, default_value_t = false, help = "Show payload previews in hex")] pub hex_payload: bool, } diff --git a/ljx/src/commands/cat.rs b/ljx/src/commands/cat.rs deleted file mode 100644 index 48e0450..0000000 --- a/ljx/src/commands/cat.rs +++ /dev/null @@ -1,8 +0,0 @@ -use crate::cli::CatArgs; -use crate::error::{Error, Result}; - -pub fn run(_args: CatArgs) -> Result<()> { - Err(Error::Unimplemented( - "cat is not implemented yet; the CLI shape is defined but record rendering still needs to be added", - )) -} diff --git a/ljx/src/commands/mod.rs b/ljx/src/commands/mod.rs index c4b9f75..6482a9c 100644 --- a/ljx/src/commands/mod.rs +++ b/ljx/src/commands/mod.rs @@ -1,6 +1,6 @@ -pub mod cat; pub mod count; pub mod filter; pub mod join; pub mod split; pub mod stats; +pub mod view; diff --git a/ljx/src/commands/view.rs b/ljx/src/commands/view.rs new file mode 100644 index 0000000..f83fec6 --- /dev/null +++ b/ljx/src/commands/view.rs @@ -0,0 +1,1683 @@ +use std::collections::{HashMap, VecDeque}; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufWriter, IsTerminal, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use chrono::{TimeZone, Utc}; +use crossterm::event::{self, Event, KeyCode, KeyEvent, KeyModifiers}; +use crossterm::execute; +use crossterm::terminal::{ + EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode, +}; +use logjet::{LogjetReader, LogjetWriter, OwnedRecord, RecordType, WriterConfig}; +use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::common::v1::any_value::Value; +use prost::Message; +use ratatui::backend::CrosstermBackend; +use ratatui::layout::{Constraint, Direction, Layout, Rect}; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span, Text}; +use ratatui::widgets::{Block, BorderType, Borders, Clear, Paragraph, Scrollbar, ScrollbarOrientation, ScrollbarState, Wrap}; +use ratatui::{Frame, Terminal}; + +use crate::cli::ViewArgs; +use crate::error::{Error, Result}; +use crate::input::InputHandle; +use crate::predicate::{FilterMode, parse_filter_query}; + +const SUMMARY_CACHE_LIMIT: usize = 256; +const DETAIL_PREVIEW_BYTES: usize = 1024; +const SCAN_BATCH_SIZE: usize = 128; +const TICK_RATE: Duration = Duration::from_millis(100); + +pub fn run(args: ViewArgs) -> Result<()> { + if !io::stdin().is_terminal() || !io::stdout().is_terminal() { + return Err(Error::Usage( + "ljx view needs an interactive terminal; pipe-oriented output belongs in `ljx filter`".to_string(), + )); + } + + let mut stdout = io::stdout(); + enable_raw_mode()?; + execute!(stdout, EnterAlternateScreen)?; + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + let mut app = ViewApp::new(args)?; + app.apply_filter()?; + let outcome = app.run(&mut terminal); + + disable_raw_mode()?; + execute!(terminal.backend_mut(), LeaveAlternateScreen)?; + terminal.show_cursor()?; + + outcome +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Focus { + Search, + List, + Modal, + SavePrompt, + SaveError, +} + +#[derive(Debug, Clone, Copy)] +struct EntryMeta { + offset: u64, + record_type: RecordType, + seq: u64, + ts_unix_ns: u64, + payload_len: u64, +} + +#[derive(Debug, Clone)] +struct DetailRecord { + meta: EntryMeta, + payload: Vec, +} + +#[derive(Debug, Clone)] +enum ScanUpdate { + Batch(Vec), + Finished { + scanned: u64, + matched: u64, + }, + Failed(String), +} + +struct ActiveScan { + rx: Receiver, + cancel: Arc, + spool_path: PathBuf, + spool_reader: File, + scanned: u64, + matched: u64, + finished: bool, +} + +impl ActiveScan { + fn cancel(&self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +struct ViewApp { + input: PathBuf, + hex_payload: bool, + focus: Focus, + filter_mode: FilterMode, + query_input: String, + applied_query: String, + status: String, + entries: Vec, + selected: usize, + list_offset: usize, + modal_scroll: u16, + detail_scroll: u16, + summary_cache: HashMap, + summary_order: VecDeque, + selected_detail: Option, + modal_text: Option, + save_filename: String, + save_message: Option, + current_scan: Option, +} + +impl ViewApp { + fn new(args: ViewArgs) -> Result { + Ok(Self { + input: args.input, + hex_payload: args.hex_payload, + focus: Focus::Search, + filter_mode: FilterMode::Strings, + query_input: String::new(), + applied_query: String::new(), + status: "Type a filter and press Enter to scan matching records".to_string(), + entries: Vec::new(), + selected: 0, + list_offset: 0, + modal_scroll: 0, + detail_scroll: 0, + summary_cache: HashMap::new(), + summary_order: VecDeque::new(), + selected_detail: None, + modal_text: None, + save_filename: String::new(), + save_message: None, + current_scan: None, + }) + } + + fn run(&mut self, terminal: &mut Terminal>) -> Result<()> { + loop { + self.drain_scan_updates()?; + terminal.draw(|frame| self.render(frame))?; + + if event::poll(TICK_RATE)? { + let Event::Key(key) = event::read()? else { + continue; + }; + + if self.handle_key(key)? { + return Ok(()); + } + } + } + } + + fn handle_key(&mut self, key: KeyEvent) -> Result { + if !matches!(self.focus, Focus::Modal | Focus::SavePrompt | Focus::SaveError) + && matches!(key.code, KeyCode::Char('q') | KeyCode::Char('Q')) + { + self.cancel_scan(); + return Ok(true); + } + + match self.focus { + Focus::Modal => self.handle_modal_key(key), + Focus::SavePrompt => self.handle_save_prompt_key(key), + Focus::SaveError => self.handle_save_error_key(), + Focus::Search => self.handle_search_key(key), + Focus::List => self.handle_list_key(key), + } + } + + fn handle_modal_key(&mut self, key: KeyEvent) -> Result { + match key.code { + KeyCode::Esc => { + self.focus = Focus::List; + self.modal_text = None; + self.modal_scroll = 0; + } + KeyCode::Up => { + self.modal_scroll = self.modal_scroll.saturating_sub(1); + } + KeyCode::Down => { + self.modal_scroll = self.modal_scroll.saturating_add(1); + } + KeyCode::PageUp => { + self.modal_scroll = self.modal_scroll.saturating_sub(10); + } + KeyCode::PageDown => { + self.modal_scroll = self.modal_scroll.saturating_add(10); + } + _ => {} + } + + Ok(false) + } + + fn handle_search_key(&mut self, key: KeyEvent) -> Result { + match key.code { + KeyCode::Tab => { + self.focus = Focus::List; + } + KeyCode::Up | KeyCode::Down => { + self.cycle_filter_mode(); + } + KeyCode::Esc => { + self.query_input.clear(); + self.apply_filter()?; + } + KeyCode::Enter => { + self.apply_filter()?; + } + KeyCode::Backspace => { + self.query_input.pop(); + } + KeyCode::Char('u') if key.modifiers.contains(KeyModifiers::CONTROL) => { + self.query_input.clear(); + } + KeyCode::Char(ch) if !key.modifiers.intersects(KeyModifiers::CONTROL | KeyModifiers::ALT) => { + self.query_input.push(ch); + } + _ => {} + } + + Ok(false) + } + + fn handle_save_prompt_key(&mut self, key: KeyEvent) -> Result { + match key.code { + KeyCode::Esc => { + self.focus = Focus::List; + self.save_message = None; + } + KeyCode::Enter => { + self.save_current_results()?; + } + KeyCode::Backspace => { + self.save_filename.pop(); + } + KeyCode::Char('u') if key.modifiers.contains(KeyModifiers::CONTROL) => { + self.save_filename.clear(); + } + KeyCode::Char(ch) if !key.modifiers.intersects(KeyModifiers::CONTROL | KeyModifiers::ALT) => { + self.save_filename.push(ch); + } + _ => {} + } + Ok(false) + } + + fn handle_save_error_key(&mut self) -> Result { + self.focus = Focus::SavePrompt; + self.save_message = None; + Ok(false) + } + + fn handle_list_key(&mut self, key: KeyEvent) -> Result { + match key.code { + KeyCode::Tab => { + self.focus = Focus::Search; + } + KeyCode::Char('s') | KeyCode::Char('S') => { + self.open_save_prompt()?; + } + KeyCode::Up => { + self.move_selection(-1)?; + } + KeyCode::Down => { + self.move_selection(1)?; + } + KeyCode::PageUp => { + self.move_selection(-10)?; + } + KeyCode::PageDown => { + self.move_selection(10)?; + } + KeyCode::Home => { + self.selected = 0; + self.list_offset = 0; + self.refresh_selected_detail()?; + } + KeyCode::End => { + if !self.entries.is_empty() { + self.selected = self.entries.len() - 1; + self.refresh_selected_detail()?; + } + } + KeyCode::Enter => { + self.open_modal()?; + } + _ => {} + } + + Ok(false) + } + + fn move_selection(&mut self, delta: isize) -> Result<()> { + if self.entries.is_empty() { + return Ok(()); + } + + let max = self.entries.len().saturating_sub(1) as isize; + let next = (self.selected as isize + delta).clamp(0, max) as usize; + if next != self.selected { + self.selected = next; + self.refresh_selected_detail()?; + } + + Ok(()) + } + + fn apply_filter(&mut self) -> Result<()> { + if let Some(scan) = self.current_scan.take() { + scan.cancel(); + drop(scan.spool_reader); + let _ = std::fs::remove_file(scan.spool_path); + } + self.entries.clear(); + self.summary_cache.clear(); + self.summary_order.clear(); + self.selected = 0; + self.list_offset = 0; + self.modal_scroll = 0; + self.detail_scroll = 0; + self.selected_detail = None; + self.modal_text = None; + self.applied_query = self.query_input.clone(); + self.focus = Focus::List; + let predicate = parse_filter_query(&self.applied_query, self.filter_mode)?; + + let spool_path = create_temp_path()?; + let spool_reader = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&spool_path)?; + let spool_writer = spool_reader.try_clone()?; + let cancel = Arc::new(AtomicBool::new(false)); + let cancel_worker = Arc::clone(&cancel); + let input = self.input.clone(); + let (tx, rx) = mpsc::channel(); + let tx_worker = tx.clone(); + + thread::spawn(move || { + let result = scan_matches(input.as_path(), predicate, spool_writer, cancel_worker, tx_worker.clone()); + match result { + Ok((scanned, matched)) => { + let _ = tx_worker.send(ScanUpdate::Finished { scanned, matched }); + } + Err(err) => { + let _ = tx_worker.send(ScanUpdate::Failed(err.to_string())); + } + } + }); + + self.status = format!("Scanning matches for {:?}", self.applied_query); + self.current_scan = Some(ActiveScan { + rx, + cancel, + spool_path, + spool_reader, + scanned: 0, + matched: 0, + finished: false, + }); + + Ok(()) + } + + fn cycle_filter_mode(&mut self) { + self.filter_mode = match self.filter_mode { + FilterMode::Strings => FilterMode::Regex, + FilterMode::Regex => FilterMode::Strings, + }; + } + + fn open_save_prompt(&mut self) -> Result<()> { + let Some(scan) = &self.current_scan else { + self.status = "No active scan to save.".to_string(); + return Ok(()); + }; + if !scan.finished { + self.status = "Wait for the scan to finish before saving.".to_string(); + return Ok(()); + } + self.save_message = None; + if self.save_filename.is_empty() { + self.save_filename = "filtered.logjet".to_string(); + } + self.focus = Focus::SavePrompt; + Ok(()) + } + + fn save_current_results(&mut self) -> Result<()> { + let filename = self.save_filename.trim(); + if filename.is_empty() { + self.save_message = Some("Filename must not be empty.".to_string()); + return Ok(()); + } + if filename.contains('/') { + self.save_message = Some("Filename must not contain path separators.".to_string()); + return Ok(()); + } + if self.input == Path::new("-") { + self.save_message = Some("Cannot infer output directory when input is stdin.".to_string()); + return Ok(()); + } + + let Some(scan) = &mut self.current_scan else { + self.save_message = Some("No scan data to save.".to_string()); + return Ok(()); + }; + let output_dir = self + .input + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let output_path = output_dir.join(filename); + if output_path == self.input || output_path.exists() { + self.save_message = Some(format!("File {filename} already exist")); + self.focus = Focus::SaveError; + return Ok(()); + } + + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&output_path)?; + let writer = BufWriter::new(file); + let mut logjet = LogjetWriter::with_config(writer, WriterConfig::default()); + for meta in &self.entries { + let detail = read_spool_record(&mut scan.spool_reader, *meta)?; + logjet.push( + detail.meta.record_type, + detail.meta.seq, + detail.meta.ts_unix_ns, + &detail.payload, + )?; + } + let mut writer = logjet.into_inner()?; + writer.flush()?; + + self.focus = Focus::List; + self.save_message = None; + self.status = format!("Saved {} records to {}", self.entries.len(), output_path.display()); + Ok(()) + } + + fn drain_scan_updates(&mut self) -> Result<()> { + let Some(scan) = &self.current_scan else { + return Ok(()); + }; + + let mut updates = Vec::new(); + while let Ok(update) = scan.rx.try_recv() { + updates.push(update); + } + + let mut finished = false; + let mut should_refresh_selection = false; + let mut status_override = None; + { + let Some(scan) = &mut self.current_scan else { + return Ok(()); + }; + for update in updates { + match update { + ScanUpdate::Batch(batch) => { + self.entries.extend(batch); + scan.matched = self.entries.len() as u64; + if self.selected_detail.is_none() && !self.entries.is_empty() { + should_refresh_selection = true; + } + } + ScanUpdate::Finished { scanned, matched } => { + scan.scanned = scanned; + scan.matched = matched; + scan.finished = true; + finished = true; + status_override = + Some(format!("Scan complete: {matched} matches out of {scanned} records")); + } + ScanUpdate::Failed(message) => { + scan.finished = true; + finished = true; + status_override = Some(format!("Scan failed: {message}")); + } + } + } + } + + if should_refresh_selection { + self.refresh_selected_detail()?; + } + + if let Some(status) = status_override { + self.status = status; + } + + if !finished { + let matched = self.entries.len(); + self.status = if self.applied_query.is_empty() { + format!("Scanning all records: {matched} matches buffered") + } else { + format!("Scanning {:?}: {matched} matches buffered", self.applied_query) + }; + } + + Ok(()) + } + + fn refresh_selected_detail(&mut self) -> Result<()> { + if self.entries.is_empty() { + self.selected_detail = None; + return Ok(()); + } + + let record = self.load_record(self.selected)?; + self.selected_detail = Some(record); + self.detail_scroll = 0; + Ok(()) + } + + fn load_record(&mut self, index: usize) -> Result { + let Some(scan) = &mut self.current_scan else { + return Err(Error::Usage("no active scan".to_string())); + }; + let meta = self.entries[index]; + read_spool_record(&mut scan.spool_reader, meta) + } + + fn summary_for(&mut self, index: usize) -> Result { + if let Some(summary) = self.summary_cache.get(&index) { + return Ok(summary.clone()); + } + + let Some(scan) = &mut self.current_scan else { + return Ok(String::new()); + }; + let meta = self.entries[index]; + let detail = read_spool_record(&mut scan.spool_reader, meta)?; + let summary = format_summary(&detail, self.hex_payload); + remember_summary( + &mut self.summary_cache, + &mut self.summary_order, + index, + summary.clone(), + ); + Ok(summary) + } + + fn open_modal(&mut self) -> Result<()> { + let Some(detail) = &self.selected_detail else { + return Ok(()); + }; + + self.modal_text = Some(render_modal_message(detail, self.hex_payload)); + self.modal_scroll = 0; + self.focus = Focus::Modal; + Ok(()) + } + + fn cancel_scan(&mut self) { + if let Some(scan) = &self.current_scan { + scan.cancel(); + } + } + + fn render(&mut self, frame: &mut Frame<'_>) { + let areas = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(3), + Constraint::Min(10), + Constraint::Length(1), + ]) + .split(frame.area()); + + self.render_search(frame, areas[0]); + + let body = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(64), Constraint::Percentage(36)]) + .split(areas[1]); + + self.render_list(frame, body[0]); + self.render_details(frame, body[1]); + self.render_status(frame, areas[2]); + + if self.focus == Focus::Modal { + self.render_modal(frame); + } else if self.focus == Focus::SaveError { + self.render_save_error(frame); + } else if self.focus == Focus::SavePrompt { + self.render_save_prompt(frame); + } + } + + fn render_search(&self, frame: &mut Frame<'_>, area: Rect) { + let title = match self.filter_mode { + FilterMode::Strings => " Filter (strings) ", + FilterMode::Regex => " Filter (regex) ", + }; + let block = pane_block(title, self.focus == Focus::Search); + let paragraph = Paragraph::new(self.query_input.as_str()) + .block(block) + .style(Style::default().fg(Color::White)); + frame.render_widget(paragraph, area); + + if self.focus == Focus::Search { + let x = area.x.saturating_add(self.query_input.chars().count() as u16 + 1); + let y = area.y.saturating_add(1); + frame.set_cursor_position((x.min(area.right().saturating_sub(1)), y)); + } + } + + fn render_list(&mut self, frame: &mut Frame<'_>, area: Rect) { + let block = pane_block(" Log entries ", self.focus == Focus::List); + let inner = block.inner(area); + frame.render_widget(block, area); + + if inner.height == 0 { + return; + } + + let visible_rows = inner.height as usize; + if self.selected < self.list_offset { + self.list_offset = self.selected; + } else if self.selected >= self.list_offset.saturating_add(visible_rows) && visible_rows > 0 { + self.list_offset = self.selected + 1 - visible_rows; + } + + let mut lines = Vec::with_capacity(visible_rows.max(1)); + if self.entries.is_empty() { + lines.push(Line::from(Span::styled( + "No matches yet. Type a filter, press Enter, then browse the result set.", + Style::default().fg(Color::Gray), + ))); + } else { + let end = (self.list_offset + visible_rows).min(self.entries.len()); + let row_width = inner.width.saturating_sub(1) as usize; + for index in self.list_offset..end { + let style = if index == self.selected { + Style::default() + .fg(Color::Black) + .bg(Color::Indexed(28)) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(Color::White) + }; + + let summary = self + .summary_for(index) + .unwrap_or_else(|_| "".to_string()); + let summary = fit_to_width(&summary, row_width); + lines.push(Line::from(Span::styled(summary, style))); + } + } + + let paragraph = Paragraph::new(Text::from(lines)) + .scroll((0, 0)) + .wrap(Wrap { trim: false }) + .style(Style::default().fg(Color::White)); + frame.render_widget(paragraph, inner); + + if !self.entries.is_empty() { + let mut scrollbar_state = ScrollbarState::new(self.entries.len()) + .position(self.selected.min(self.entries.len().saturating_sub(1))); + frame.render_stateful_widget( + Scrollbar::new(ScrollbarOrientation::VerticalRight), + inner, + &mut scrollbar_state, + ); + } + } + + fn render_details(&self, frame: &mut Frame<'_>, area: Rect) { + let block = pane_block(" Info ", false); + let inner = block.inner(area); + frame.render_widget(block, area); + + let lines = if let Some(detail) = &self.selected_detail { + render_detail_lines(detail, self.hex_payload) + } else { + vec![Line::from("No record selected yet.")] + }; + + let paragraph = Paragraph::new(Text::from(lines)) + .scroll((self.detail_scroll, 0)) + .wrap(Wrap { trim: false }) + .style(Style::default().fg(Color::White)); + frame.render_widget(paragraph, inner); + } + + fn render_status(&self, frame: &mut Frame<'_>, area: Rect) { + if area.width == 0 || area.height == 0 { + return; + } + + let bar_style = Style::default().bg(Color::Indexed(28)); + let buf = frame.buffer_mut(); + buf.set_style(area, bar_style); + let y = area.y; + + if self.focus == Focus::Modal { + draw_status_spans( + buf, + area.x, + y, + area.width, + &[ + status_key("ESC"), + status_text(" to close "), + status_key("UP/DOWN"), + status_text(" scroll"), + ], + ); + return; + } + if self.focus == Focus::SavePrompt { + draw_status_spans( + buf, + area.x, + y, + area.width, + &[ + status_key("ENTER"), + status_text(" save "), + status_key("ESC"), + status_text(" cancel"), + ], + ); + return; + } + if self.focus == Focus::SaveError { + draw_status_spans(buf, area.x, y, area.width, &[status_text("Press any key to return")]); + return; + } + + let left_spans = status_help_spans(self.focus); + let status = trim_single_line(&self.status, area.width as usize); + let status_width = status.chars().count().min(area.width as usize) as u16; + let gap_width = if area.width > status_width { 1 } else { 0 }; + let left_width = area.width.saturating_sub(status_width).saturating_sub(gap_width); + draw_status_spans(buf, area.x, y, left_width, &left_spans); + + if status_width > 0 { + let status_x = area.right().saturating_sub(status_width); + buf.set_stringn( + status_x, + y, + status, + status_width as usize, + Style::default() + .fg(Color::LightGreen) + .bg(Color::Indexed(28)), + ); + } + } + + fn render_save_prompt(&self, frame: &mut Frame<'_>) { + let area = centered_rect(52, 10, frame.area()); + frame.render_widget(Clear, area); + let block = Block::default() + .title(Span::styled( + " Save current content ", + Style::default().fg(Color::Black).bg(Color::Gray).add_modifier(Modifier::BOLD), + )) + .borders(Borders::ALL) + .border_type(BorderType::Plain) + .border_style(Style::default().fg(Color::White).bg(Color::Gray)) + .style(Style::default().fg(Color::Black).bg(Color::Gray)); + let inner = block.inner(area); + frame.render_widget(block, area); + + let label = "Filename: "; + let input_width = inner.width.saturating_sub(label.chars().count() as u16 + 2); + let row = Rect { + x: inner.x, + y: inner.y, + width: inner.width, + height: 1, + }; + frame.render_widget( + Paragraph::new(Line::from(vec![ + Span::styled(label, Style::default().fg(Color::Black).bg(Color::Gray)), + Span::styled( + fit_to_width(&self.save_filename, input_width as usize), + Style::default().fg(Color::Black).bg(Color::White), + ), + ])), + row, + ); + let cursor_x = row + .x + .saturating_add(label.chars().count() as u16) + .saturating_add(1) + .saturating_add(self.save_filename.chars().count() as u16) + .min(row.x.saturating_add(label.chars().count() as u16 + input_width)); + let cursor_y = row.y; + frame.set_cursor_position((cursor_x, cursor_y)); + } + + fn render_save_error(&self, frame: &mut Frame<'_>) { + let area = centered_rect(38, 12, frame.area()); + frame.render_widget(Clear, area); + let block = Block::default() + .title(Span::styled( + " Error ", + Style::default() + .fg(Color::Red) + .bg(Color::White) + .add_modifier(Modifier::BOLD), + )) + .borders(Borders::ALL) + .border_type(BorderType::Double) + .border_style(Style::default().fg(Color::White).bg(Color::Red)) + .style(Style::default().fg(Color::White).bg(Color::Red)); + let inner = block.inner(area); + frame.render_widget(block, area); + if let Some(message) = &self.save_message { + frame.render_widget( + Paragraph::new(render_save_error_message(message)) + .style(Style::default().bg(Color::Red)) + .wrap(Wrap { trim: false }), + inner, + ); + } + } + + fn render_modal(&self, frame: &mut Frame<'_>) { + let area = centered_rect(80, 80, frame.area()); + frame.render_widget(Clear, area); + + let block = Block::default() + .title(Span::styled( + " Log record ", + Style::default() + .fg(Color::Black) + .bg(Color::Indexed(30)) + .add_modifier(Modifier::BOLD), + )) + .borders(Borders::ALL) + .border_type(BorderType::Double) + .border_style(Style::default().fg(Color::Indexed(30)).bg(Color::Gray)) + .style(Style::default().fg(Color::Black).bg(Color::Gray)); + let inner = block.inner(area); + frame.render_widget(block, area); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(1), Constraint::Length(1)]) + .split(inner); + + let info_entries = if let Some(detail) = &self.selected_detail { + render_modal_info_entries(detail) + } else { + vec![("info".to_string(), "No record loaded.".to_string())] + }; + let key_width = info_entries + .iter() + .map(|(key, _)| key.chars().count()) + .max() + .unwrap_or(4) + .max(4); + let preferred_info_width = info_entries + .iter() + .map(|(_, value)| (key_width + 2 + value.chars().count() + 1) as u16) + .max() + .unwrap_or((key_width + 3) as u16); + let max_info_width = chunks[0].width.saturating_div(2).max(16); + let info_width = preferred_info_width.min(max_info_width).max(16); + let divider_width = 1; + let message_width = chunks[0] + .width + .saturating_sub(info_width) + .saturating_sub(divider_width); + + let body = Layout::default() + .direction(Direction::Horizontal) + .constraints([ + Constraint::Length(message_width), + Constraint::Length(divider_width), + Constraint::Length(info_width), + ]) + .split(chunks[0]); + + let divider = (0..body[1].height) + .map(|_| Line::from(Span::styled("│", Style::default().fg(Color::Indexed(30)).bg(Color::Gray)))) + .collect::>(); + frame.render_widget( + Paragraph::new(divider).style(Style::default().bg(Color::Gray)), + body[1], + ); + + let footer = if let Some(detail) = &self.selected_detail { + render_modal_footer(detail) + } else { + render_modal_footer_placeholder() + }; + let message = self.modal_text.as_deref().unwrap_or("No record loaded."); + let paragraph = Paragraph::new(message) + .style(Style::default().fg(Color::Black).bg(Color::Gray)) + .scroll((self.modal_scroll, 0)) + .wrap(Wrap { trim: false }); + frame.render_widget(paragraph, body[0]); + + let value_width = info_width + .saturating_sub((key_width + 2 + 1) as u16) as usize; + let info_lines = info_entries + .into_iter() + .map(|(key, value)| modal_info_line(&key, value, key_width, value_width)) + .collect::>(); + let info = Paragraph::new(Text::from(info_lines)) + .style(Style::default().fg(Color::Black).bg(Color::Gray)) + .scroll((0, 0)); + frame.render_widget(info, body[2]); + frame.render_widget( + Paragraph::new(footer).style(Style::default().bg(Color::Indexed(30))), + chunks[1], + ); + } +} + +fn scan_matches( + input_path: &Path, + predicate: crate::predicate::RecordPredicate, + mut spool: File, + cancel: Arc, + tx: mpsc::Sender, +) -> Result<(u64, u64)> { + let input = InputHandle::open(input_path)?; + let mut reader = LogjetReader::new(input.into_buf_reader()); + let mut tx_batch = Vec::with_capacity(SCAN_BATCH_SIZE); + let mut scanned = 0u64; + let mut matched = 0u64; + + while !cancel.load(Ordering::Relaxed) { + let Some(record) = reader.next_record()? else { + break; + }; + scanned = scanned + .checked_add(1) + .ok_or(logjet::Error::NumericOverflow("view scanned"))?; + + if predicate.matches(&record) { + let meta = write_spool_record(&mut spool, &record)?; + tx_batch.push(meta); + matched = matched + .checked_add(1) + .ok_or(logjet::Error::NumericOverflow("view matched"))?; + + if tx_batch.len() >= SCAN_BATCH_SIZE { + tx.send(ScanUpdate::Batch(std::mem::take(&mut tx_batch))) + .map_err(|err| Error::Usage(err.to_string()))?; + } + } + } + + if !tx_batch.is_empty() { + tx.send(ScanUpdate::Batch(tx_batch)) + .map_err(|err| Error::Usage(err.to_string()))?; + } + + Ok((scanned, matched)) +} + +fn write_spool_record(file: &mut File, record: &OwnedRecord) -> Result { + let offset = file.seek(SeekFrom::End(0))?; + file.write_all(&[record.record_type as u8])?; + file.write_all(&record.seq.to_le_bytes())?; + file.write_all(&record.ts_unix_ns.to_le_bytes())?; + let payload_len = u64::try_from(record.payload.len()) + .map_err(|_| logjet::Error::NumericOverflow("view payload_len"))?; + file.write_all(&payload_len.to_le_bytes())?; + file.write_all(&record.payload)?; + file.flush()?; + + Ok(EntryMeta { + offset, + record_type: record.record_type, + seq: record.seq, + ts_unix_ns: record.ts_unix_ns, + payload_len, + }) +} + +fn read_spool_record(file: &mut File, meta: EntryMeta) -> Result { + file.seek(SeekFrom::Start(meta.offset + 1 + 8 + 8 + 8))?; + let mut payload = vec![0u8; meta.payload_len as usize]; + file.read_exact(&mut payload)?; + Ok(DetailRecord { meta, payload }) +} + +fn remember_summary( + cache: &mut HashMap, + order: &mut VecDeque, + index: usize, + summary: String, +) { + cache.insert(index, summary); + order.push_back(index); + while order.len() > SUMMARY_CACHE_LIMIT { + if let Some(old) = order.pop_front() { + cache.remove(&old); + } + } +} + +fn format_summary(detail: &DetailRecord, hex_payload: bool) -> String { + let preview = if hex_payload { + hex_preview(&detail.payload, 32) + } else if let Some(message) = extract_otlp_log_message(&detail.payload) { + trim_single_line(&message, 160) + } else { + text_preview(&detail.payload, 160) + }; + preview +} + +fn render_detail_lines(detail: &DetailRecord, hex_payload: bool) -> Vec> { + let mut lines = vec![ + key_value_line( + "Record type:", + record_kind_label(detail.meta.record_type).to_string(), + Style::default().fg(Color::LightGreen).add_modifier(Modifier::BOLD), + ), + key_value_line("Sequence:", detail.meta.seq.to_string(), Style::default().fg(Color::White)), + key_value_line( + "Timestamp:", + format_timestamp(detail.meta.ts_unix_ns), + Style::default().fg(Color::White), + ), + key_value_line( + "Payload:", + format!("{} bytes", detail.meta.payload_len), + Style::default().fg(Color::White), + ), + Line::from(""), + ]; + + lines.extend(render_otlp_lines(detail)); + if lines.len() == 5 { + let preview = if hex_payload { + hex_preview(&detail.payload, 64) + } else { + text_preview(&detail.payload, DETAIL_PREVIEW_BYTES) + }; + lines.push(key_value_line("Preview:", preview, Style::default().fg(Color::White))); + } + + lines +} + +fn render_otlp_lines(detail: &DetailRecord) -> Vec> { + if detail.meta.record_type != RecordType::Logs { + return Vec::new(); + } + + let Ok(batch) = ExportLogsServiceRequest::decode(detail.payload.as_slice()) else { + return vec![Line::from(vec![ + Span::styled("OTLP logs: ", Style::default().add_modifier(Modifier::BOLD)), + Span::raw("payload decode failed; showing raw preview"), + ])]; + }; + + let mut services = Vec::new(); + let mut severities = Vec::new(); + let mut record_count = 0usize; + let mut scope_count = 0usize; + + for resource_logs in &batch.resource_logs { + if let Some(resource) = &resource_logs.resource { + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !services.iter().any(|existing| existing == service) + { + services.push(service.clone()); + } + } + } + + for scope_logs in &resource_logs.scope_logs { + scope_count += 1; + for log_record in &scope_logs.log_records { + record_count += 1; + if !log_record.severity_text.is_empty() + && !severities.iter().any(|existing| existing == &log_record.severity_text) + { + severities.push(log_record.severity_text.clone()); + } + } + } + } + + let mut lines = vec![ + key_value_line("OTLP kind:", "logs".to_string(), Style::default().fg(Color::White)), + key_value_line("Resources:", batch.resource_logs.len().to_string(), Style::default().fg(Color::White)), + key_value_line("Scopes:", scope_count.to_string(), Style::default().fg(Color::White)), + key_value_line("Log records:", record_count.to_string(), Style::default().fg(Color::White)), + ]; + + if !services.is_empty() { + lines.push(key_value_line( + "Services:", + services.join(", "), + Style::default().fg(Color::White), + )); + } + if !severities.is_empty() { + lines.push(key_value_line( + "Severity:", + severities.join(", "), + severity_style(severities.first().map(String::as_str).unwrap_or("")), + )); + } + + lines +} + +fn extract_otlp_log_message(payload: &[u8]) -> Option { + let batch = ExportLogsServiceRequest::decode(payload).ok()?; + for resource_logs in &batch.resource_logs { + for scope_logs in &resource_logs.scope_logs { + for log_record in &scope_logs.log_records { + if let Some(body) = &log_record.body + && let Some(Value::StringValue(message)) = &body.value + { + return Some(message.clone()); + } + } + } + } + None +} + +fn render_modal_message(detail: &DetailRecord, hex_payload: bool) -> String { + if let Some(message) = extract_otlp_log_message(&detail.payload) { + return message; + } + + if hex_payload { + hex_dump(&detail.payload) + } else { + String::from_utf8_lossy(&detail.payload).into_owned() + } +} + +fn render_modal_footer(detail: &DetailRecord) -> Line<'static> { + let (size_num, size_unit) = format_size_parts(detail.meta.payload_len); + Line::from(vec![ + Span::styled( + format!("#{}", detail.meta.seq), + Style::default().fg(Color::LightGreen), + ), + footer_sep(), + Span::styled( + format_timestamp(detail.meta.ts_unix_ns), + Style::default().fg(Color::White), + ), + footer_sep(), + Span::styled( + record_kind_label(detail.meta.record_type).to_string(), + Style::default().fg(Color::Black).add_modifier(Modifier::BOLD), + ), + footer_sep(), + Span::styled(size_num, Style::default().fg(Color::Yellow)), + Span::styled(size_unit, Style::default().fg(Color::Black)), + ]) +} + +fn render_modal_footer_placeholder() -> Line<'static> { + Line::from(vec![ + Span::styled("#", Style::default().fg(Color::LightGreen)), + footer_sep(), + Span::styled("", Style::default().fg(Color::White)), + footer_sep(), + Span::styled("", Style::default().fg(Color::Black).add_modifier(Modifier::BOLD)), + footer_sep(), + Span::styled("", Style::default().fg(Color::Yellow)), + Span::styled("", Style::default().fg(Color::Black)), + ]) +} + +fn render_modal_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { + let mut lines = vec![ + ("type".to_string(), record_kind_label(detail.meta.record_type).to_string()), + ("seq".to_string(), detail.meta.seq.to_string()), + ("ts_unix_ns".to_string(), detail.meta.ts_unix_ns.to_string()), + ("time".to_string(), format_timestamp(detail.meta.ts_unix_ns)), + ("payload_bytes".to_string(), detail.meta.payload_len.to_string()), + ]; + + if detail.meta.record_type != RecordType::Logs { + return lines; + } + + let Ok(batch) = ExportLogsServiceRequest::decode(detail.payload.as_slice()) else { + lines.push(("otlp".to_string(), "decode failed".to_string())); + return lines; + }; + + let mut service_names = Vec::new(); + let mut scopes = Vec::new(); + let mut severities = Vec::new(); + let mut event_names = Vec::new(); + let mut resource_attr_count = 0usize; + let mut record_attr_count = 0usize; + let mut trace_ids = 0usize; + let mut span_ids = 0usize; + + for resource_logs in &batch.resource_logs { + if let Some(resource) = &resource_logs.resource { + resource_attr_count += resource.attributes.len(); + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !service_names.iter().any(|existing| existing == service) + { + service_names.push(service.clone()); + } + } + } + + for scope_logs in &resource_logs.scope_logs { + if let Some(scope) = &scope_logs.scope + && !scope.name.is_empty() + && !scopes.iter().any(|existing| existing == &scope.name) + { + scopes.push(scope.name.clone()); + } + for record in &scope_logs.log_records { + record_attr_count += record.attributes.len(); + if !record.severity_text.is_empty() + && !severities.iter().any(|existing| existing == &record.severity_text) + { + severities.push(record.severity_text.clone()); + } + if !record.event_name.is_empty() + && !event_names.iter().any(|existing| existing == &record.event_name) + { + event_names.push(record.event_name.clone()); + } + if !record.trace_id.is_empty() { + trace_ids += 1; + } + if !record.span_id.is_empty() { + span_ids += 1; + } + } + } + } + + lines.push(("otlp.kind".to_string(), "logs".to_string())); + lines.push(("resources".to_string(), batch.resource_logs.len().to_string())); + if !service_names.is_empty() { + lines.push(("service.name".to_string(), service_names.join(", "))); + } + if !scopes.is_empty() { + lines.push(("scope".to_string(), scopes.join(", "))); + } + if !severities.is_empty() { + lines.push(("severity".to_string(), severities.join(", "))); + } + if !event_names.is_empty() { + lines.push(("event".to_string(), event_names.join(", "))); + } + lines.push(("resource.attrs".to_string(), resource_attr_count.to_string())); + lines.push(("record.attrs".to_string(), record_attr_count.to_string())); + if trace_ids > 0 { + lines.push(("trace_id".to_string(), format!("{trace_ids} present"))); + } + if span_ids > 0 { + lines.push(("span_id".to_string(), format!("{span_ids} present"))); + } + + lines +} + +fn modal_info_line(key: &str, value: String, key_width: usize, value_width: usize) -> Line<'static> { + let value = trim_single_line(&value, value_width); + Line::from(vec![ + Span::styled( + format!("{key: Span<'static> { + Span::styled(" | ", Style::default().fg(Color::Black)) +} + +fn format_size_parts(bytes: u64) -> (String, String) { + if bytes >= 1024 * 1024 { + (format!("{:.1}", bytes as f64 / (1024.0 * 1024.0)), " Mb".to_string()) + } else if bytes >= 1024 { + (format!("{:.1}", bytes as f64 / 1024.0), " Kb".to_string()) + } else { + (bytes.to_string(), " Bt".to_string()) + } +} + +fn record_kind_label(record_type: RecordType) -> &'static str { + match record_type { + RecordType::Logs => "logs", + RecordType::Metrics => "metrics", + RecordType::Traces => "traces", + } +} + +fn text_preview(bytes: &[u8], limit: usize) -> String { + trim_single_line(&String::from_utf8_lossy(bytes), limit) +} + +fn trim_single_line(input: &str, limit: usize) -> String { + let flattened = input + .chars() + .map(|ch| match ch { + '\n' | '\r' | '\t' => ' ', + other if other.is_control() => ' ', + other => other, + }) + .collect::(); + + let mut output = flattened.chars().take(limit).collect::(); + if flattened.chars().count() > limit { + output.push_str("..."); + } + output +} + +fn fit_to_width(input: &str, width: usize) -> String { + if width == 0 { + return String::new(); + } + let char_count = input.chars().count(); + if char_count <= width { + let mut padded = input.to_string(); + padded.push_str(&" ".repeat(width - char_count)); + return padded; + } + if width <= 3 { + return ".".repeat(width); + } + let mut out = input.chars().take(width - 3).collect::(); + out.push_str("..."); + out +} + +fn key_value_line(label: &str, value: String, value_style: Style) -> Line<'static> { + Line::from(vec![ + Span::styled(format!("{label:<12} "), Style::default().fg(Color::Indexed(136))), + Span::styled(value, value_style), + ]) +} + +fn severity_style(value: &str) -> Style { + let upper = value.to_ascii_uppercase(); + let color = if upper.contains("ERROR") || upper.contains("ERR") || upper.contains("FATAL") { + Color::LightRed + } else if upper.contains("WARN") { + Color::Indexed(214) + } else if upper.contains("INFO") { + Color::LightGreen + } else if upper.contains("DEBUG") { + Color::LightCyan + } else if upper.contains("TRACE") { + Color::LightBlue + } else { + Color::White + }; + Style::default().fg(color).add_modifier(Modifier::BOLD) +} + +fn format_timestamp(ts_unix_ns: u64) -> String { + let secs = (ts_unix_ns / 1_000_000_000) as i64; + let nanos = (ts_unix_ns % 1_000_000_000) as u32; + match Utc.timestamp_opt(secs, nanos).single() { + Some(ts) => ts.format("%Y-%m-%d %H:%M:%S.%f UTC").to_string(), + None => ts_unix_ns.to_string(), + } +} + +fn hex_preview(bytes: &[u8], limit: usize) -> String { + let shown = bytes.iter().take(limit); + let mut out = shown + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" "); + if bytes.len() > limit { + out.push_str(" ..."); + } + out +} + +fn hex_dump(bytes: &[u8]) -> String { + let mut out = String::new(); + for (chunk_index, chunk) in bytes.chunks(16).enumerate() { + out.push_str(&format!("{:08x}: ", chunk_index * 16)); + for byte in chunk { + out.push_str(&format!("{byte:02x} ")); + } + out.push('\n'); + } + out +} + +fn centered_rect(width_percent: u16, height_percent: u16, area: Rect) -> Rect { + let popup = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Percentage((100 - height_percent) / 2), + Constraint::Percentage(height_percent), + Constraint::Percentage((100 - height_percent) / 2), + ]) + .split(area); + Layout::default() + .direction(Direction::Horizontal) + .constraints([ + Constraint::Percentage((100 - width_percent) / 2), + Constraint::Percentage(width_percent), + Constraint::Percentage((100 - width_percent) / 2), + ]) + .split(popup[1])[1] +} + +fn pane_block<'a>(title: &'a str, active: bool) -> Block<'a> { + let title_style = if active { + Style::default().fg(Color::Black).bg(Color::LightGreen) + } else { + Style::default().fg(Color::Indexed(28)) + }; + let border_style = if active { + Style::default().fg(Color::LightGreen) + } else { + Style::default().fg(Color::Indexed(28)) + }; + + Block::default() + .title(Span::styled(title, title_style)) + .borders(Borders::ALL) + .border_type(if active { BorderType::Double } else { BorderType::Plain }) + .border_style(border_style) + .style(Style::default().fg(Color::White)) +} + +fn status_help_spans(focus: Focus) -> Vec> { + match focus { + Focus::Search => vec![ + status_key("Q"), + status_text(" quit "), + status_key("TAB"), + status_text(" switch "), + status_key("ENTER"), + status_text(" apply "), + status_key("ESC"), + status_text(" clear filter "), + status_key("UP/DOWN"), + status_text(" change mode"), + ], + Focus::List => vec![ + status_key("Q"), + status_text(" quit "), + status_key("TAB"), + status_text(" switch "), + status_key("ENTER"), + status_text(" open "), + status_key("S"), + status_text(" save to file "), + status_key("UP/DOWN"), + status_text(" navigate"), + ], + Focus::Modal => Vec::new(), + Focus::SavePrompt => Vec::new(), + Focus::SaveError => Vec::new(), + } +} + +fn status_key(text: &str) -> Span<'static> { + Span::styled( + text.to_string(), + Style::default() + .fg(Color::White) + .bg(Color::Indexed(28)) + .add_modifier(Modifier::BOLD), + ) +} + +fn status_text(text: &str) -> Span<'static> { + Span::styled( + text.to_string(), + Style::default().fg(Color::Black).bg(Color::Indexed(28)), + ) +} + +fn draw_status_spans( + buf: &mut ratatui::buffer::Buffer, + x: u16, + y: u16, + width: u16, + spans: &[Span<'static>], +) { + let mut cursor_x = x; + let mut remaining = width; + for span in spans { + if remaining == 0 { + break; + } + let next_x = buf.set_stringn(cursor_x, y, span.content.as_ref(), remaining as usize, span.style); + remaining = remaining.saturating_sub(next_x.0.saturating_sub(cursor_x)); + cursor_x = next_x.0; + } +} + +fn render_save_error_message(message: &str) -> Line<'static> { + const PREFIX: &str = "File "; + const SUFFIX: &str = " already exist"; + + if let Some(filename) = message + .strip_prefix(PREFIX) + .and_then(|rest| rest.strip_suffix(SUFFIX)) + { + return Line::from(vec![ + Span::styled(PREFIX, Style::default().fg(Color::White).bg(Color::Red)), + Span::styled( + filename.to_string(), + Style::default() + .fg(Color::Yellow) + .bg(Color::Red) + .add_modifier(Modifier::BOLD), + ), + Span::styled(SUFFIX, Style::default().fg(Color::White).bg(Color::Red)), + ]); + } + + Line::from(Span::styled( + message.to_string(), + Style::default().fg(Color::White).bg(Color::Red), + )) +} + +fn create_temp_path() -> Result { + let base = std::env::temp_dir(); + let pid = std::process::id(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| Error::Usage(format!("system clock error: {err}")))? + .as_nanos(); + for attempt in 0..1000u32 { + let candidate = base.join(format!("ljx-view-{pid}-{nanos}-{attempt}.tmp")); + if !candidate.exists() { + return Ok(candidate); + } + } + Err(Error::Usage("unable to allocate a temporary view file".to_string())) +} + +#[cfg(test)] +mod tests { + use super::{ + DetailRecord, EntryMeta, extract_otlp_log_message, format_summary, + render_modal_message, text_preview, + }; + use logjet::RecordType; + use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; + use opentelemetry_proto::tonic::common::v1::any_value::Value; + use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope}; + use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; + use opentelemetry_proto::tonic::resource::v1::Resource; + use prost::Message; + + #[test] + fn text_preview_flattens_newlines() { + assert_eq!(text_preview(b"hello\nworld", 32), "hello world"); + } + + #[test] + fn summary_uses_trimmed_single_line_preview() { + let detail = DetailRecord { + meta: EntryMeta { + offset: 0, + record_type: RecordType::Logs, + seq: 7, + ts_unix_ns: 9, + payload_len: 13, + }, + payload: b"line one\nline two".to_vec(), + }; + let summary = format_summary(&detail, false); + assert_eq!(summary, "line one line two"); + } + + #[test] + fn summary_prefers_decoded_otlp_log_message() { + let batch = ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(Resource { + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + scope_logs: vec![ScopeLogs { + scope: Some(InstrumentationScope { + name: "test".to_string(), + version: String::new(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + log_records: vec![LogRecord { + time_unix_nano: 0, + observed_time_unix_nano: 0, + severity_number: 0, + severity_text: String::new(), + body: Some(AnyValue { + value: Some(Value::StringValue("hello from body".to_string())), + }), + attributes: Vec::new(), + dropped_attributes_count: 0, + flags: 0, + trace_id: Vec::new(), + span_id: Vec::new(), + event_name: String::new(), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { + offset: 0, + record_type: RecordType::Logs, + seq: 1, + ts_unix_ns: 2, + payload_len: payload.len() as u64, + }, + payload, + }; + + assert_eq!(extract_otlp_log_message(&detail.payload).as_deref(), Some("hello from body")); + assert_eq!(format_summary(&detail, false), "hello from body"); + } + + #[test] + fn modal_falls_back_to_raw_payload() { + let detail = DetailRecord { + meta: EntryMeta { + offset: 0, + record_type: RecordType::Metrics, + seq: 1, + ts_unix_ns: 2, + payload_len: 5, + }, + payload: b"hello".to_vec(), + }; + let body = render_modal_message(&detail, false); + assert_eq!(body, "hello"); + } +} diff --git a/ljx/src/main.rs b/ljx/src/main.rs index a72a2cd..ce3f762 100644 --- a/ljx/src/main.rs +++ b/ljx/src/main.rs @@ -27,6 +27,6 @@ fn run() -> Result<()> { Command::Filter(args) => commands::filter::run(args), Command::Count(args) => commands::count::run(args), Command::Stats(args) => commands::stats::run(args), - Command::Cat(args) => commands::cat::run(args), + Command::View(args) => commands::view::run(args), } } diff --git a/ljx/src/predicate.rs b/ljx/src/predicate.rs index 5869849..2ed0de5 100644 --- a/ljx/src/predicate.rs +++ b/ljx/src/predicate.rs @@ -1,9 +1,15 @@ -use clap::{ArgGroup, Args, ValueEnum}; +use clap::{ArgGroup, Args, CommandFactory, FromArgMatches, Parser, ValueEnum}; use logjet::{OwnedRecord, RecordType}; use regex::bytes::{Regex, RegexBuilder}; use crate::error::{Error, Result}; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FilterMode { + Strings, + Regex, +} + #[derive(Debug, Clone, Args, Default)] #[command(group( ArgGroup::new("payload_match") @@ -75,6 +81,49 @@ impl PredicateArgs { } } +#[derive(Debug, Parser)] +struct PredicateCli { + #[command(flatten)] + predicate: PredicateArgs, +} + +pub fn parse_filter_query(query: &str, mode: FilterMode) -> Result { + let trimmed = query.trim(); + if trimmed.is_empty() { + return PredicateArgs::default().build(); + } + + // Bare text in the TUI stays ergonomic and depends on the active filter mode. + if !trimmed.starts_with('-') { + return match mode { + FilterMode::Strings => PredicateArgs { + fixed_string: Some(trimmed.to_string()), + ..PredicateArgs::default() + } + .build(), + FilterMode::Regex => PredicateArgs { + grep: Some(trimmed.to_string()), + ..PredicateArgs::default() + } + .build(), + }; + } + + let argv = shlex::split(trimmed) + .ok_or_else(|| Error::Usage("invalid filter expression: unterminated quotes".to_string()))?; + let mut full_argv = Vec::with_capacity(argv.len() + 1); + full_argv.push("view-filter".to_string()); + full_argv.extend(argv); + + let mut command = PredicateCli::command(); + let mut matches = command + .try_get_matches_from_mut(full_argv) + .map_err(|err| Error::Usage(err.to_string()))?; + let parsed = + PredicateCli::from_arg_matches_mut(&mut matches).map_err(|err| Error::Usage(err.to_string()))?; + parsed.predicate.build() +} + impl RecordPredicate { pub fn matches(&self, record: &OwnedRecord) -> bool { if let Some(expected) = self.record_type && record.record_type != expected { @@ -138,7 +187,7 @@ impl From for RecordType { #[cfg(test)] mod tests { - use super::{PredicateArgs, RecordKind}; + use super::{FilterMode, PredicateArgs, RecordKind, parse_filter_query}; use logjet::{OwnedRecord, RecordType}; fn sample_record(payload: &[u8]) -> OwnedRecord { @@ -227,4 +276,30 @@ mod tests { assert!(error.to_string().contains("invalid payload matcher")); } + + #[test] + fn parse_filter_query_treats_bare_text_as_fixed_string() { + let predicate = parse_filter_query("hello world", FilterMode::Strings).unwrap(); + assert!(predicate.matches(&sample_record(b"say hello world now"))); + assert!(!predicate.matches(&sample_record(b"say hello now"))); + } + + #[test] + fn parse_filter_query_supports_cli_style_flags() { + let predicate = parse_filter_query(r#"--type logs -e "error|panic" -i"#, FilterMode::Strings).unwrap(); + assert!(predicate.matches(&sample_record(b"PANIC happened"))); + assert!(!predicate.matches(&OwnedRecord { + record_type: RecordType::Metrics, + seq: 42, + ts_unix_ns: 1_700_000_000, + payload: b"panic".to_vec(), + })); + } + + #[test] + fn parse_filter_query_uses_regex_mode_for_bare_text() { + let predicate = parse_filter_query("reb.*", FilterMode::Regex).unwrap(); + assert!(predicate.matches(&sample_record(b"rebooted node"))); + assert!(!predicate.matches(&sample_record(b"stopped node"))); + } } From 647310f34f1a52eee6d9b22490a45c750c5d7a8c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:46:25 +0100 Subject: [PATCH 13/17] Update docs --- doc/ljx.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/ljx.md b/doc/ljx.md index 815eecf..a4db715 100644 --- a/doc/ljx.md +++ b/doc/ljx.md @@ -34,7 +34,7 @@ Documented command set: - `count` - `filter` - `stats` -- `cat` +- `view` - `split` - `join` @@ -42,7 +42,7 @@ Current implementation status for release `0.1`: - implemented first: `count` - implemented first: `filter` -- planned after that: `stats`, `cat`, `split`, `join` +- planned after that: `stats`, `split`, `join` The CLI may already expose planned command names, but release `0.1` should only promise the commands that are actually complete and tested. @@ -124,14 +124,17 @@ Intended summary fields: - timestamp range - optional per-type or per-field summaries -## `ljx cat` +## `ljx view` -Render records in a human-readable form suitable for terminal inspection. +Browse filtered records in an interactive terminal UI. -Open questions: +Current shape: -- whether payload bytes should default to hex, escaped text, or a compact mixed format -- how much payload to print before truncating +- search field at the top, applied with `Enter` +- matching records on the left in a one-line-per-record list +- dynamic details for the selected record on the right +- `Enter` opens a full-record popup, `Esc` closes it +- bounded-memory scan that spools matched records to a temp file instead of loading the whole input ## `ljx split` From 3d39ac648417b8b5c0b98a011ec4b3a3a20f8ae6 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:46:49 +0100 Subject: [PATCH 14/17] Update manpage --- doc/manpage/ljx.1.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manpage/ljx.1.md b/doc/manpage/ljx.1.md index 56dbd24..2e0b663 100644 --- a/doc/manpage/ljx.1.md +++ b/doc/manpage/ljx.1.md @@ -14,7 +14,7 @@ ljx - offline toolbox for inspecting and transforming `.logjet` files `ljx` `stats` *input* -`ljx` `cat` *input* +`ljx` `view` *input* `ljx` `split` *input* *output-prefix* @@ -86,11 +86,11 @@ Planned summaries include: `stats` is planned but may not be complete in release `0.1`. -## cat +## view -Print records in a human-readable form for inspection. +Browse filtered records in an interactive terminal UI. -`cat` is planned but may not be complete in release `0.1`. +`view` is available for interactive inspection and uses a bounded-memory scan. ## split @@ -290,7 +290,7 @@ Use this when you need cardinality instead of an output file. ## 15. Print records for terminal inspection ```text -ljx cat telemetry.logjet +ljx view telemetry.logjet ``` Use this when you want a human-readable record listing. @@ -298,7 +298,7 @@ Use this when you want a human-readable record listing. ## 16. Print records with hex payload rendering ```text -ljx cat telemetry.logjet --hex-payload +ljx view telemetry.logjet --hex-payload ``` Use this when payload bytes are binary and text rendering is misleading. From a5b1a4622ca1d54b5cf1437010d12b8a1de6b8f5 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:47:11 +0100 Subject: [PATCH 15/17] Add a simple TUI demo --- Cargo.lock | 391 ++++++++++++++++++- demo/README.md | 2 + demo/src/bin/otlp-random-logjet-generator.rs | 124 ++++++ demo/tui-view/README.md | 49 +++ demo/tui-view/run-demo.sh | 27 ++ demo/tui-view/tui-view.conf | 4 + 6 files changed, 594 insertions(+), 3 deletions(-) create mode 100644 demo/src/bin/otlp-random-logjet-generator.rs create mode 100644 demo/tui-view/README.md create mode 100755 demo/tui-view/run-demo.sh create mode 100644 demo/tui-view/tui-view.conf diff --git a/Cargo.lock b/Cargo.lock index 211fdda..ff31a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "0.6.21" @@ -189,6 +195,21 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -205,6 +226,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "num-traits", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -267,6 +297,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -276,6 +320,65 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -310,6 +413,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "futures-channel" version = "0.3.32" @@ -415,6 +524,17 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -534,6 +654,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" version = "1.9.3" @@ -554,12 +680,43 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -591,6 +748,12 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -601,10 +764,25 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" name = "ljx" version = "0.1.0" dependencies = [ + "chrono", "clap", "colored", + "crossterm", "logjet", + "opentelemetry-proto", + "prost", + "ratatui", "regex", + "shlex", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", ] [[package]] @@ -639,6 +817,15 @@ dependencies = [ "tonic", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lz4_flex" version = "0.11.5" @@ -670,10 +857,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -749,6 +946,35 @@ dependencies = [ "tonic", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -822,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -867,6 +1093,36 @@ dependencies = [ "getrandom", ] +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.3" @@ -919,6 +1175,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -928,7 +1197,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -988,6 +1257,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.27" @@ -1056,6 +1331,37 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "slab" version = "0.4.12" @@ -1088,12 +1394,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1123,7 +1457,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-sys 0.60.2", ] @@ -1340,6 +1674,35 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1418,6 +1781,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/demo/README.md b/demo/README.md index 390ba14..72248fd 100644 --- a/demo/README.md +++ b/demo/README.md @@ -39,6 +39,8 @@ It also contains scenario demos under subdirectories: - replay stored `.logjet` files into a collector - [`file-tooling`](./file-tooling) - inspect rotated file segments and prune archived files deliberately +- [`tui-view`](./tui-view) + - generate 1000 randomized log entries and open `ljx view` on the result - [`bridge-resume`](./bridge-resume) - consumer restart resumes from persisted sequence state without replaying from zero - [`upstream-reset-resume`](./upstream-reset-resume) diff --git a/demo/src/bin/otlp-random-logjet-generator.rs b/demo/src/bin/otlp-random-logjet-generator.rs new file mode 100644 index 0000000..9926925 --- /dev/null +++ b/demo/src/bin/otlp-random-logjet-generator.rs @@ -0,0 +1,124 @@ +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::PathBuf; + +use logjet::{LogjetWriter, RecordType, WriterConfig}; +use otlp_demo::build_message_request_for_service; +use prost::Message; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let output = match args.next() { + Some(path) => PathBuf::from(path), + None => { + eprintln!("usage: otlp-random-logjet-generator [count] [seed]"); + std::process::exit(2); + } + }; + let count = args + .next() + .map(|value| value.parse::()) + .transpose()? + .unwrap_or(1000); + let seed = args + .next() + .map(|value| value.parse::()) + .transpose()? + .unwrap_or(0x5eed_1234_u64); + + let file = File::create(&output)?; + let writer = BufWriter::new(file); + let mut logjet = LogjetWriter::with_config(writer, WriterConfig::default()); + let mut rng = Lcg::new(seed); + + for seq in 1..=count { + let service = SERVICES[rng.next_index(SERVICES.len())]; + let severity = LEVELS[rng.next_index(LEVELS.len())]; + let message = format_message(seq, &mut rng); + let request = build_message_request_for_service(seq, service, severity, message); + logjet.push(RecordType::Logs, seq, unix_time_nanos(seq), &request.encode_to_vec())?; + } + + let mut writer = logjet.into_inner()?; + writer.flush()?; + println!("wrote {count} random log records to {}", output.display()); + Ok(()) +} + +const SERVICES: &[&str] = &[ + "bofh-emitter", + "kill-bill", + "garage-rig", + "bridge-alpha", + "night-shift", + "coffee-daemon", +]; + +const LEVELS: &[&str] = &["trace", "debug", "info", "warn", "error"]; + +const SUBJECTS: &[&str] = &[ + "rebooted node", + "dropped packet train", + "rewired pipeline", + "latched failover switch", + "purged stale checkpoint", + "stalled replay queue", + "overfed metrics sink", + "misread tape robot", +]; + +const CONTEXTS: &[&str] = &[ + "after a coffee spill", + "during a midnight deploy", + "under synthetic backpressure", + "while testing replay recovery", + "after rotating the file segment", + "while the collector blinked twice", + "after a heroic config change", + "while tracing vanished quietly", +]; + +const OUTCOMES: &[&str] = &[ + "and everything looked suspiciously fine", + "and the bridge resumed from the right offset", + "but the checksum still looked offended", + "so the operator blamed cosmic rays", + "and the daemon recovered without drama", + "yet the dashboard remained emotionally unavailable", + "and the logs kept flowing anyway", + "before the intern touched production again", +]; + +fn format_message(seq: u64, rng: &mut Lcg) -> String { + let subject = SUBJECTS[rng.next_index(SUBJECTS.len())]; + let context = CONTEXTS[rng.next_index(CONTEXTS.len())]; + let outcome = OUTCOMES[rng.next_index(OUTCOMES.len())]; + format!("#{seq}: {subject} {context} {outcome}") +} + +fn unix_time_nanos(seq: u64) -> u64 { + let base = 1_773_000_000_000_000_000u64; + base.saturating_add(seq.saturating_mul(1_000_000)) +} + +struct Lcg { + state: u64, +} + +impl Lcg { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next(&mut self) -> u64 { + self.state = self + .state + .wrapping_mul(6364136223846793005) + .wrapping_add(1); + self.state + } + + fn next_index(&mut self, len: usize) -> usize { + (self.next() % len as u64) as usize + } +} diff --git a/demo/tui-view/README.md b/demo/tui-view/README.md new file mode 100644 index 0000000..9b31f43 --- /dev/null +++ b/demo/tui-view/README.md @@ -0,0 +1,49 @@ +# TUI View Demo + +This demo generates one `.logjet` file with 1000 randomized OTLP log records +and immediately opens the interactive `ljx view` TUI on that file. + +## Build First + +From the project root: + +```bash +make demo +``` + +## Run + +From this directory: + +```bash +./run-demo.sh +``` + +## What It Does + +The script does this: + +1. reads demo settings from `./tui-view.conf` +2. generates 1000 randomized OTLP log records into `./logs/tui-view.logjet` +3. opens `ljx view` on that file + +The generated records vary: + +- service name +- severity level +- message text +- sequence number + +That gives the TUI enough variety to exercise: + +- literal-string filtering +- regex filtering +- popup inspection +- navigation over a larger result set + +## Files + +- `./tui-view.conf` + - demo settings such as output path, count, and seed +- `./logs/tui-view.logjet` + - generated input file for `ljx view` diff --git a/demo/tui-view/run-demo.sh b/demo/tui-view/run-demo.sh new file mode 100755 index 0000000..33150bb --- /dev/null +++ b/demo/tui-view/run-demo.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +GENERATOR="$TARGET_DIR/otlp-random-logjet-generator" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/tui-view.conf" + +if [ ! -x "$GENERATOR" ] || [ ! -x "$LJX" ]; then + echo "missing demo binaries" + echo "build them first with: make demo" + exit 1 +fi + +. "$CONFIG" + +cd "$SCRIPT_DIR" +mkdir -p logs +rm -f "$OUTPUT_FILE" + +echo "generating $COUNT random log entries into $OUTPUT_FILE" +"$GENERATOR" "$OUTPUT_FILE" "$COUNT" "$SEED" + +echo +echo "opening ljx view on $OUTPUT_FILE" +exec "$LJX" view "$OUTPUT_FILE" diff --git a/demo/tui-view/tui-view.conf b/demo/tui-view/tui-view.conf new file mode 100644 index 0000000..8187582 --- /dev/null +++ b/demo/tui-view/tui-view.conf @@ -0,0 +1,4 @@ +# shell-style demo config +COUNT=1000 +SEED=424242 +OUTPUT_FILE="logs/tui-view.logjet" From e991f900a8ca250a9db6369bb9d104228c02847d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:47:56 +0100 Subject: [PATCH 16/17] Lintfix: let_and_return --- ljx/src/commands/view.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ljx/src/commands/view.rs b/ljx/src/commands/view.rs index f83fec6..9bc880d 100644 --- a/ljx/src/commands/view.rs +++ b/ljx/src/commands/view.rs @@ -1027,14 +1027,14 @@ fn remember_summary( } fn format_summary(detail: &DetailRecord, hex_payload: bool) -> String { - let preview = if hex_payload { + + if hex_payload { hex_preview(&detail.payload, 32) } else if let Some(message) = extract_otlp_log_message(&detail.payload) { trim_single_line(&message, 160) } else { text_preview(&detail.payload, 160) - }; - preview + } } fn render_detail_lines(detail: &DetailRecord, hex_payload: bool) -> Vec> { From 440b4d47c54bb8b4d725592418d1405ef15a2f8c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 9 Mar 2026 18:49:51 +0100 Subject: [PATCH 17/17] Fix integration tests --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 8805bdb..551660b 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ fix: setup cargo clippy $(CORE_WORKSPACE) --fix --all-targets --all-features --allow-dirty --allow-staged -- -D warnings test: setup + cargo build -p logjetd -p ljx + cargo build -p otlp-demo --bin otlp-bofh-emitter @if command -v cargo-nextest >/dev/null 2>&1; then \ cargo nextest run $(CORE_WORKSPACE); \ else \