Disk buffer instrumented

Cargo.toml

@@ -19,7 +19,6 @@ version = "0.0.34"
 anyhow = "1.0.97"
 apache-avro = { version = "0.17.0", features = ["derive"] }
 async-trait = "0.1.88"
-axum = { version = "0.8.3", features = ["json", "ws", "query"] }
 chrono = { version = "0.4.40", features = ["serde"] }
 clap = { version = "4.5.35", features = ["derive", "env"] }
 derive_more = { version = "2.0.1", features = [
@@ -44,11 +43,9 @@ fuel-data-parser = { version = "0.0.34", path = "crates/data-parser" }
 fuel-streams-domains = { version = "0.0.34", path = "crates/domains" }
 fuel-streams-types = { version = "0.0.34", path = "crates/types" }
 fuel-web-utils = { version = "0.0.34", path = "crates/web-utils" }
-futures = "0.3.31"
 hex = "0.4.3"
 pretty_assertions = "1.4.1"
 rand = "0.9.0"
-rayon = "1.10.0"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
 serde_urlencoded = "0.7.1"
@@ -61,7 +58,6 @@ sqlx = { version = "0.8.3", default-features = false, features = [
 ] }
 strum = { version = "0.27.1", features = ["derive"] }
 strum_macros = "0.27.1"
-test-case = "3.3.1"
 thiserror = "2.0.12"
 tokio = { version = "1.44.1", features = [
     "io-util",

cluster/docker/sv-dune.Dockerfile

@@ -56,7 +56,7 @@ FROM debian:bookworm-slim AS run
 WORKDIR /usr/src
 
 RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends ca-certificates curl \
+    && apt-get install -y --no-install-recommends ca-certificates curl jq less unzip zip procps \
     # Clean up
     && apt-get autoremove -y \
     && apt-get clean -y \
@@ -65,4 +65,7 @@ RUN apt-get update -y \
 COPY --from=builder /root/sv-dune .
 COPY --from=builder /root/sv-dune.d .
 
+# run app as non-root user
+USER nobody
+
 CMD ["./sv-dune"]

services/dune/Cargo.toml

@@ -34,7 +34,6 @@ fuel-streams-domains = { workspace = true, features = ["test-helpers"] }
 fuel-streams-types = { workspace = true, features = ["test-helpers"] }
 fuel-web-utils = { workspace = true, features = ["test-helpers"] }
 futures = "0.3.31"
-itertools = "0.14.0"
 rand.workspace = true
 serde.workspace = true
 serde_bytes = "0.11.17"
@@ -46,6 +45,7 @@ url = "2.5.7"
 
 [dev-dependencies]
 pretty_assertions.workspace = true
+tempfile = "3"
 tokio = { workspace = true, features = [
     "rt-multi-thread",
     "macros",

services/dune/src/alloc_counter.rs

@@ -0,0 +1,47 @@
+//! Diagnostic allocation counters for tracking object lifetimes.
+//!
+//! Each counter is incremented on object creation and decremented **only** via
+//! `Drop` impls, so a counter that trends upward over time is proof of a leak —
+//! the object was created but never deallocated.
+//!
+//! Call [`log_all`] periodically to emit all live counts.
+
+use std::sync::atomic::{AtomicI64, Ordering};
+
+macro_rules! define_counters {
+    ($($name:ident),+ $(,)?) => {
+        $(pub static $name: AtomicI64 = AtomicI64::new(0);)+
+
+        /// Log all counters at INFO level, including tokio task count.
+        pub fn log_all() {
+            let tokio_tasks = tokio::runtime::Handle::current()
+                .metrics()
+                .num_alive_tasks();
+            tracing::info!(
+                concat!("alloc_counters: ", $(stringify!($name), "={} ",)+ "TOKIO_TASKS={}"),
+                $($name.load(Ordering::Relaxed),)+
+                tokio_tasks,
+            );
+        }
+    };
+}
+
+define_counters! {
+    GRAPHQL_FETCHER,
+    BLOCK_STREAM,
+    AVRO_FILE_WRITERS,
+    AVRO_FILE_WRITER,
+    FINALIZED_BATCH_FILES,
+}
+
+/// Increment a counter (call from constructors).
+#[inline]
+pub fn inc(counter: &AtomicI64) {
+    counter.fetch_add(1, Ordering::Relaxed);
+}
+
+/// Decrement a counter (call **only** from `Drop` impls).
+#[inline]
+pub fn dec(counter: &AtomicI64) {
+    counter.fetch_sub(1, Ordering::Relaxed);
+}