diff --git a/Cargo.lock b/Cargo.lock index e94bb80f13..619f703076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,7 +839,7 @@ dependencies = [ [[package]] name = "clickhouse-inserter" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-channel", @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "clickhouse-user-query" -version = "2.0.38" +version = "2.0.39" dependencies = [ "clickhouse", "serde", @@ -1501,7 +1501,7 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "epoxy" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -1540,7 +1540,7 @@ dependencies = [ [[package]] name = "epoxy-protocol" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "rivet-util", @@ -1827,7 +1827,7 @@ dependencies = [ [[package]] name = "gasoline" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-stream", @@ -1875,7 +1875,7 @@ dependencies = [ [[package]] name = "gasoline-macros" -version = "2.0.38" +version = "2.0.39" dependencies = [ "proc-macro2", "quote", @@ -2921,7 +2921,7 @@ dependencies = [ [[package]] name = "namespace" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "epoxy", @@ -3436,7 +3436,7 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pegboard" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "base64 0.22.1", @@ -3474,7 +3474,7 @@ dependencies = [ [[package]] name = "pegboard-actor-kv" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "futures-util", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "pegboard-gateway" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -3531,7 +3531,7 @@ dependencies = [ [[package]] name = "pegboard-runner" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -4269,7 +4269,7 @@ dependencies = [ [[package]] name = "rivet-api-builder" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -4312,7 +4312,7 @@ dependencies = [ [[package]] name = "rivet-api-peer" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -4343,7 +4343,7 @@ dependencies = [ [[package]] name = "rivet-api-public" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -4376,7 +4376,7 @@ dependencies = [ [[package]] name = "rivet-api-public-openapi-gen" -version = "2.0.38" +version = "2.0.39" dependencies = [ "rivet-api-public", "serde_json", @@ -4385,7 +4385,7 @@ dependencies = [ [[package]] name = "rivet-api-types" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "gasoline", @@ -4400,7 +4400,7 @@ dependencies = [ [[package]] name = "rivet-api-util" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -4420,7 +4420,7 @@ dependencies = [ [[package]] name = "rivet-bootstrap" -version = "2.0.38" +version = "2.0.39" dependencies = [ "epoxy", "gasoline", @@ -4439,7 +4439,7 @@ dependencies = [ [[package]] name = "rivet-cache" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "futures-util", @@ -4480,14 +4480,14 @@ dependencies = [ [[package]] name = "rivet-cache-result" -version = "2.0.38" +version = "2.0.39" dependencies = [ "rivet-util", ] [[package]] name = "rivet-config" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "config", @@ -4506,7 +4506,7 @@ dependencies = [ [[package]] name = "rivet-data" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "gasoline", @@ -4520,7 +4520,7 @@ dependencies = [ [[package]] name = "rivet-engine" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -4589,7 +4589,7 @@ dependencies = [ [[package]] name = "rivet-env" -version = "2.0.38" +version = "2.0.39" dependencies = [ "lazy_static", "uuid", @@ -4597,7 +4597,7 @@ dependencies = [ [[package]] name = "rivet-error" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "indoc", @@ -4609,7 +4609,7 @@ dependencies = [ [[package]] name = "rivet-error-macros" -version = "2.0.38" +version = "2.0.39" dependencies = [ "indoc", "proc-macro2", @@ -4620,7 +4620,7 @@ dependencies = [ [[package]] name = "rivet-guard" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "axum 0.8.4", @@ -4666,7 +4666,7 @@ dependencies = [ [[package]] name = "rivet-guard-core" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -4713,7 +4713,7 @@ dependencies = [ [[package]] name = "rivet-logs" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "chrono", @@ -4727,7 +4727,7 @@ dependencies = [ [[package]] name = "rivet-metrics" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "console-subscriber", @@ -4746,7 +4746,7 @@ dependencies = [ [[package]] name = "rivet-metrics-aggregator" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "epoxy", @@ -4761,7 +4761,7 @@ dependencies = [ [[package]] name = "rivet-pools" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-nats", @@ -4794,7 +4794,7 @@ dependencies = [ [[package]] name = "rivet-postgres-util" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "rustls 0.23.29", @@ -4805,7 +4805,7 @@ dependencies = [ [[package]] name = "rivet-runner-protocol" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "gasoline", @@ -4822,7 +4822,7 @@ dependencies = [ [[package]] name = "rivet-runtime" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "console-subscriber", @@ -4849,7 +4849,7 @@ dependencies = [ [[package]] name = "rivet-service-manager" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "chrono", @@ -4866,7 +4866,7 @@ dependencies = [ [[package]] name = "rivet-telemetry" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "rivet-config", @@ -4890,7 +4890,7 @@ dependencies = [ [[package]] name = "rivet-test-deps" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "futures-util", @@ -4908,7 +4908,7 @@ dependencies = [ [[package]] name = "rivet-test-deps-docker" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "portpicker", @@ -4939,7 +4939,7 @@ dependencies = [ [[package]] name = "rivet-tracing-utils" -version = "2.0.38" +version = "2.0.39" dependencies = [ "futures-util", "lazy_static", @@ -4949,7 +4949,7 @@ dependencies = [ [[package]] name = "rivet-types" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "gasoline", @@ -4966,7 +4966,7 @@ dependencies = [ [[package]] name = "rivet-ups-protocol" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "base64 0.22.1", @@ -4979,7 +4979,7 @@ dependencies = [ [[package]] name = "rivet-util" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -5011,7 +5011,7 @@ dependencies = [ [[package]] name = "rivet-util-id" -version = "2.0.38" +version = "2.0.39" dependencies = [ "serde", "thiserror 1.0.69", @@ -5022,7 +5022,7 @@ dependencies = [ [[package]] name = "rivet-workflow-worker" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "epoxy", @@ -6817,7 +6817,7 @@ checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "universaldb" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-trait", @@ -6848,7 +6848,7 @@ dependencies = [ [[package]] name = "universalpubsub" -version = "2.0.38" +version = "2.0.39" dependencies = [ "anyhow", "async-nats", diff --git a/engine/artifacts/openapi.json b/engine/artifacts/openapi.json index 9b97c86f55..45b9052e97 100644 --- a/engine/artifacts/openapi.json +++ b/engine/artifacts/openapi.json @@ -11,7 +11,7 @@ "name": "Apache-2.0", "identifier": "Apache-2.0" }, - "version": "2.0.38" + "version": "2.0.39" }, "paths": { "/actors": { diff --git a/engine/packages/config/src/config/mod.rs b/engine/packages/config/src/config/mod.rs index f82fd2122a..9d270288b6 100644 --- a/engine/packages/config/src/config/mod.rs +++ b/engine/packages/config/src/config/mod.rs @@ -213,6 +213,18 @@ impl Root { } } + // Validate force_shutdown_duration is greater than worker and guard shutdown durations + let worker = self.runtime.worker_shutdown_duration(); + let guard = self.runtime.guard_shutdown_duration(); + let force = self.runtime.force_shutdown_duration(); + let max_graceful = worker.max(guard); + if force <= max_graceful { + bail!( + "force_shutdown_duration ({force:?}) must be greater than both \ + worker_shutdown_duration ({worker:?}) and guard_shutdown_duration ({guard:?})" + ); + } + Ok(()) } diff --git a/engine/packages/config/src/config/runtime.rs b/engine/packages/config/src/config/runtime.rs index 083fbcc64a..cf47be1cba 100644 --- a/engine/packages/config/src/config/runtime.rs +++ b/engine/packages/config/src/config/runtime.rs @@ -15,6 +15,10 @@ pub struct Runtime { /// Time (in seconds) to allow for guard to wait for pending requests after receiving SIGTERM. Defaults /// to 1 hour. guard_shutdown_duration: Option, + /// Time (in seconds) after which the engine process will forcibly exit after receiving SIGTERM. + /// Must be greater than both worker_shutdown_duration and guard_shutdown_duration. + /// Defaults to guard_shutdown_duration + 30 seconds. + force_shutdown_duration: Option, /// Whether or not to allow running the engine when the previous version that was run is higher than /// the current version. allow_version_rollback: Option, @@ -29,6 +33,14 @@ impl Runtime { Duration::from_secs(self.guard_shutdown_duration.unwrap_or(60 * 60) as u64) } + /// Returns the force shutdown duration, defaulting to guard_shutdown_duration + 30 seconds. + pub fn force_shutdown_duration(&self) -> Duration { + self.force_shutdown_duration.map_or_else( + || self.guard_shutdown_duration() + Duration::from_secs(30), + |secs| Duration::from_secs(secs as u64), + ) + } + pub fn allow_version_rollback(&self) -> bool { self.allow_version_rollback.unwrap_or_default() } diff --git a/engine/packages/service-manager/src/lib.rs b/engine/packages/service-manager/src/lib.rs index 3b1883f63a..712b0bf0f9 100644 --- a/engine/packages/service-manager/src/lib.rs +++ b/engine/packages/service-manager/src/lib.rs @@ -339,6 +339,15 @@ pub async fn start( abort = term_signal.recv() => { shutting_down.store(true, Ordering::SeqCst); + // Spawn force exit task in case of a lingering task + let force_shutdown_duration = config.runtime.force_shutdown_duration(); + tokio::spawn(async move { + tracing::info!(?force_shutdown_duration, "force shutdown timer started"); + tokio::time::sleep(force_shutdown_duration).await; + tracing::warn!("force shutdown timeout reached, exiting process, this indicates a bug"); + std::process::exit(1); + }); + // Abort services that don't require graceful shutdown running_services.retain(|(requires_graceful_shutdown, handle)| { if !requires_graceful_shutdown { diff --git a/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts b/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts index 9ab2353356..055a33620d 100644 --- a/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts @@ -118,6 +118,8 @@ export async function ensureEngineProcess( // Reduce shutdown durations for faster development iteration (in seconds) RIVET__RUNTIME__WORKER_SHUTDOWN_DURATION: "1", RIVET__RUNTIME__GUARD_SHUTDOWN_DURATION: "1", + // Force exit after this duration (must be > worker and guard shutdown durations) + RIVET__RUNTIME__FORCE_SHUTDOWN_DURATION: "2", }, });