feat: DAG workflow engine (phases 2-8)

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
           components: rustfmt, clippy
 
       - name: Rust cache
-        uses: Swatinem/rust-cache@v2.8.2
+        uses: Swatinem/rust-cache@v2.9.1
 
       - name: Cargo fmt
         run: cargo fmt --all --check
@@ -39,7 +39,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.10.12"
-          cache-dependency-glob: uv.lock
+          cache-dependency-glob: pyproject.toml
 
       - name: Install Python dev deps
         run: uv sync --extra dev
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: "22"
           cache: "npm"
@@ -91,7 +91,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Rust cache
-        uses: Swatinem/rust-cache@v2.8.2
+        uses: Swatinem/rust-cache@v2.9.1
         with:
           save-if: false
 
@@ -137,15 +137,15 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Rust cache
-        uses: Swatinem/rust-cache@v2.8.2
+        uses: Swatinem/rust-cache@v2.9.1
         with:
           save-if: ${{ matrix.os != 'ubuntu-latest' }}
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.10.12"
-          cache-dependency-glob: uv.lock
+          cache-dependency-glob: pyproject.toml
 
       - name: Install dependencies
         run: uv sync --extra dev
@@ -154,7 +154,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           command: develop
-          args: --release --features native-async
+          args: --release --features extension-module,postgres,redis,native-async,workflows
 
       - name: Run Python tests
         run: |

.github/workflows/publish.yml

@@ -24,7 +24,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --features extension-module,postgres,redis --interpreter 3.10 3.11 3.12 3.13
+          args: --release --out dist --features extension-module,postgres,redis,native-async,workflows --interpreter 3.10 3.11 3.12 3.13
           manylinux: 2_28
           before-script-linux: |
             dnf install -y openssl-devel perl-core perl-IPC-Cmd
@@ -51,7 +51,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --features extension-module,postgres,redis --interpreter 3.10 3.11 3.12 3.13
+          args: --release --out dist --features extension-module,postgres,redis,native-async,workflows --interpreter 3.10 3.11 3.12 3.13
           manylinux: musllinux_1_2
           before-script-linux: |
             apt-get update && apt-get install -y libssl-dev perl pkg-config
@@ -96,7 +96,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --features extension-module,postgres,redis --interpreter 3.10 3.11 3.12 3.13
+          args: --release --out dist --features extension-module,postgres,redis,native-async,workflows --interpreter 3.10 3.11 3.12 3.13
 
       - name: Upload wheels
         uses: actions/upload-artifact@v7
@@ -137,7 +137,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --features extension-module,postgres,redis --interpreter 3.10 3.11 3.12 3.13
+          args: --release --out dist --features extension-module,postgres,redis,native-async,workflows --interpreter 3.10 3.11 3.12 3.13
 
       - name: Upload wheels
         uses: actions/upload-artifact@v7

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["crates/taskito-core", "crates/taskito-python", "crates/taskito-async"]
+members = ["crates/taskito-core", "crates/taskito-python", "crates/taskito-async", "crates/taskito-workflows"]
 resolver = "2"
 
 [workspace.dependencies]
@@ -21,3 +21,4 @@ redis = { version = "0.27", features = ["script"] }
 openssl-sys = { version = "0.9", features = ["vendored"] }
 pyo3 = { version = "0.22", features = ["multiple-pymethods"] }
 async-trait = "0.1"
+dagron-core = { git = "https://github.com/ByteVeda/dagron.git" }

crates/taskito-async/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-async"
-version = "0.10.1"
+version = "0.11.0"
 edition = "2021"
 
 [dependencies]

crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.10.1"
+version = "0.11.0"
 edition = "2021"
 
 [features]

crates/taskito-core/src/storage/postgres/mod.rs

@@ -98,9 +98,7 @@ impl PostgresStorage {
         Ok(storage)
     }
 
-    pub(crate) fn conn(
-        &self,
-    ) -> Result<diesel::r2d2::PooledConnection<ConnectionManager<PgConnection>>> {
+    pub fn conn(&self) -> Result<diesel::r2d2::PooledConnection<ConnectionManager<PgConnection>>> {
         let mut conn = self.pool.get()?;
         diesel::sql_query(format!("SET search_path TO {}", self.schema))
             .execute(&mut conn)

crates/taskito-core/src/storage/redis_backend/mod.rs

@@ -58,7 +58,7 @@ impl RedisStorage {
     }
 
     /// Get a Redis connection.
-    fn conn(&self) -> Result<redis::Connection> {
+    pub fn conn(&self) -> Result<redis::Connection> {
         self.client
             .get_connection()
             .map_err(|e| QueueError::Other(format!("Redis connection error: {e}")))

crates/taskito-core/src/storage/sqlite/mod.rs

@@ -95,7 +95,7 @@ impl SqliteStorage {
         Ok(storage)
     }
 
-    pub(crate) fn conn(
+    pub fn conn(
         &self,
     ) -> Result<diesel::r2d2::PooledConnection<ConnectionManager<SqliteConnection>>> {
         Ok(self.pool.get()?)

crates/taskito-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-python"
-version = "0.10.1"
+version = "0.11.0"
 edition = "2021"
 
 [features]
@@ -9,6 +9,7 @@ extension-module = ["pyo3/extension-module"]
 postgres = ["taskito-core/postgres"]
 redis = ["taskito-core/redis"]
 native-async = ["dep:taskito-async"]
+workflows = ["dep:taskito-workflows"]
 
 [lib]
 name = "_taskito"
@@ -22,6 +23,7 @@ crossbeam-channel = { workspace = true }
 uuid = { workspace = true }
 async-trait = { workspace = true }
 taskito-async = { path = "../taskito-async", optional = true }
+taskito-workflows = { path = "../taskito-workflows", optional = true }
 serde_json = { workspace = true }
 serde = { workspace = true }
 base64 = "0.22"

crates/taskito-python/src/lib.rs

@@ -7,6 +7,8 @@ mod py_config;
 mod py_job;
 mod py_queue;
 pub mod py_worker;
+#[cfg(feature = "workflows")]
+mod py_workflow;
 
 use py_config::PyTaskConfig;
 use py_job::PyJob;
@@ -19,5 +21,11 @@ fn _taskito(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyTaskConfig>()?;
     #[cfg(feature = "native-async")]
     m.add_class::<taskito_async::PyResultSender>()?;
+    #[cfg(feature = "workflows")]
+    {
+        m.add_class::<py_workflow::PyWorkflowBuilder>()?;
+        m.add_class::<py_workflow::PyWorkflowHandle>()?;
+        m.add_class::<py_workflow::PyWorkflowRunStatus>()?;
+    }
     Ok(())
 }

crates/taskito-python/src/py_queue/mod.rs

@@ -3,6 +3,8 @@
 
 mod inspection;
 mod worker;
+#[cfg(feature = "workflows")]
+mod workflow_ops;
 
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;