From 72081272ccfdc4110a43253f8f4e85d0dc562cb9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 11:43:55 -0700 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20add=20RawStepChain=20serde=20types?= =?UTF-8?q?=20for=20Python=E2=86=92Rust=20step=20chain=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 + crates/hm-dsl-engine/Cargo.toml | 2 + crates/hm-dsl-engine/src/lib.rs | 1 + crates/hm-dsl-engine/src/step_chain.rs | 78 ++++++++ .../hm-dsl-engine/tests/step_chain_deser.rs | 167 ++++++++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 crates/hm-dsl-engine/src/step_chain.rs create mode 100644 crates/hm-dsl-engine/tests/step_chain_deser.rs diff --git a/Cargo.lock b/Cargo.lock index 217d31c1..3689fcb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1242,9 +1242,11 @@ dependencies = [ "anyhow", "async-trait", "derive_more", + "hm-pipeline-ir", "include_dir", "serde", "serde_json", + "sha2", "tempfile", "tokio", "tracing", diff --git a/crates/hm-dsl-engine/Cargo.toml b/crates/hm-dsl-engine/Cargo.toml index 4d0eb80c..56e8f348 100644 --- a/crates/hm-dsl-engine/Cargo.toml +++ b/crates/hm-dsl-engine/Cargo.toml @@ -12,11 +12,13 @@ categories = ["command-line-utilities"] path = "src/lib.rs" [dependencies] +hm-pipeline-ir = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } derive_more = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = "0.10" include_dir = "0.7" tempfile = "3" tokio = { version = "1", features = ["process", "fs"] } diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index 6430b520..00b8b30c 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use serde::Deserialize; pub mod detect; +pub mod step_chain; mod python_engine; pub use python_engine::{SubprocessPythonEngine, engine as python_engine}; diff --git a/crates/hm-dsl-engine/src/step_chain.rs b/crates/hm-dsl-engine/src/step_chain.rs new file mode 100644 index 00000000..dc10d265 --- /dev/null +++ b/crates/hm-dsl-engine/src/step_chain.rs @@ -0,0 +1,78 @@ +//! Raw step-chain wire format emitted by the Python DSL. +//! +//! Python serializes each pipeline's `Step` chain into a [`RawStepChain`]: a +//! flat list of steps that reference their parents by index, plus the indices +//! of the chain's leaves and pipeline-level metadata. Rust lowers this into the +//! canonical [`hm_pipeline_ir::PipelineGraph`]. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// A pipeline's step chain as emitted by the Python DSL. +/// +/// Steps form a forest referenced by index: each [`RawStep::parent_idx`] points +/// at an earlier entry in [`steps`](Self::steps), and [`leaf_indices`](Self::leaf_indices) +/// names the terminal steps. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RawStepChain { + pub steps: Vec, + pub leaf_indices: Vec, + #[serde(default)] + pub pipeline_env: Option>, + #[serde(default)] + pub pipeline_timeout_seconds: Option, +} + +/// A single step in a [`RawStepChain`]. +/// +/// A step is either a command (`cmd` set) or a `wait` barrier (`is_wait`). +/// `parent_idx` is `None` for a step that boots from a fresh image (a root). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RawStep { + pub cmd: Option, + pub parent_idx: Option, + #[serde(default)] + pub is_wait: bool, + #[serde(default)] + pub continue_on_failure: bool, + #[serde(default)] + pub label: Option, + #[serde(default)] + pub cache: Option, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub timeout_seconds: Option, + #[serde(default)] + pub image: Option, + #[serde(default)] + pub runner: Option, + #[serde(default)] + pub runner_args: Option, + #[serde(default)] + pub key_override: Option, +} + +/// Cache policy for a step, tagged by its `policy` field on the wire. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "policy")] +pub enum RawCachePolicy { + #[serde(rename = "none")] + None, + #[serde(rename = "forever")] + Forever { + #[serde(default)] + env_keys: Vec, + }, + #[serde(rename = "ttl")] + Ttl { + duration_seconds: u64, + #[serde(default)] + env_keys: Vec, + }, + #[serde(rename = "on_change")] + OnChange { paths: Vec }, + #[serde(rename = "compose")] + Compose { sub_policies: Vec }, +} diff --git a/crates/hm-dsl-engine/tests/step_chain_deser.rs b/crates/hm-dsl-engine/tests/step_chain_deser.rs new file mode 100644 index 00000000..a0ea233d --- /dev/null +++ b/crates/hm-dsl-engine/tests/step_chain_deser.rs @@ -0,0 +1,167 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use hm_dsl_engine::step_chain::{RawCachePolicy, RawStepChain}; + +#[test] +fn minimal_scratch_plus_command() { + let json = r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": "make build", "parent_idx": 0} + ], + "leaf_indices": [1] + }"#; + let chain: RawStepChain = serde_json::from_str(json).unwrap(); + assert_eq!(chain.steps.len(), 2); + assert_eq!(chain.leaf_indices, vec![1]); + assert!(chain.steps[0].cmd.is_none()); + assert!(chain.steps[0].parent_idx.is_none()); + assert_eq!(chain.steps[1].cmd.as_deref(), Some("make build")); + assert_eq!(chain.steps[1].parent_idx, Some(0)); + assert!(chain.pipeline_env.is_none()); + assert!(chain.pipeline_timeout_seconds.is_none()); +} + +#[test] +fn step_with_all_fields_populated() { + let json = r#"{ + "steps": [{ + "cmd": "pytest", + "parent_idx": 3, + "is_wait": false, + "continue_on_failure": true, + "label": "run tests", + "cache": {"policy": "none"}, + "env": {"CI": "1", "FOO": "bar"}, + "timeout_seconds": 600, + "image": "python:3.12", + "runner": "docker", + "runner_args": {"privileged": true}, + "key_override": "custom-key" + }], + "leaf_indices": [0] + }"#; + let chain: RawStepChain = serde_json::from_str(json).unwrap(); + let step = &chain.steps[0]; + assert_eq!(step.cmd.as_deref(), Some("pytest")); + assert_eq!(step.parent_idx, Some(3)); + assert!(!step.is_wait); + assert!(step.continue_on_failure); + assert_eq!(step.label.as_deref(), Some("run tests")); + assert!(matches!(step.cache, Some(RawCachePolicy::None))); + let env = step.env.as_ref().unwrap(); + assert_eq!(env.get("CI").map(String::as_str), Some("1")); + assert_eq!(env.get("FOO").map(String::as_str), Some("bar")); + assert_eq!(step.timeout_seconds, Some(600)); + assert_eq!(step.image.as_deref(), Some("python:3.12")); + assert_eq!(step.runner.as_deref(), Some("docker")); + assert_eq!(step.runner_args.as_ref().unwrap()["privileged"], true); + assert_eq!(step.key_override.as_deref(), Some("custom-key")); +} + +#[test] +fn cache_policy_none() { + let step = single_step_with_cache(r#"{"policy": "none"}"#); + assert!(matches!(step, RawCachePolicy::None)); +} + +#[test] +fn cache_policy_forever() { + let step = single_step_with_cache(r#"{"policy": "forever", "env_keys": ["A", "B"]}"#); + match step { + RawCachePolicy::Forever { env_keys } => assert_eq!(env_keys, vec!["A", "B"]), + other => panic!("expected Forever, got {other:?}"), + } +} + +#[test] +fn cache_policy_forever_defaults_env_keys() { + let step = single_step_with_cache(r#"{"policy": "forever"}"#); + match step { + RawCachePolicy::Forever { env_keys } => assert!(env_keys.is_empty()), + other => panic!("expected Forever, got {other:?}"), + } +} + +#[test] +fn cache_policy_ttl() { + let step = + single_step_with_cache(r#"{"policy": "ttl", "duration_seconds": 3600, "env_keys": ["X"]}"#); + match step { + RawCachePolicy::Ttl { + duration_seconds, + env_keys, + } => { + assert_eq!(duration_seconds, 3600); + assert_eq!(env_keys, vec!["X"]); + } + other => panic!("expected Ttl, got {other:?}"), + } +} + +#[test] +fn cache_policy_on_change() { + let step = + single_step_with_cache(r#"{"policy": "on_change", "paths": ["src/**", "Cargo.toml"]}"#); + match step { + RawCachePolicy::OnChange { paths } => assert_eq!(paths, vec!["src/**", "Cargo.toml"]), + other => panic!("expected OnChange, got {other:?}"), + } +} + +#[test] +fn cache_policy_compose() { + let step = single_step_with_cache( + r#"{"policy": "compose", "sub_policies": [ + {"policy": "forever"}, + {"policy": "on_change", "paths": ["a"]} + ]}"#, + ); + match step { + RawCachePolicy::Compose { sub_policies } => { + assert_eq!(sub_policies.len(), 2); + assert!(matches!(sub_policies[0], RawCachePolicy::Forever { .. })); + assert!(matches!(sub_policies[1], RawCachePolicy::OnChange { .. })); + } + other => panic!("expected Compose, got {other:?}"), + } +} + +#[test] +fn wait_step_with_continue_on_failure() { + let json = r#"{ + "steps": [ + {"cmd": "a", "parent_idx": null}, + {"cmd": "b", "parent_idx": null}, + {"cmd": null, "parent_idx": 1, "is_wait": true, "continue_on_failure": true} + ], + "leaf_indices": [2] + }"#; + let chain: RawStepChain = serde_json::from_str(json).unwrap(); + let wait = &chain.steps[2]; + assert!(wait.is_wait); + assert!(wait.continue_on_failure); + assert!(wait.cmd.is_none()); +} + +#[test] +fn pipeline_level_env_and_timeout() { + let json = r#"{ + "steps": [{"cmd": "build", "parent_idx": null}], + "leaf_indices": [0], + "pipeline_env": {"GLOBAL": "yes"}, + "pipeline_timeout_seconds": 1800 + }"#; + let chain: RawStepChain = serde_json::from_str(json).unwrap(); + let env = chain.pipeline_env.as_ref().unwrap(); + assert_eq!(env.get("GLOBAL").map(String::as_str), Some("yes")); + assert_eq!(chain.pipeline_timeout_seconds, Some(1800)); +} + +fn single_step_with_cache(cache_json: &str) -> RawCachePolicy { + let json = format!( + r#"{{"steps": [{{"cmd": "x", "parent_idx": null, "cache": {cache_json}}}], "leaf_indices": [0]}}"# + ); + let chain: RawStepChain = serde_json::from_str(&json).unwrap(); + chain.steps.into_iter().next().unwrap().cache.unwrap() +} From 1414fbe0d6189309c0fdc771847ef52c05bdafe5 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:33:07 -0700 Subject: [PATCH 2/9] feat: implement step chain lowering pass in Rust --- Cargo.lock | 1 + crates/hm-dsl-engine/Cargo.toml | 1 + crates/hm-dsl-engine/src/lib.rs | 1 + crates/hm-dsl-engine/src/lower.rs | 326 +++++++++++++++++ crates/hm-dsl-engine/tests/lower_test.rs | 436 +++++++++++++++++++++++ 5 files changed, 765 insertions(+) create mode 100644 crates/hm-dsl-engine/src/lower.rs create mode 100644 crates/hm-dsl-engine/tests/lower_test.rs diff --git a/Cargo.lock b/Cargo.lock index 3689fcb9..8260ce3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,6 +1241,7 @@ version = "0.0.0-dev" dependencies = [ "anyhow", "async-trait", + "daggy", "derive_more", "hm-pipeline-ir", "include_dir", diff --git a/crates/hm-dsl-engine/Cargo.toml b/crates/hm-dsl-engine/Cargo.toml index 56e8f348..395c42e3 100644 --- a/crates/hm-dsl-engine/Cargo.toml +++ b/crates/hm-dsl-engine/Cargo.toml @@ -26,6 +26,7 @@ tracing = "0.1" which = "7" [dev-dependencies] +daggy = { workspace = true } tempfile = "3" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } which = "7" diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index 00b8b30c..7152ea78 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use serde::Deserialize; pub mod detect; +pub mod lower; pub mod step_chain; mod python_engine; diff --git a/crates/hm-dsl-engine/src/lower.rs b/crates/hm-dsl-engine/src/lower.rs new file mode 100644 index 00000000..990c7104 --- /dev/null +++ b/crates/hm-dsl-engine/src/lower.rs @@ -0,0 +1,326 @@ +//! Lowering pass: [`RawStepChain`] → canonical v0 [`PipelineGraph`]. +//! +//! Ports the Python lowering (`harmont-py/harmont/_pipeline.py` and +//! `_keys.py`) into Rust. The chain is a forest of steps referenced by +//! index; this walks it back from each leaf, topo-sorts parent-before-child, +//! drops scratch/fork passthrough nodes and `wait` barriers, resolves each +//! command step's cross-reference key, and emits the petgraph-serde graph the +//! IR deserializes from. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt::Write as _; + +use anyhow::{Context, Result}; +use hm_pipeline_ir::PipelineGraph; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; + +use crate::step_chain::{RawCachePolicy, RawStepChain}; + +/// Across-the-board default image for imageless root steps. The SDK's +/// toolchains assume an apt-capable base, so `ubuntu:24.04` is the universal +/// default; child steps boot from their parent's snapshot and stay imageless. +const DEFAULT_IMAGE: &str = "ubuntu:24.04"; + +/// Lower a raw step chain into the canonical [`PipelineGraph`]. +/// +/// # Errors +/// +/// Returns an error if the emitted graph fails to deserialize into a +/// [`PipelineGraph`] — e.g. a step declares a zero-second timeout, which the +/// IR rejects at the wire boundary. +pub fn lower(chain: &RawStepChain) -> Result { + let ordered = topo_collect(chain); + let command_steps: Vec = ordered + .iter() + .copied() + .filter(|&i| chain.steps[i].cmd.is_some() && !chain.steps[i].is_wait) + .collect(); + let keys = resolve_keys(chain, &command_steps); + + // Dense node indices in emission order, keyed by raw step index. + let idx_by_raw: HashMap = command_steps + .iter() + .enumerate() + .map(|(node_idx, &raw)| (raw, node_idx)) + .collect(); + + let mut nodes: Vec = Vec::with_capacity(command_steps.len()); + let mut edges: Vec = Vec::new(); + + // Command-step node indices emitted since the last `wait` barrier, and the + // sources carried over from the most recent barrier. + let mut pre_wait_indices: Vec = Vec::new(); + let mut pending_depends_on: Vec = Vec::new(); + + for &raw in &ordered { + let step = &chain.steps[raw]; + if step.is_wait { + pending_depends_on = std::mem::take(&mut pre_wait_indices); + continue; + } + let Some(cmd) = &step.cmd else { + // scratch or fork — passthrough, not emitted as a node. + continue; + }; + + let node_idx = idx_by_raw[&raw]; + let parent = resolved_parent_idx(chain, raw); + + let mut step_dict = Map::new(); + step_dict.insert("key".into(), Value::String(keys[&raw].clone())); + step_dict.insert("cmd".into(), Value::String(cmd.clone())); + if let Some(label) = &step.label { + step_dict.insert("label".into(), Value::String(label.clone())); + } + if let Some(cache) = &step.cache { + step_dict.insert("cache".into(), cache_to_value(cache)); + } + if let Some(timeout) = step.timeout_seconds { + step_dict.insert("timeout_seconds".into(), Value::from(timeout)); + } + if let Some(image) = &step.image { + step_dict.insert("image".into(), Value::String(image.clone())); + } + if let Some(runner) = &step.runner { + step_dict.insert("runner".into(), Value::String(runner.clone())); + } + if let Some(runner_args) = &step.runner_args { + step_dict.insert("runner_args".into(), runner_args.clone()); + } + + // Root command steps (no builds_in parent) that declare no image get + // the pipeline-wide default; child steps inherit the parent snapshot. + if parent.is_none() && !step_dict.contains_key("image") { + step_dict.insert("image".into(), Value::String(DEFAULT_IMAGE.into())); + } + + nodes.push(json!({"step": Value::Object(step_dict), "env": merged_env(chain, raw)})); + + if let Some(parent_raw) = parent { + edges.push(json!([idx_by_raw[&parent_raw], node_idx, "builds_in"])); + } + for &dep in &pending_depends_on { + edges.push(json!([dep, node_idx, "depends_on"])); + } + pre_wait_indices.push(node_idx); + } + + let mut top = Map::new(); + top.insert("version".into(), Value::String("0".into())); + if let Some(timeout) = chain.pipeline_timeout_seconds { + top.insert("timeout_seconds".into(), Value::from(timeout)); + } + top.insert( + "graph".into(), + json!({ + "nodes": nodes, + "node_holes": [], + "edge_property": "directed", + "edges": edges, + }), + ); + + serde_json::from_value(Value::Object(top)) + .context("failed to deserialize lowered pipeline graph") +} + +/// Merge env in layers: non-interactive baseline, then pipeline-level, then +/// per-step overrides. +fn merged_env(chain: &RawStepChain, raw: usize) -> BTreeMap { + let mut env = BTreeMap::from([ + ("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string()), + ("TERM".to_string(), "dumb".to_string()), + ]); + if let Some(pipeline_env) = &chain.pipeline_env { + env.extend(pipeline_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + if let Some(step_env) = &chain.steps[raw].env { + env.extend(step_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + env +} + +/// Collect every step reachable from `leaves` via `parent_idx`, in +/// parent-before-child order. Tiebreak by leaf order, then DFS-pre on each leaf +/// chain. `wait` leaves are inserted at their leaf position. +fn topo_collect(chain: &RawStepChain) -> Vec { + let mut seen: HashSet = HashSet::new(); + let mut ordered: Vec = Vec::new(); + + for &leaf in &chain.leaf_indices { + if chain.steps[leaf].is_wait { + ordered.push(leaf); + continue; + } + // Walk leaf -> root, stopping at the first already-seen ancestor. + let mut walk: Vec = Vec::new(); + let mut node = Some(leaf); + while let Some(n) = node { + if seen.contains(&n) { + break; + } + walk.push(n); + node = chain.steps[n].parent_idx; + } + for &s in walk.iter().rev() { + if seen.insert(s) { + ordered.push(s); + } + } + } + ordered +} + +/// Walk back through scratch/fork nodes to the nearest emitted command +/// ancestor, returning its raw index (the `builds_in` parent). +fn resolved_parent_idx(chain: &RawStepChain, raw: usize) -> Option { + let mut node = chain.steps[raw].parent_idx; + while let Some(n) = node { + let step = &chain.steps[n]; + if step.cmd.is_some() && !step.is_wait { + return Some(n); + } + node = step.parent_idx; + } + None +} + +/// Render a raw cache policy to its IR `Cache` shape. Only the policy name +/// survives here; cache-key resolution is a separate pass. +fn cache_to_value(cache: &RawCachePolicy) -> Value { + let policy = match cache { + RawCachePolicy::None => "none", + RawCachePolicy::Forever { .. } => "forever", + RawCachePolicy::Ttl { .. } => "ttl", + RawCachePolicy::OnChange { .. } => "on_change", + RawCachePolicy::Compose { .. } => "compose", + }; + json!({ "policy": policy }) +} + +/// Resolve each command step's cross-reference key, keyed by raw index. +/// +/// Precedence: explicit `key_override`, then a unique slugified label, then a +/// stable hash of `(parent_key, cmd, position)`. When two steps' natural slugs +/// collide and neither claimed it via override, both fall back to hash; an +/// override reserves its string even against a peer's identical natural slug. +fn resolve_keys(chain: &RawStepChain, command_steps: &[usize]) -> HashMap { + let mut overrides: HashMap = HashMap::new(); + let mut natural_slugs: HashMap = HashMap::new(); + for &raw in command_steps { + let step = &chain.steps[raw]; + if let Some(over) = &step.key_override { + overrides.insert(raw, over.clone()); + } + if let Some(label) = &step.label { + let slug = slugify_label(label); + if !slug.is_empty() { + natural_slugs.insert(raw, slug); + } + } + } + + // Every override reserves its string; a natural slug matching a reserved + // override collides for the slug claimant. + let mut reserved: HashSet = overrides.values().cloned().collect(); + + // Slug collision counts span every labeled step, including override-bearing + // ones — an override step still "claims" its natural slug. + let mut slug_counts: HashMap<&str, usize> = HashMap::new(); + for slug in natural_slugs.values() { + *slug_counts.entry(slug.as_str()).or_insert(0) += 1; + } + + let mut keys: HashMap = HashMap::new(); + for (position, &raw) in command_steps.iter().enumerate() { + if let Some(over) = overrides.get(&raw) { + keys.insert(raw, over.clone()); + continue; + } + if let Some(slug) = natural_slugs.get(&raw) + && !reserved.contains(slug) + && slug_counts.get(slug.as_str()) == Some(&1) + { + reserved.insert(slug.clone()); + keys.insert(raw, slug.clone()); + continue; + } + // Hash fallback. The parent key is the direct parent's key only when + // that parent is itself an already-resolved command step. + let parent_key = chain.steps[raw] + .parent_idx + .and_then(|pidx| keys.get(&pidx)) + .map_or("", String::as_str); + let cmd = chain.steps[raw].cmd.as_deref().unwrap_or_default(); + keys.insert(raw, hash_key(parent_key, cmd, position)); + } + keys +} + +/// Lowercase, strip `:emoji_codes:`, collapse non-alphanumeric runs to `-`, +/// and trim leading/trailing dashes. Non-ASCII characters are separators, so +/// slugs are ASCII-only; a label that reduces to empty falls back to a hash. +fn slugify_label(label: &str) -> String { + let lowered = label.to_lowercase(); + let stripped = strip_emoji_shortcodes(&lowered); + + let mut out = String::with_capacity(stripped.len()); + let mut pending_dash = false; + for c in stripped.chars() { + if c.is_ascii_lowercase() || c.is_ascii_digit() { + out.push(c); + pending_dash = false; + } else if !pending_dash { + pending_dash = true; + out.push('-'); + } + } + out.trim_matches('-').to_string() +} + +/// Replace `:[a-z0-9_+-]+:` shortcodes with a space (matching Python's +/// `re.sub` on the already-lowercased label). +fn strip_emoji_shortcodes(s: &str) -> String { + let chars: Vec = s.chars().collect(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < chars.len() { + if chars[i] == ':' { + let mut j = i + 1; + while j < chars.len() && is_shortcode_char(chars[j]) { + j += 1; + } + // `+` needs at least one inner char, and a closing colon. + if j > i + 1 && chars.get(j) == Some(&':') { + out.push(' '); + i = j + 1; + continue; + } + } + out.push(chars[i]); + i += 1; + } + out +} + +const fn is_shortcode_char(c: char) -> bool { + c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '+' | '-') +} + +/// Stable 12-char SHA-256 prefix over `(parent_key, cmd, position)`. +fn hash_key(parent_key: &str, cmd: &str, position: usize) -> String { + let mut hasher = Sha256::new(); + hasher.update(parent_key.as_bytes()); + hasher.update([0u8]); + hasher.update(cmd.as_bytes()); + hasher.update([0u8]); + hasher.update(position.to_string().as_bytes()); + let digest = hasher.finalize(); + + let mut out = String::with_capacity(12); + for byte in digest.iter().take(6) { + let _ = write!(out, "{byte:02x}"); + } + out +} diff --git a/crates/hm-dsl-engine/tests/lower_test.rs b/crates/hm-dsl-engine/tests/lower_test.rs new file mode 100644 index 00000000..fa434a85 --- /dev/null +++ b/crates/hm-dsl-engine/tests/lower_test.rs @@ -0,0 +1,436 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::{BTreeMap, BTreeSet}; + +use daggy::petgraph::visit::{EdgeRef, IntoNodeReferences}; +use hm_dsl_engine::lower::lower; +use hm_dsl_engine::step_chain::RawStepChain; +use hm_pipeline_ir::{EdgeKind, PipelineGraph}; + +const fn kind_str(kind: EdgeKind) -> &'static str { + match kind { + EdgeKind::BuildsIn => "builds_in", + EdgeKind::DependsOn => "depends_on", + } +} + +fn chain(json: &str) -> RawStepChain { + serde_json::from_str(json).expect("parse RawStepChain") +} + +/// Map of `key -> Transition`-ish accessors, keyed for order-independent asserts. +fn nodes_by_key(g: &PipelineGraph) -> BTreeMap { + g.dag() + .graph() + .node_references() + .map(|(_, t)| { + ( + t.step.key.clone(), + NodeView { + image: t.step.image.clone(), + label: t.step.label.clone(), + cmd: t.step.cmd.clone(), + env: t.env.clone(), + cache_policy: t.step.cache.as_ref().map(|c| c.policy.clone()), + }, + ) + }) + .collect() +} + +struct NodeView { + image: Option, + label: Option, + cmd: String, + env: BTreeMap, + cache_policy: Option, +} + +/// `(source_key, target_key, kind)` for every edge. +fn edges_by_key(g: &PipelineGraph) -> BTreeSet<(String, String, &'static str)> { + let dag = g.dag(); + let graph = dag.graph(); + let key = |idx| graph.node_weight(idx).unwrap().step.key.clone(); + graph + .edge_references() + .map(|e| (key(e.source()), key(e.target()), kind_str(*e.weight()))) + .collect() +} + +fn keys(g: &PipelineGraph) -> BTreeSet { + nodes_by_key(g).into_keys().collect() +} + +#[test] +fn linear_chain_builds_in_edge() { + // scratch -> a -> b + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": "echo a", "parent_idx": 0, "label": "a"}, + {"cmd": "echo b", "parent_idx": 1, "label": "b"} + ], + "leaf_indices": [2] + }"#, + )) + .unwrap(); + + assert_eq!(g.node_count(), 2); + let nodes = nodes_by_key(&g); + // Root a is imageless -> stamped; child b inherits parent snapshot. + assert_eq!(nodes["a"].image.as_deref(), Some("ubuntu:24.04")); + assert_eq!(nodes["b"].image, None); + assert_eq!(nodes["a"].cmd, "echo a"); + + assert_eq!( + edges_by_key(&g), + BTreeSet::from([("a".into(), "b".into(), "builds_in")]) + ); +} + +#[test] +fn fork_produces_two_independent_roots() { + // scratch -> a, scratch -> b + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": "echo a", "parent_idx": 0, "label": "a"}, + {"cmd": "echo b", "parent_idx": 0, "label": "b"} + ], + "leaf_indices": [1, 2] + }"#, + )) + .unwrap(); + + assert_eq!(g.node_count(), 2); + let nodes = nodes_by_key(&g); + assert_eq!(nodes["a"].image.as_deref(), Some("ubuntu:24.04")); + assert_eq!(nodes["b"].image.as_deref(), Some("ubuntu:24.04")); + assert!(edges_by_key(&g).is_empty(), "forked roots share no edges"); +} + +#[test] +fn wait_barrier_creates_depends_on() { + // pipeline([a, wait, b]): wait sits between two roots in leaf order. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "echo a", "parent_idx": null, "label": "a"}, + {"cmd": null, "parent_idx": null, "is_wait": true}, + {"cmd": "echo b", "parent_idx": null, "label": "b"} + ], + "leaf_indices": [0, 1, 2] + }"#, + )) + .unwrap(); + + assert_eq!(g.node_count(), 2); + assert_eq!( + edges_by_key(&g), + BTreeSet::from([("a".into(), "b".into(), "depends_on")]) + ); +} + +#[test] +fn wait_fans_out_and_in() { + // pipeline([a, b, wait, c, d]): a,b before the barrier; c,d after. + // Every post-wait step depends on every pre-wait step. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "a", "parent_idx": null, "label": "a"}, + {"cmd": "b", "parent_idx": null, "label": "b"}, + {"cmd": null, "parent_idx": null, "is_wait": true}, + {"cmd": "c", "parent_idx": null, "label": "c"}, + {"cmd": "d", "parent_idx": null, "label": "d"} + ], + "leaf_indices": [0, 1, 2, 3, 4] + }"#, + )) + .unwrap(); + + assert_eq!( + edges_by_key(&g), + BTreeSet::from([ + ("a".into(), "c".into(), "depends_on"), + ("a".into(), "d".into(), "depends_on"), + ("b".into(), "c".into(), "depends_on"), + ("b".into(), "d".into(), "depends_on"), + ]) + ); +} + +#[test] +fn explicit_key_override_wins() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "x", "parent_idx": null, "label": "Nice Label", "key_override": "custom"} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + assert_eq!(keys(&g), BTreeSet::from(["custom".to_string()])); +} + +#[test] +fn slug_derived_from_label() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "x", "parent_idx": null, "label": ":rocket: Build & Test!"} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + // ":rocket:" stripped, "&"/"!"/spaces collapse to dashes, trimmed. + assert_eq!(keys(&g), BTreeSet::from(["build-test".to_string()])); +} + +#[test] +fn labelless_step_gets_hash_key() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "make", "parent_idx": null} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + let k = keys(&g).into_iter().next().unwrap(); + assert_eq!(k.len(), 12, "hash key is a 12-char hex prefix"); + assert!(k.chars().all(|c| c.is_ascii_hexdigit())); + // Byte-exact match with Python's hash_key("", "make", 0). + assert_eq!(k, "9f3f33e47e82"); +} + +#[test] +fn empty_slug_falls_back_to_hash() { + // A label that slugifies to "" (non-ASCII only) must not yield an empty key. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "build", "parent_idx": null, "label": "构建"} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + let k = keys(&g).into_iter().next().unwrap(); + assert_eq!(k.len(), 12); + let nodes = nodes_by_key(&g); + // Display label is preserved even though the key is hash-based. + assert_eq!(nodes[&k].label.as_deref(), Some("构建")); +} + +#[test] +fn slug_collision_both_fall_back_to_hash() { + // Two steps share a label; neither claimed it via override -> both hash. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "one", "parent_idx": null, "label": "Test"}, + {"cmd": "two", "parent_idx": null, "label": "Test"} + ], + "leaf_indices": [0, 1] + }"#, + )) + .unwrap(); + let ks = keys(&g); + assert!(!ks.contains("test"), "colliding slug must not be claimed"); + assert_eq!(ks.len(), 2, "distinct hash keys"); + assert!(ks.iter().all(|k| k.len() == 12)); +} + +#[test] +fn override_reserves_slug_against_peer() { + // A step whose natural slug is "test" but key=override still "claims" + // "test"; a peer with label "Test" cannot take the slug and hashes. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "one", "parent_idx": null, "label": "Test", "key_override": "explicit"}, + {"cmd": "two", "parent_idx": null, "label": "Test"} + ], + "leaf_indices": [0, 1] + }"#, + )) + .unwrap(); + let ks = keys(&g); + assert!(ks.contains("explicit")); + assert!(!ks.contains("test"), "peer cannot claim the reserved slug"); + // The peer got a hash, not the slug. + let peer = ks.iter().find(|k| *k != "explicit").unwrap(); + assert_eq!(peer.len(), 12); +} + +#[test] +fn natural_slug_matching_reserved_override_collides() { + // Step 0 overrides its key to "build"; step 1's natural slug is also + // "build" -> reserved, so step 1 hashes. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "one", "parent_idx": null, "key_override": "build"}, + {"cmd": "two", "parent_idx": null, "label": "Build"} + ], + "leaf_indices": [0, 1] + }"#, + )) + .unwrap(); + let ks = keys(&g); + assert!(ks.contains("build")); + let other = ks.iter().find(|k| *k != "build").unwrap(); + assert_eq!(other.len(), 12, "natural slug lost to the reserved override"); +} + +#[test] +fn env_merges_baseline_pipeline_and_step_layers() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "x", "parent_idx": null, "label": "a", + "env": {"TERM": "xterm", "STEP": "1"}} + ], + "leaf_indices": [0], + "pipeline_env": {"CI": "true", "STEP": "0"} + }"#, + )) + .unwrap(); + let env = &nodes_by_key(&g)["a"].env; + assert_eq!(env.get("DEBIAN_FRONTEND").map(String::as_str), Some("noninteractive")); + assert_eq!(env.get("CI").map(String::as_str), Some("true")); + // Step layer overrides both baseline (TERM) and pipeline (STEP). + assert_eq!(env.get("TERM").map(String::as_str), Some("xterm")); + assert_eq!(env.get("STEP").map(String::as_str), Some("1")); +} + +#[test] +fn scratch_and_fork_nodes_are_not_emitted() { + // scratch(0) -> fork(1) -> a(2); fork nodes carry no cmd. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": null, "parent_idx": 0}, + {"cmd": "echo a", "parent_idx": 1, "label": "a"} + ], + "leaf_indices": [2] + }"#, + )) + .unwrap(); + assert_eq!(g.node_count(), 1); + assert_eq!(keys(&g), BTreeSet::from(["a".to_string()])); + // a walks back through both passthrough nodes to no command ancestor, + // so it is a root and gets the default image. + assert_eq!(nodes_by_key(&g)["a"].image.as_deref(), Some("ubuntu:24.04")); +} + +#[test] +fn builds_in_walks_through_scratch_nodes() { + // a(1) -> scratch(2) -> b(3): b's nearest command ancestor is a. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": "echo a", "parent_idx": 0, "label": "a"}, + {"cmd": null, "parent_idx": 1}, + {"cmd": "echo b", "parent_idx": 2, "label": "b"} + ], + "leaf_indices": [3] + }"#, + )) + .unwrap(); + assert_eq!(g.node_count(), 2); + assert_eq!( + edges_by_key(&g), + BTreeSet::from([("a".into(), "b".into(), "builds_in")]) + ); + // b has a command ancestor -> imageless (inherits snapshot). + assert_eq!(nodes_by_key(&g)["b"].image, None); +} + +#[test] +fn explicit_image_on_root_is_not_overwritten() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "x", "parent_idx": null, "label": "a", "image": "alpine:3"} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + assert_eq!(nodes_by_key(&g)["a"].image.as_deref(), Some("alpine:3")); +} + +#[test] +fn cache_policy_lowered_to_name() { + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": "x", "parent_idx": null, "label": "a", + "cache": {"policy": "ttl", "duration_seconds": 3600, "env_keys": ["K"]}} + ], + "leaf_indices": [0] + }"#, + )) + .unwrap(); + assert_eq!(nodes_by_key(&g)["a"].cache_policy.as_deref(), Some("ttl")); +} + +#[test] +fn pipeline_timeout_is_carried_through() { + let g = lower(&chain( + r#"{ + "steps": [{"cmd": "x", "parent_idx": null, "label": "a"}], + "leaf_indices": [0], + "pipeline_timeout_seconds": 1800 + }"#, + )) + .unwrap(); + assert_eq!(g.timeout_seconds().map(std::num::NonZeroU32::get), Some(1800)); +} + +#[test] +fn zero_step_timeout_is_rejected_at_wire_boundary() { + let err = lower(&chain( + r#"{ + "steps": [{"cmd": "x", "parent_idx": null, "label": "a", "timeout_seconds": 0}], + "leaf_indices": [0] + }"#, + )); + assert!(err.is_err(), "a zero-second step timeout must be rejected"); +} + +#[test] +fn diamond_shares_no_duplicate_ancestors() { + // scratch -> a -> {b, c}; both b and c build_in a exactly once. + let g = lower(&chain( + r#"{ + "steps": [ + {"cmd": null, "parent_idx": null}, + {"cmd": "a", "parent_idx": 0, "label": "a"}, + {"cmd": "b", "parent_idx": 1, "label": "b"}, + {"cmd": "c", "parent_idx": 1, "label": "c"} + ], + "leaf_indices": [2, 3] + }"#, + )) + .unwrap(); + assert_eq!(g.node_count(), 3); + assert_eq!( + edges_by_key(&g), + BTreeSet::from([ + ("a".into(), "b".into(), "builds_in"), + ("a".into(), "c".into(), "builds_in"), + ]) + ); + // a appears once in the topo order despite being reached via two leaves. + assert_eq!(nodes_by_key(&g)["a"].image.as_deref(), Some("ubuntu:24.04")); +} From 28627259893e8905049138eede4b27fffa9ea343 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:37:44 -0700 Subject: [PATCH 3/9] feat: add Python step chain serializer --- .../harmont-py/harmont/_serialize.py | 92 +++++++ .../harmont-py/tests/test_serialize.py | 253 ++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 crates/hm-dsl-engine/harmont-py/harmont/_serialize.py create mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_serialize.py diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py b/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py new file mode 100644 index 00000000..ec187672 --- /dev/null +++ b/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py @@ -0,0 +1,92 @@ +"""Serialize a pipeline's Step chain to the raw step-chain wire format. + +Replaces the Python lowering pass: instead of lowering Step chains all the +way to the petgraph-serde IR, this emits the flat ``RawStepChain`` format — +a list of steps that reference their parents by index — which Rust +(`crates/hm-dsl-engine/src/step_chain.rs`) deserializes and lowers. + +Every reachable node is serialized, including scratch/fork passthroughs and +`wait` barriers; Rust drops the passthroughs and translates barriers into +`depends_on` edges. Env layering, key resolution, and the default-image +stamp also live on the Rust side, so this module emits only the raw +per-step fields. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ._duration import parse_duration +from ._pipeline import _cache_to_dict, _topo_collect + +if TYPE_CHECKING: + from ._step import Step + + +def serialize_step_chain( + leaves: list[Step] | tuple[Step, ...], + *, + env: dict[str, str] | None = None, + timeout: str | int | None = None, +) -> dict[str, Any]: + """Serialize the Step chains behind ``leaves`` into ``RawStepChain`` shape. + + Walks back from each leaf via ``Step.parent``, assigns every unique step + (by ``id``) a dense index in parent-before-child order, and emits each + step with its ``parent_idx`` pointing at the parent's index. Steps are + keyed by identity, so structurally-equal forks keep distinct indices. + + Args: + leaves: Terminal step(s) of each branch. Must be non-empty. + env: Pipeline-level environment variables. Layered under per-step + env by the Rust lowering pass. + timeout: Whole-build wall-clock budget (``"30m"``, an int number of + seconds, or a ``timedelta``). + + Returns: + A JSON-shaped dict with ``steps``, ``leaf_indices``, and optional + ``pipeline_env`` / ``pipeline_timeout_seconds``. + """ + ordered = _topo_collect(list(leaves)) + idx_by_id: dict[int, int] = {id(s): i for i, s in enumerate(ordered)} + + steps = [_serialize_step(s, idx_by_id) for s in ordered] + leaf_indices = [idx_by_id[id(leaf)] for leaf in leaves] + + out: dict[str, Any] = {"steps": steps, "leaf_indices": leaf_indices} + if env is not None: + out["pipeline_env"] = dict(env) + if timeout is not None: + out["pipeline_timeout_seconds"] = parse_duration(timeout) + return out + + +def _serialize_step(step: Step, idx_by_id: dict[int, int]) -> dict[str, Any]: + """Serialize a single step. ``cmd`` and ``parent_idx`` are always present + (Rust requires them); every other field is emitted only when set, and the + two bool flags only when true, matching Rust's serde defaults.""" + d: dict[str, Any] = { + "cmd": step.cmd, + "parent_idx": idx_by_id[id(step.parent)] if step.parent is not None else None, + } + if step.is_wait: + d["is_wait"] = True + if step.continue_on_failure: + d["continue_on_failure"] = True + if step.label is not None: + d["label"] = step.label + if step.cache is not None: + d["cache"] = _cache_to_dict(step.cache) + if step.env is not None: + d["env"] = dict(step.env) + if step.timeout_seconds is not None: + d["timeout_seconds"] = step.timeout_seconds + if step.image is not None: + d["image"] = step.image + if step.runner is not None: + d["runner"] = step.runner + if step.runner_args is not None: + d["runner_args"] = step.runner_args + if step.key_override is not None: + d["key_override"] = step.key_override + return d diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_serialize.py b/crates/hm-dsl-engine/harmont-py/tests/test_serialize.py new file mode 100644 index 00000000..8464ac56 --- /dev/null +++ b/crates/hm-dsl-engine/harmont-py/tests/test_serialize.py @@ -0,0 +1,253 @@ +"""Step-chain serializer: Step chains -> RawStepChain wire dict.""" + +from __future__ import annotations + +from datetime import timedelta + +import harmont as hm +from harmont._serialize import serialize_step_chain +from harmont._step import scratch, wait +from harmont.cache import ( + CacheCompose, + CacheForever, + CacheNone, + CacheOnChange, + CacheTTL, +) + + +def _by_cmd(out, cmd): + """Return the single serialized step whose cmd matches.""" + matches = [s for s in out["steps"] if s["cmd"] == cmd] + assert len(matches) == 1, f"expected exactly one step with cmd={cmd!r}" + return matches[0] + + +def test_linear_chain_parent_indices(): + a = scratch().sh("a") + b = a.sh("b") + out = serialize_step_chain([b]) + + # scratch root + a + b + assert len(out["steps"]) == 3 + assert out["leaf_indices"] == [out["steps"].index(_by_cmd(out, "b"))] + + root = _by_cmd(out, None) + step_a = _by_cmd(out, "a") + step_b = _by_cmd(out, "b") + root_idx = out["steps"].index(root) + a_idx = out["steps"].index(step_a) + + assert root["parent_idx"] is None + assert step_a["parent_idx"] == root_idx + assert step_b["parent_idx"] == a_idx + + +def test_parent_before_child_ordering(): + b = scratch().sh("a").sh("b") + out = serialize_step_chain([b]) + # Each step's parent_idx must reference an earlier entry. + for i, s in enumerate(out["steps"]): + if s["parent_idx"] is not None: + assert s["parent_idx"] < i + + +def test_fork_shares_ancestor_dedup(): + base = scratch().sh("install") + left = base.fork().sh("test-left") + right = base.fork().sh("test-right") + out = serialize_step_chain([left, right]) + + # 'install' and its scratch root appear exactly once each. + assert len([s for s in out["steps"] if s["cmd"] == "install"]) == 1 + assert len(out["leaf_indices"]) == 2 + + install_idx = out["steps"].index(_by_cmd(out, "install")) + # Both fork nodes descend (transitively) from the shared install step. + for cmd in ("test-left", "test-right"): + leaf = _by_cmd(out, cmd) + # leaf.parent is the fork passthrough; walk one hop up. + fork_idx = leaf["parent_idx"] + assert out["steps"][fork_idx]["parent_idx"] == install_idx + + +def test_distinct_objects_keep_distinct_indices(): + # Structurally identical but distinct leaves must not be deduped. + a = scratch().sh("same") + b = scratch().sh("same") + out = serialize_step_chain([a, b]) + assert len([s for s in out["steps"] if s["cmd"] == "same"]) == 2 + assert len(out["leaf_indices"]) == 2 + assert len(set(out["leaf_indices"])) == 2 + + +def test_wait_barrier_serialized(): + build = hm.sh("make build") + barrier = wait() + deploy = hm.sh("make deploy") + out = serialize_step_chain([build, barrier, deploy]) + + wait_steps = [s for s in out["steps"] if s.get("is_wait")] + assert len(wait_steps) == 1 + w = wait_steps[0] + assert w["is_wait"] is True + assert w["cmd"] is None + # continue_on_failure defaults false -> omitted. + assert "continue_on_failure" not in w + + +def test_wait_continue_on_failure(): + out = serialize_step_chain([wait(continue_on_failure=True)]) + w = out["steps"][out["leaf_indices"][0]] + assert w["is_wait"] is True + assert w["continue_on_failure"] is True + + +def test_cache_none(): + s = scratch().sh("x", cache=CacheNone()) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["cache"] == {"policy": "none"} + + +def test_cache_forever_with_env_keys(): + s = scratch().sh("x", cache=CacheForever(env_keys=("A", "B"))) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["cache"] == { + "policy": "forever", + "env_keys": ["A", "B"], + } + + +def test_cache_ttl(): + s = scratch().sh("x", cache=CacheTTL(duration=timedelta(hours=2), env_keys=("K",))) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["cache"] == { + "policy": "ttl", + "duration_seconds": 7200, + "env_keys": ["K"], + } + + +def test_cache_on_change(): + s = scratch().sh("x", cache=CacheOnChange(paths=("Cargo.toml", "Cargo.lock"))) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["cache"] == { + "policy": "on_change", + "paths": ["Cargo.toml", "Cargo.lock"], + } + + +def test_cache_compose(): + s = scratch().sh( + "x", + cache=CacheCompose( + policies=( + CacheTTL(duration=timedelta(days=1)), + CacheOnChange(paths=("api/cabal.project",)), + ) + ), + ) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["cache"] == { + "policy": "compose", + "sub_policies": [ + {"policy": "ttl", "duration_seconds": 86400, "env_keys": []}, + {"policy": "on_change", "paths": ["api/cabal.project"]}, + ], + } + + +def test_no_cache_field_when_unset(): + out = serialize_step_chain([scratch().sh("x")]) + assert "cache" not in _by_cmd(out, "x") + + +def test_pipeline_env_propagated(): + out = serialize_step_chain([hm.sh("x")], env={"FOO": "bar"}) + assert out["pipeline_env"] == {"FOO": "bar"} + + +def test_pipeline_env_omitted_when_none(): + out = serialize_step_chain([hm.sh("x")]) + assert "pipeline_env" not in out + + +def test_pipeline_timeout_string(): + out = serialize_step_chain([hm.sh("x")], timeout="30m") + assert out["pipeline_timeout_seconds"] == 1800 + + +def test_pipeline_timeout_int(): + out = serialize_step_chain([hm.sh("x")], timeout=45) + assert out["pipeline_timeout_seconds"] == 45 + + +def test_pipeline_timeout_omitted_when_none(): + out = serialize_step_chain([hm.sh("x")]) + assert "pipeline_timeout_seconds" not in out + + +def test_step_env_propagated(): + out = serialize_step_chain([hm.sh("x", env={"A": "1"})]) + assert _by_cmd(out, "x")["env"] == {"A": "1"} + + +def test_step_timeout_propagated(): + s = hm.timeout("10s", hm.sh("x")) + out = serialize_step_chain([s]) + assert _by_cmd(out, "x")["timeout_seconds"] == 10 + + +def test_key_override_propagated(): + out = serialize_step_chain([hm.sh("x", key="my-key")]) + assert _by_cmd(out, "x")["key_override"] == "my-key" + + +def test_label_propagated(): + out = serialize_step_chain([hm.sh("x", label="My Step")]) + assert _by_cmd(out, "x")["label"] == "My Step" + + +def test_image_runner_runner_args_propagated(): + out = serialize_step_chain( + [ + scratch().sh( + "x", + image="alpine:3.20", + runner="fly", + runner_args={"size": "large", "n": 2}, + ) + ] + ) + step = _by_cmd(out, "x") + assert step["image"] == "alpine:3.20" + assert step["runner"] == "fly" + assert step["runner_args"] == {"size": "large", "n": 2} + + +def test_optional_fields_omitted_by_default(): + out = serialize_step_chain([hm.sh("x")]) + step = _by_cmd(out, "x") + for field in ( + "label", + "cache", + "env", + "timeout_seconds", + "runner", + "runner_args", + "key_override", + "is_wait", + "continue_on_failure", + ): + assert field not in step, f"{field} should be omitted when unset" + # cmd and parent_idx are always present. + assert "cmd" in step + assert "parent_idx" in step + + +def test_empty_leaves_still_serializes_empty(): + # Guard: serializer itself does not enforce non-empty (the pipeline + # factory does); an empty forest yields empty steps. + out = serialize_step_chain([]) + assert out["steps"] == [] + assert out["leaf_indices"] == [] From 0d72e59b7bc86b9eb4d1f372728bf100b22603a1 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:41:05 -0700 Subject: [PATCH 4/9] feat: switch envelope to emit step chains instead of lowered IR --- .../harmont-py/harmont/_envelope.py | 64 +++-------- .../harmont-py/tests/test_envelope.py | 103 ++++++------------ 2 files changed, 46 insertions(+), 121 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_envelope.py b/crates/hm-dsl-engine/harmont-py/harmont/_envelope.py index 936cb379..47601ba2 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_envelope.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_envelope.py @@ -3,76 +3,45 @@ See docs/superpowers/specs/2026-05-10-har-9-imperfect-dsl-design.md § "The envelope" for the wire format. -Each registered pipeline carries its resolved v0 IR as a nested -``definition`` object. Consumers (api, cli) read that directly — no -intermediate Scheme stage exists since HAR-16. +Each registered pipeline carries its raw step chain as a nested +``step_chain`` object. Rust (`crates/hm-dsl-engine/src/step_chain.rs`) +lowers that into the v0 IR — env layering, key resolution, and the +default-image stamp all live on the Rust side. """ from __future__ import annotations import json -import os -import time -from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any -from ._pipeline import pipeline as _assemble from ._registry import REGISTRATIONS, PipelineRegistration +from ._serialize import serialize_step_chain from ._target import clear_target_memo from ._unwrap import as_leaves -from .keygen import resolve_pipeline_keys -if TYPE_CHECKING: - from collections.abc import Mapping - -def _render_one( - reg: PipelineRegistration, - *, - pipeline_org: str, - now: int, - base_path: Path, - env: Mapping[str, str], -) -> dict[str, Any]: +def _render_one(reg: PipelineRegistration) -> dict[str, Any]: raw = reg.fn() try: leaves = as_leaves(raw) except TypeError as e: msg = f"pipeline {reg.slug!r}: invalid return value\n → {e}" raise TypeError(msg) from e - ir = _assemble(leaves, env=reg.env, timeout=reg.timeout) - resolve_pipeline_keys( - ir.get("graph", {}), - pipeline_org=pipeline_org, - pipeline_slug=reg.slug, - now=now, - base_path=base_path, - env=env, - ) + step_chain = serialize_step_chain(leaves, env=reg.env, timeout=reg.timeout) return { "slug": reg.slug, "name": reg.name, "allow_manual": reg.allow_manual, "triggers": [t.to_dict() for t in reg.triggers], - "definition": ir, + "step_chain": step_chain, } -def dump_registry_json( - *, - pipeline_org: str | None = None, - now: int | None = None, - base_path: Path | None = None, - env: Mapping[str, str] | None = None, -) -> str: +def dump_registry_json() -> str: """Emit the schema_version=1 envelope JSON. - Defaults mirror ``pipeline_to_json``: - ``pipeline_org`` <- ``env["HM_PIPELINE_ORG"]`` or ``"default"`` - ``now`` <- ``int(time.time())`` - ``base_path`` <- ``Path.cwd()`` (resolves ``on_change`` cache paths) - ``env`` <- ``os.environ`` - Per-pipeline slug is read from each registration. + Each pipeline's raw step chain is serialized; Rust performs the + lowering (env layering, cache-key resolution, default-image stamp). The target memoization cache is cleared at the start of each render so per-pipeline target invocations dedup within a single render but @@ -80,16 +49,9 @@ def dump_registry_json( so pipeline fixture-style params can resolve their dependencies. """ clear_target_memo() - env_map: Mapping[str, str] = env if env is not None else os.environ - org = pipeline_org if pipeline_org is not None else env_map.get("HM_PIPELINE_ORG", "default") - render_now = now if now is not None else int(time.time()) - bp = base_path if base_path is not None else Path.cwd() return json.dumps( { "schema_version": "1", - "pipelines": [ - _render_one(reg, pipeline_org=org, now=render_now, base_path=bp, env=env_map) - for reg in REGISTRATIONS - ], + "pipelines": [_render_one(reg) for reg in REGISTRATIONS], } ) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py b/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py index 48fe783b..408a2be0 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py @@ -21,33 +21,17 @@ def _reset_registry(): clear_target_names() -def _graph_nodes(definition): - return definition["graph"]["nodes"] +def _steps(step_chain): + return step_chain["steps"] -def _graph_edges(definition): - return definition["graph"]["edges"] +def _cmds(step_chain): + return [s.get("cmd") for s in step_chain["steps"]] -def _step_cmds(definition): - return [n["step"].get("cmd") for n in _graph_nodes(definition)] - - -def _builds_in_children(definition, parent_key): - """Return nodes whose builds_in parent is parent_key.""" - nodes = _graph_nodes(definition) - parent_idx = None - for i, n in enumerate(nodes): - if n["step"]["key"] == parent_key: - parent_idx = i - break - if parent_idx is None: - return [] - children = [] - for src, dst, kind in _graph_edges(definition): - if kind == "builds_in" and src == parent_idx: - children.append(nodes[dst]) - return children +def _children_of(step_chain, parent_idx): + """Return steps whose parent_idx is parent_idx.""" + return [s for s in step_chain["steps"] if s.get("parent_idx") == parent_idx] def test_empty_registry_emits_empty_pipelines_list(): @@ -68,12 +52,14 @@ def ci() -> hm.Step: assert p["name"] == "ci" assert p["allow_manual"] is True assert p["triggers"] == [] - definition = p["definition"] - assert definition["version"] == "0" - nodes = _graph_nodes(definition) - assert len(nodes) == 1 - assert nodes[0]["step"]["cmd"] == "echo hi" - assert nodes[0]["step"]["label"] == "hi" + step_chain = p["step_chain"] + steps = _steps(step_chain) + # The scratch root rides along as a passthrough (cmd=None); Rust drops it. + cmd_steps = [s for s in steps if s.get("cmd") == "echo hi"] + assert len(cmd_steps) == 1 + assert cmd_steps[0]["label"] == "hi" + (leaf,) = step_chain["leaf_indices"] + assert steps[leaf]["cmd"] == "echo hi" def test_pipeline_with_triggers(): @@ -107,39 +93,20 @@ def ci() -> hm.Pipeline: out = json.loads(hm.dump_registry_json()) p = out["pipelines"][0] - cmds = sorted(n["step"]["cmd"] for n in _graph_nodes(p["definition"])) + cmds = sorted(c for c in _cmds(p["step_chain"]) if c in ("a", "b")) assert cmds == ["a", "b"] + assert len(p["step_chain"]["leaf_indices"]) == 2 -def test_pipeline_forwards_env_to_assemble(): +def test_pipeline_forwards_env_to_step_chain(): @hm.pipeline("ci", env={"CI": "true"}) def ci() -> hm.Step: return hm.scratch().sh("echo") out = json.loads(hm.dump_registry_json()) - definition = out["pipelines"][0]["definition"] - # Pipeline-level env is merged into node env dicts. - for node in _graph_nodes(definition): - assert node["env"].get("CI") == "true" - - -def test_envelope_resolves_cache_keys(tmp_path): - @hm.pipeline("ci") - def ci() -> hm.Step: - return hm.scratch().sh("echo", label="run", cache=hm.forever()) - - out = json.loads( - hm.dump_registry_json( - pipeline_org="acme", - now=1700000000, - base_path=tmp_path, - env={}, - ) - ) - step = _graph_nodes(out["pipelines"][0]["definition"])[0]["step"] - assert step["cache"]["policy"] == "forever" - assert "key" in step["cache"] - assert len(step["cache"]["key"]) == 64 + step_chain = out["pipelines"][0]["step_chain"] + # Pipeline-level env rides on the step chain; Rust layers it per node. + assert step_chain["pipeline_env"] == {"CI": "true"} def test_envelope_auto_unwraps_go_toolchain(): @@ -150,12 +117,11 @@ def ci(): return hm.go(path="api").build() out = json.loads(hm.dump_registry_json()) - nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"].get("cmd") for n in nodes] + cmds = _cmds(out["pipelines"][0]["step_chain"]) assert any("go build" in (c or "") for c in cmds) -def test_envelope_composes_targets_with_dedup(tmp_path, monkeypatch): +def test_envelope_composes_targets_with_dedup(): """Two pipelines depending on the same target share the target step.""" from harmont._target import clear_target_cache @@ -173,13 +139,12 @@ def ci() -> tuple[hm.Step, ...]: ) out = json.loads(hm.dump_registry_json()) - definition = out["pipelines"][0]["definition"] - nodes = _graph_nodes(definition) - apt_nodes = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] - assert len(apt_nodes) == 1 # deduplicated via target memoization - children = _builds_in_children(definition, apt_nodes[0]["step"]["key"]) - assert len(children) == 2 - child_cmds = sorted(n["step"]["cmd"] for n in children) + step_chain = out["pipelines"][0]["step_chain"] + steps = _steps(step_chain) + apt_indices = [i for i, s in enumerate(steps) if s.get("cmd") == "apt-get update"] + assert len(apt_indices) == 1 # deduplicated via target memoization + children = _children_of(step_chain, apt_indices[0]) + child_cmds = sorted(c["cmd"] for c in children) assert child_cmds == ["cabal build", "pytest"] @@ -197,9 +162,7 @@ def ci() -> hm.Step: hm.dump_registry_json() # After render, cache has one entry from the in-flight render. Trigger # a second render and verify the cache is cleared at render start - # by re-running and confirming success (would TypeError otherwise if - # the first render's cached Step somehow propagated through dataclass - # frozen-equality into the second render's IR). + # by re-running and confirming success. hm.dump_registry_json() @@ -221,7 +184,7 @@ def test_decorator_pipeline_timeout_in_envelope(): def _timed() -> hm.Step: return hm.sh("make test") - env = json.loads(hm.dump_registry_json(now=0)) - defn = env["pipelines"][0]["definition"] - assert defn["timeout_seconds"] == 1200 + env = json.loads(hm.dump_registry_json()) + step_chain = env["pipelines"][0]["step_chain"] + assert step_chain["pipeline_timeout_seconds"] == 1200 REGISTRATIONS.clear() From 48f928ca44ffc7dfff179ad65a15e9c5a8c717b6 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:41:26 -0700 Subject: [PATCH 5/9] feat: add raw envelope deserialization and processing --- crates/hm-dsl-engine/src/lib.rs | 1 + crates/hm-dsl-engine/src/raw_envelope.rs | 82 +++++++++++ .../hm-dsl-engine/tests/raw_envelope_test.rs | 136 ++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 crates/hm-dsl-engine/src/raw_envelope.rs create mode 100644 crates/hm-dsl-engine/tests/raw_envelope_test.rs diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index 7152ea78..db3d6bea 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -5,6 +5,7 @@ use serde::Deserialize; pub mod detect; pub mod lower; +pub mod raw_envelope; pub mod step_chain; mod python_engine; diff --git a/crates/hm-dsl-engine/src/raw_envelope.rs b/crates/hm-dsl-engine/src/raw_envelope.rs new file mode 100644 index 00000000..9111e169 --- /dev/null +++ b/crates/hm-dsl-engine/src/raw_envelope.rs @@ -0,0 +1,82 @@ +//! Raw discovery envelope emitted by the Python DSL, and its lowering. +//! +//! The Python runtime emits a [`RawEnvelope`]: per-pipeline metadata plus each +//! pipeline's [`RawStepChain`] (not yet lowered IR). Rust lowers every chain via +//! [`lower::lower`] and produces a [`FinalEnvelope`] whose `definition` field +//! carries the canonical v0 [`hm_pipeline_ir::PipelineGraph`], ready for the +//! backend's pipeline discovery to consume. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::lower; +use crate::step_chain::RawStepChain; + +/// The discovery envelope as emitted by the Python DSL, before lowering. +#[derive(Debug, Clone, Deserialize)] +pub struct RawEnvelope { + pub schema_version: String, + pub pipelines: Vec, +} + +/// One pipeline in a [`RawEnvelope`]: metadata plus its raw step chain. +#[derive(Debug, Clone, Deserialize)] +pub struct RawPipelineEntry { + pub slug: String, + pub name: String, + #[serde(default)] + pub allow_manual: bool, + #[serde(default)] + pub triggers: Vec, + pub step_chain: RawStepChain, +} + +/// The lowered discovery envelope handed to consumers. +#[derive(Debug, Clone, Serialize)] +pub struct FinalEnvelope { + pub schema_version: String, + pub pipelines: Vec, +} + +/// One pipeline in a [`FinalEnvelope`]: metadata plus its lowered definition. +#[derive(Debug, Clone, Serialize)] +pub struct FinalPipelineEntry { + pub slug: String, + pub name: String, + pub allow_manual: bool, + pub triggers: Vec, + /// The serialized [`hm_pipeline_ir::PipelineGraph`] (v0 IR). + pub definition: serde_json::Value, +} + +/// Lower every pipeline's step chain and produce the final envelope. +/// +/// # Errors +/// +/// Returns an error if any pipeline's step chain fails to lower, or if the +/// resulting graph cannot be serialized. +pub fn process_raw_envelope(raw: RawEnvelope) -> Result { + let pipelines = raw + .pipelines + .into_iter() + .map(process_entry) + .collect::>>()?; + Ok(FinalEnvelope { + schema_version: raw.schema_version, + pipelines, + }) +} + +fn process_entry(entry: RawPipelineEntry) -> Result { + let graph = lower::lower(&entry.step_chain) + .with_context(|| format!("failed to lower pipeline '{}'", entry.slug))?; + let definition = serde_json::to_value(&graph) + .with_context(|| format!("failed to serialize definition for pipeline '{}'", entry.slug))?; + Ok(FinalPipelineEntry { + slug: entry.slug, + name: entry.name, + allow_manual: entry.allow_manual, + triggers: entry.triggers, + definition, + }) +} diff --git a/crates/hm-dsl-engine/tests/raw_envelope_test.rs b/crates/hm-dsl-engine/tests/raw_envelope_test.rs new file mode 100644 index 00000000..a62a86cb --- /dev/null +++ b/crates/hm-dsl-engine/tests/raw_envelope_test.rs @@ -0,0 +1,136 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use hm_dsl_engine::raw_envelope::{RawEnvelope, process_raw_envelope}; +use serde_json::json; + +/// A raw envelope with a single one-step pipeline. +fn single_pipeline_envelope() -> serde_json::Value { + json!({ + "schema_version": "1", + "pipelines": [{ + "slug": "ci", + "name": "CI", + "allow_manual": true, + "triggers": [{"kind": "push", "branch": "main"}], + "step_chain": { + "steps": [ + {"cmd": "make build", "parent_idx": null, "label": "build"} + ], + "leaf_indices": [0] + } + }] + }) +} + +#[test] +fn deserializes_and_processes_single_pipeline() { + let raw: RawEnvelope = serde_json::from_value(single_pipeline_envelope()).unwrap(); + assert_eq!(raw.schema_version, "1"); + assert_eq!(raw.pipelines.len(), 1); + + let final_env = process_raw_envelope(raw).unwrap(); + assert_eq!(final_env.schema_version, "1"); + assert_eq!(final_env.pipelines.len(), 1); + + let entry = &final_env.pipelines[0]; + // The final entry carries a lowered `definition`, never a `step_chain`. + let serialized = serde_json::to_value(entry).unwrap(); + assert!(serialized.get("definition").is_some()); + assert!(serialized.get("step_chain").is_none()); +} + +#[test] +fn definition_is_valid_v0_ir() { + let raw: RawEnvelope = serde_json::from_value(single_pipeline_envelope()).unwrap(); + let final_env = process_raw_envelope(raw).unwrap(); + let def = &final_env.pipelines[0].definition; + + assert_eq!(def.get("version").and_then(|v| v.as_str()), Some("0")); + + let graph = def.get("graph").expect("definition has a graph"); + let nodes = graph.get("nodes").and_then(|n| n.as_array()).unwrap(); + assert_eq!(nodes.len(), 1); + assert!(graph.get("edges").and_then(|e| e.as_array()).unwrap().is_empty()); + + let step = &nodes[0]["step"]; + assert_eq!(step["cmd"].as_str(), Some("make build")); + assert_eq!(step["label"].as_str(), Some("build")); + // Imageless root steps get the pipeline-wide default image. + assert_eq!(step["image"].as_str(), Some("ubuntu:24.04")); + + // The `definition` round-trips as a real PipelineGraph. + let parsed: hm_pipeline_ir::PipelineGraph = serde_json::from_value(def.clone()).unwrap(); + assert_eq!(parsed.node_count(), 1); +} + +#[test] +fn metadata_passes_through() { + let raw: RawEnvelope = serde_json::from_value(single_pipeline_envelope()).unwrap(); + let final_env = process_raw_envelope(raw).unwrap(); + let entry = &final_env.pipelines[0]; + + assert_eq!(entry.slug, "ci"); + assert_eq!(entry.name, "CI"); + assert!(entry.allow_manual); + assert_eq!(entry.triggers.len(), 1); + assert_eq!(entry.triggers[0]["kind"].as_str(), Some("push")); +} + +#[test] +fn processes_multiple_pipelines() { + let raw: RawEnvelope = serde_json::from_value(json!({ + "schema_version": "1", + "pipelines": [ + { + "slug": "one", + "name": "One", + "step_chain": { + "steps": [{"cmd": "echo a", "parent_idx": null}], + "leaf_indices": [0] + } + }, + { + "slug": "two", + "name": "Two", + "step_chain": { + "steps": [ + {"cmd": "echo a", "parent_idx": null}, + {"cmd": "echo b", "parent_idx": 0} + ], + "leaf_indices": [1] + } + } + ] + })) + .unwrap(); + + let final_env = process_raw_envelope(raw).unwrap(); + assert_eq!(final_env.pipelines.len(), 2); + assert_eq!(final_env.pipelines[0].slug, "one"); + assert_eq!(final_env.pipelines[1].slug, "two"); + + let two: hm_pipeline_ir::PipelineGraph = + serde_json::from_value(final_env.pipelines[1].definition.clone()).unwrap(); + assert_eq!(two.node_count(), 2); +} + +#[test] +fn metadata_defaults_when_omitted() { + let raw: RawEnvelope = serde_json::from_value(json!({ + "schema_version": "1", + "pipelines": [{ + "slug": "min", + "name": "Minimal", + "step_chain": { + "steps": [{"cmd": "true", "parent_idx": null}], + "leaf_indices": [0] + } + }] + })) + .unwrap(); + + let final_env = process_raw_envelope(raw).unwrap(); + let entry = &final_env.pipelines[0]; + assert!(!entry.allow_manual); + assert!(entry.triggers.is_empty()); +} From a0ee09a3f1b2fa0036d2fa0f82d5453d3bd8c273 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:43:07 -0700 Subject: [PATCH 6/9] feat: implement cache key resolution in Rust (byte-exact parity with Python) --- Cargo.lock | 7 + crates/hm-dsl-engine/Cargo.toml | 1 + crates/hm-dsl-engine/src/keygen.rs | 233 +++++++++++++++++++ crates/hm-dsl-engine/src/lib.rs | 1 + crates/hm-dsl-engine/src/lower.rs | 86 ++++++- crates/hm-dsl-engine/tests/keygen_test.rs | 264 ++++++++++++++++++++++ 6 files changed, 585 insertions(+), 7 deletions(-) create mode 100644 crates/hm-dsl-engine/src/keygen.rs create mode 100644 crates/hm-dsl-engine/tests/keygen_test.rs diff --git a/Cargo.lock b/Cargo.lock index 8260ce3c..97916212 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1019,6 +1019,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "globset" version = "0.4.18" @@ -1243,6 +1249,7 @@ dependencies = [ "async-trait", "daggy", "derive_more", + "glob", "hm-pipeline-ir", "include_dir", "serde", diff --git a/crates/hm-dsl-engine/Cargo.toml b/crates/hm-dsl-engine/Cargo.toml index 395c42e3..48b7bcf0 100644 --- a/crates/hm-dsl-engine/Cargo.toml +++ b/crates/hm-dsl-engine/Cargo.toml @@ -16,6 +16,7 @@ hm-pipeline-ir = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } derive_more = { workspace = true } +glob = "0.3" serde = { workspace = true } serde_json = { workspace = true } sha2 = "0.10" diff --git a/crates/hm-dsl-engine/src/keygen.rs b/crates/hm-dsl-engine/src/keygen.rs new file mode 100644 index 00000000..71e4bc39 --- /dev/null +++ b/crates/hm-dsl-engine/src/keygen.rs @@ -0,0 +1,233 @@ +//! Cache-key resolver — byte-exact port of `harmont-py/harmont/keygen.py`. +//! +//! The output bytes MUST match the Python (and, before it, Scheme) resolver so +//! cache snapshots persisted by earlier versions stay reachable. The outer key +//! is the sha256 of the preimage +//! +//! ```text +//! pipeline_org NUL pipeline_slug NUL step_key NUL parent_resolved NUL policy_resolution +//! ``` +//! +//! where `NUL` is a single `\x00` byte and `policy_resolution` branches on the +//! step's cache policy (see [`resolve_policy`]). + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use sha2::{Digest, Sha256}; + +use crate::step_chain::RawCachePolicy; + +/// The `\x00` separator woven through every preimage. +const NUL: &str = "\x00"; + +/// Parameters cache-key resolution needs beyond the raw step chain. +/// +/// Carried through the lowering pass so keys can be resolved while the rich +/// [`RawCachePolicy`] data (env keys, durations, paths, sub-policies) is still +/// in scope, before it is flattened into the IR `Cache` shape. +#[derive(Debug, Clone)] +pub struct LowerOptions { + /// Owning organization slug — first field of the outer preimage. + pub pipeline_org: String, + /// Pipeline slug — second field of the outer preimage. + pub pipeline_slug: String, + /// Wall-clock unix timestamp used to bucket `ttl` policies. + pub now: u64, + /// Project directory that `on_change` paths and globs resolve against. + pub base_path: PathBuf, + /// Process environment sampled for `env_subset` of `forever`/`ttl` policies. + pub env: BTreeMap, +} + +/// Hex-encode `sha256(s)` as lowercase, matching `hashlib.sha256(...).hexdigest()`. +#[must_use] +pub fn sha256_hex(s: &str) -> String { + to_hex(&Sha256::digest(s.as_bytes())) +} + +/// Resolve a step's full outer cache key. +/// +/// `parent_resolved` is the already-resolved key of the `builds_in` parent, or +/// `"scratch"` when the step has no cached parent. +/// +/// # Errors +/// +/// Propagates errors from [`resolve_policy`] — chiefly missing `on_change` +/// paths or filesystem read failures. +pub fn compute_cache_key( + step_key: &str, + cmd: &str, + policy: &RawCachePolicy, + parent_resolved: &str, + opts: &LowerOptions, +) -> Result { + let policy_res = resolve_policy(policy, cmd, opts)?; + let preimage = [ + opts.pipeline_org.as_str(), + opts.pipeline_slug.as_str(), + step_key, + parent_resolved, + policy_res.as_str(), + ] + .join(NUL); + Ok(sha256_hex(&preimage)) +} + +/// Resolve the `policy_resolution` fragment for a single policy. +/// +/// # Errors +/// +/// Returns an error if an `on_change` path is missing (and not silently +/// skippable) or a file/directory cannot be read. +pub fn resolve_policy(policy: &RawCachePolicy, cmd: &str, opts: &LowerOptions) -> Result { + match policy { + RawCachePolicy::None => Ok("none".to_owned()), + RawCachePolicy::Forever { env_keys } => { + let inner = [cmd, &env_subset(env_keys, &opts.env)].join(NUL); + Ok(format!("forever-{}", sha256_hex(&inner))) + } + RawCachePolicy::Ttl { + duration_seconds, + env_keys, + } => { + let bucket = opts.now / duration_seconds; + let inner = [cmd, &env_subset(env_keys, &opts.env)].join(NUL); + Ok(format!("ttl-{bucket}-{}", sha256_hex(&inner))) + } + RawCachePolicy::OnChange { paths } => { + Ok(format!("sha-{}", sha256_hex(&on_change_preimage(paths, &opts.base_path)?))) + } + RawCachePolicy::Compose { sub_policies } => { + let mut parts = String::new(); + for sub in sub_policies { + if matches!(sub, RawCachePolicy::None) { + parts.push_str("none"); + } else { + parts.push_str(&resolve_policy(sub, cmd, opts)?); + } + } + Ok(format!("compose-{}", sha256_hex(&parts))) + } + } +} + +/// Concatenate `key=value\x00` for each sorted env key, reading `value` from +/// `env` (empty string when absent). +#[must_use] +pub fn env_subset(env_keys: &[String], env: &BTreeMap) -> String { + let mut sorted: Vec<&String> = env_keys.iter().collect(); + sorted.sort(); + let mut out = String::new(); + for k in sorted { + out.push_str(k); + out.push('='); + out.push_str(env.get(k).map_or("", String::as_str)); + out.push_str(NUL); + } + out +} + +/// Build the `on_change` preimage: `file_hash(p) NUL` for each resolved path. +/// +/// Path strings are sorted first; glob patterns (`*`, `?`, `[`) expand against +/// `base` and their matches are sorted; plain paths that don't exist are +/// skipped (mirroring `keygen.py`). +fn on_change_preimage(paths: &[String], base: &Path) -> Result { + let mut sorted: Vec<&String> = paths.iter().collect(); + sorted.sort(); + + let mut resolved: Vec = Vec::new(); + for p in sorted { + if p.contains('*') || p.contains('?') || p.contains('[') { + let pattern = base.join(p); + let pattern = pattern.to_str().with_context(|| { + format!("on_change glob pattern is not valid UTF-8: {}", pattern.display()) + })?; + let mut matches: Vec = glob::glob(pattern) + .with_context(|| format!("invalid on_change glob pattern: {pattern}"))? + .collect::, _>>() + .with_context(|| format!("failed to read on_change glob matches for {pattern}"))?; + matches.sort(); + resolved.extend(matches); + } else { + let full = base.join(p); + if full.exists() { + resolved.push(full); + } + } + } + + let mut pre = String::new(); + for r in &resolved { + pre.push_str(&path_hash(r)?); + pre.push_str(NUL); + } + Ok(pre) +} + +/// Hash a path for an `on_change` key. +/// +/// Files hash their bytes. Directories fold each descendant file's POSIX +/// relative path + bytes into one stream in sorted order. Missing paths fail +/// loudly. +fn path_hash(path: &Path) -> Result { + if path.is_file() { + let bytes = + std::fs::read(path).with_context(|| format!("reading on_change file {}", path.display()))?; + return Ok(to_hex(&Sha256::digest(&bytes))); + } + if path.is_dir() { + let mut files: Vec = Vec::new(); + collect_files(path, &mut files)?; + files.sort(); + + let mut h = Sha256::new(); + for child in &files { + let rel = child + .strip_prefix(path) + .with_context(|| format!("relativizing {}", child.display()))?; + let rel_posix = rel + .components() + .map(|c| c.as_os_str().to_string_lossy()) + .collect::>() + .join("/"); + h.update(rel_posix.as_bytes()); + h.update([0u8]); + let bytes = std::fs::read(child) + .with_context(|| format!("reading on_change file {}", child.display()))?; + h.update(&bytes); + h.update([0u8]); + } + return Ok(to_hex(&h.finalize())); + } + bail!("on_change path does not exist: {}", path.display()); +} + +/// Collect every regular file under `dir`, recursing into subdirectories. +fn collect_files(dir: &Path, out: &mut Vec) -> Result<()> { + let entries = + std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?; + for entry in entries { + let path = entry + .with_context(|| format!("reading entry in {}", dir.display()))? + .path(); + if path.is_dir() { + collect_files(&path, out)?; + } else if path.is_file() { + out.push(path); + } + } + Ok(()) +} + +/// Lowercase hex encoding of a byte slice. +fn to_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for b in bytes { + let _ = write!(out, "{b:02x}"); + } + out +} diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index db3d6bea..cc9a1d90 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use serde::Deserialize; pub mod detect; +pub mod keygen; pub mod lower; pub mod raw_envelope; pub mod step_chain; diff --git a/crates/hm-dsl-engine/src/lower.rs b/crates/hm-dsl-engine/src/lower.rs index 990c7104..8ba1cef6 100644 --- a/crates/hm-dsl-engine/src/lower.rs +++ b/crates/hm-dsl-engine/src/lower.rs @@ -10,11 +10,12 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::Write as _; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, anyhow}; use hm_pipeline_ir::PipelineGraph; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; +use crate::keygen::{self, LowerOptions}; use crate::step_chain::{RawCachePolicy, RawStepChain}; /// Across-the-board default image for imageless root steps. The SDK's @@ -22,7 +23,11 @@ use crate::step_chain::{RawCachePolicy, RawStepChain}; /// default; child steps boot from their parent's snapshot and stay imageless. const DEFAULT_IMAGE: &str = "ubuntu:24.04"; -/// Lower a raw step chain into the canonical [`PipelineGraph`]. +/// Lower a raw step chain into the canonical [`PipelineGraph`] without +/// resolving cache keys. +/// +/// Equivalent to [`lower_with_options`] with `None` — cached steps emit a bare +/// `{"policy": ...}` with no `key`. Use [`lower_with_options`] to resolve keys. /// /// # Errors /// @@ -30,6 +35,23 @@ const DEFAULT_IMAGE: &str = "ubuntu:24.04"; /// [`PipelineGraph`] — e.g. a step declares a zero-second timeout, which the /// IR rejects at the wire boundary. pub fn lower(chain: &RawStepChain) -> Result { + lower_with_options(chain, None) +} + +/// Lower a raw step chain into the canonical [`PipelineGraph`], optionally +/// resolving cache keys. +/// +/// When `opts` is `Some`, every non-`none` cache policy gets a deterministic +/// `key` resolved byte-for-byte identically to the Python resolver (see +/// [`crate::keygen`]). When `None`, cache-key resolution is skipped. +/// +/// # Errors +/// +/// Returns an error if the emitted graph fails to deserialize into a +/// [`PipelineGraph`], if a cached step's `builds_in` parent is itself +/// uncached, or if cache-key resolution fails (e.g. a missing `on_change` +/// path). +pub fn lower_with_options(chain: &RawStepChain, opts: Option<&LowerOptions>) -> Result { let ordered = topo_collect(chain); let command_steps: Vec = ordered .iter() @@ -53,6 +75,10 @@ pub fn lower(chain: &RawStepChain) -> Result { let mut pre_wait_indices: Vec = Vec::new(); let mut pending_depends_on: Vec = Vec::new(); + // Resolved cache key per step key, populated as nodes are emitted so a + // child can read its `builds_in` parent's key. Only used when `opts` set. + let mut resolved_cache: HashMap = HashMap::new(); + for &raw in &ordered { let step = &chain.steps[raw]; if step.is_wait { @@ -74,7 +100,8 @@ pub fn lower(chain: &RawStepChain) -> Result { step_dict.insert("label".into(), Value::String(label.clone())); } if let Some(cache) = &step.cache { - step_dict.insert("cache".into(), cache_to_value(cache)); + let key = resolve_cache_key(chain, raw, cache, cmd, &keys, opts, &mut resolved_cache)?; + step_dict.insert("cache".into(), cache_to_value(cache, key)); } if let Some(timeout) = step.timeout_seconds { step_dict.insert("timeout_seconds".into(), Value::from(timeout)); @@ -186,9 +213,10 @@ fn resolved_parent_idx(chain: &RawStepChain, raw: usize) -> Option { None } -/// Render a raw cache policy to its IR `Cache` shape. Only the policy name -/// survives here; cache-key resolution is a separate pass. -fn cache_to_value(cache: &RawCachePolicy) -> Value { +/// Render a raw cache policy to its IR `Cache` shape. The rich policy data +/// (env keys, durations, paths, sub-policies) is dropped; only the policy name +/// and the pre-resolved `key` survive. +fn cache_to_value(cache: &RawCachePolicy, key: Option) -> Value { let policy = match cache { RawCachePolicy::None => "none", RawCachePolicy::Forever { .. } => "forever", @@ -196,7 +224,51 @@ fn cache_to_value(cache: &RawCachePolicy) -> Value { RawCachePolicy::OnChange { .. } => "on_change", RawCachePolicy::Compose { .. } => "compose", }; - json!({ "policy": policy }) + let mut map = Map::new(); + map.insert("policy".into(), Value::String(policy.into())); + if let Some(key) = key { + map.insert("key".into(), Value::String(key)); + } + Value::Object(map) +} + +/// Resolve a step's cache key during lowering, recording it for descendants. +/// +/// Returns `None` when key resolution is disabled (`opts` is `None`) or the +/// policy is `none`. The `builds_in` parent's resolved key is looked up from +/// `resolved` — a cached step whose parent is itself uncached is an error, +/// matching the Python resolver's `_lookup_parent`. +fn resolve_cache_key( + chain: &RawStepChain, + raw: usize, + cache: &RawCachePolicy, + cmd: &str, + keys: &HashMap, + opts: Option<&LowerOptions>, + resolved: &mut HashMap, +) -> Result> { + let Some(opts) = opts else { return Ok(None) }; + if matches!(cache, RawCachePolicy::None) { + return Ok(None); + } + + let step_key = &keys[&raw]; + let parent_resolved = match resolved_parent_idx(chain, raw) { + None => "scratch".to_owned(), + Some(parent_raw) => { + let parent_key = &keys[&parent_raw]; + resolved.get(parent_key).cloned().ok_or_else(|| { + anyhow!( + "step {step_key:?} references builds_in {parent_key:?} which has no \ + cached key (parent must be defined upstream and cached)" + ) + })? + } + }; + + let key = keygen::compute_cache_key(step_key, cmd, cache, &parent_resolved, opts)?; + resolved.insert(step_key.clone(), key.clone()); + Ok(Some(key)) } /// Resolve each command step's cross-reference key, keyed by raw index. diff --git a/crates/hm-dsl-engine/tests/keygen_test.rs b/crates/hm-dsl-engine/tests/keygen_test.rs new file mode 100644 index 00000000..02eeb5fe --- /dev/null +++ b/crates/hm-dsl-engine/tests/keygen_test.rs @@ -0,0 +1,264 @@ +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::too_many_lines +)] + +//! Byte-exact parity checks against the Python `harmont.keygen` resolver. +//! Reference hashes are produced by running the Python resolver directly. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use hm_dsl_engine::keygen::{ + LowerOptions, compute_cache_key, env_subset, resolve_policy, sha256_hex, +}; +use hm_dsl_engine::lower::lower_with_options; +use hm_dsl_engine::step_chain::{RawCachePolicy, RawStepChain}; +use hm_pipeline_ir::PipelineGraph; + +fn opts() -> LowerOptions { + LowerOptions { + pipeline_org: "myorg".into(), + pipeline_slug: "ci".into(), + now: 1_000_000, + base_path: PathBuf::from("."), + env: BTreeMap::from([ + ("RUST_VERSION".into(), "1.80".into()), + ("OTHER".into(), "x".into()), + ]), + } +} + +fn forever(env_keys: &[&str]) -> RawCachePolicy { + RawCachePolicy::Forever { + env_keys: env_keys.iter().map(ToString::to_string).collect(), + } +} + +fn cache_key_for(g: &PipelineGraph, step_key: &str) -> Option { + use daggy::petgraph::visit::IntoNodeReferences; + g.dag() + .graph() + .node_references() + .find(|(_, t)| t.step.key == step_key) + .and_then(|(_, t)| t.step.cache.as_ref()) + .and_then(|c| c.key.clone()) +} + +#[test] +fn sha256_hex_matches_python() { + assert_eq!( + sha256_hex(""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex("hello"), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); +} + +#[test] +fn env_subset_sorts_and_appends_nul() { + let env = BTreeMap::from([ + ("A".to_string(), "1".to_string()), + ("B".to_string(), "2".to_string()), + ("C".to_string(), "3".to_string()), + ]); + // Unsorted keys, missing "C" from the subset -> only A and B, sorted. + assert_eq!( + env_subset(&["B".to_string(), "A".to_string()], &env), + "A=1\u{0}B=2\u{0}" + ); +} + +#[test] +fn forever_policy_key_byte_exact() { + let key = compute_cache_key("test", "cargo test", &forever(&["RUST_VERSION"]), "scratch", &opts()) + .unwrap(); + assert_eq!( + key, + "02cdd7a86196443ba51cac9e7e3987bb127af122f1386cfc6a843fe0c85e2499" + ); +} + +#[test] +fn forever_policy_no_env_keys_byte_exact() { + let key = compute_cache_key("test", "cargo test", &forever(&[]), "scratch", &opts()).unwrap(); + assert_eq!( + key, + "61b02abc60e2e2472835d4dbbc8f84c85197cbdda3af24c7cacf462ccaa410f8" + ); +} + +#[test] +fn ttl_policy_key_byte_exact() { + let policy = RawCachePolicy::Ttl { + duration_seconds: 3600, + env_keys: vec!["RUST_VERSION".into()], + }; + // now = 1_000_000, bucket = 1_000_000 // 3600 = 277. + let res = resolve_policy(&policy, "cargo test", &opts()).unwrap(); + assert_eq!( + res, + "ttl-277-6d7d30b69066a5c5913d44213c25029bc892aa872d95dd97511de7e2345f4ee8" + ); + let key = compute_cache_key("test", "cargo test", &policy, "scratch", &opts()).unwrap(); + assert_eq!( + key, + "a937615f49e155edf8a059753c760bd285e0c088007f833f467864ada7cdbcab" + ); +} + +#[test] +fn compose_policy_key_byte_exact() { + let policy = RawCachePolicy::Compose { + sub_policies: vec![ + forever(&["RUST_VERSION"]), + RawCachePolicy::Ttl { + duration_seconds: 3600, + env_keys: vec!["RUST_VERSION".into()], + }, + ], + }; + let key = compute_cache_key("test", "cargo test", &policy, "scratch", &opts()).unwrap(); + assert_eq!( + key, + "d66b14459d672af0fe020875f8b411a0598afc23597a37e1dd0f47d5109bbda2" + ); +} + +#[test] +fn compose_with_none_sub_byte_exact() { + let policy = RawCachePolicy::Compose { + sub_policies: vec![forever(&["RUST_VERSION"]), RawCachePolicy::None], + }; + let key = compute_cache_key("test", "cargo test", &policy, "scratch", &opts()).unwrap(); + assert_eq!( + key, + "2b16eee0e344f933a360c7f8f1f88b0dd2692efd44690f949af8b27be4fe91a9" + ); +} + +#[test] +fn parent_resolved_key_threads_through() { + let parent = compute_cache_key("build", "cargo test", &forever(&["RUST_VERSION"]), "scratch", &opts()) + .unwrap(); + assert_eq!( + parent, + "0db96595eb9bcdadc0b34da262e816652127f6d19a7fe6f6b6b374c40629987a" + ); + let child = compute_cache_key("test", "cargo build", &forever(&[]), &parent, &opts()).unwrap(); + assert_eq!( + child, + "3c9085b03deb8ce7034eabd47a83675793b92f69918c605b5a1f1a31575766a2" + ); +} + +#[test] +fn on_change_file_dir_and_glob_byte_exact() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path(); + std::fs::write(base.join("a.txt"), b"hello").unwrap(); + std::fs::create_dir(base.join("sub")).unwrap(); + std::fs::write(base.join("sub/b.txt"), b"world").unwrap(); + std::fs::write(base.join("sub/c.txt"), b"!").unwrap(); + + let mut o = opts(); + o.base_path = base.to_path_buf(); + + let on = |paths: &[&str]| { + let policy = RawCachePolicy::OnChange { + paths: paths.iter().map(ToString::to_string).collect(), + }; + compute_cache_key("test", "cargo test", &policy, "scratch", &o).unwrap() + }; + + assert_eq!( + on(&["a.txt"]), + "1402470939ef19cd0e247f455f92f5430c9d3f1b6a7ebb59d2c1d249b39aecd4" + ); + assert_eq!( + on(&["sub"]), + "556e5a08d960fb2799dc319ced1907d18bfec3458fd2396d0316682c84894408" + ); + // Glob "*.txt" matches only a.txt at the top level -> same as ["a.txt"]. + assert_eq!( + on(&["*.txt"]), + "1402470939ef19cd0e247f455f92f5430c9d3f1b6a7ebb59d2c1d249b39aecd4" + ); + // A plain missing path is silently skipped -> hashes the empty preimage. + assert_eq!( + on(&["nope.txt"]), + "9dd1f573515149a2f42d1f7ed1ede5ca192d54c94f3367f6fcd4d3d3fc5b5c2c" + ); + assert_eq!( + on(&["a.txt", "sub"]), + "93adc574b21838be2ee79ec97053a2d1b5c2f20dcd514162131c7eb83f2563db" + ); +} + +#[test] +fn lower_with_options_resolves_chain_keys() { + let chain: RawStepChain = serde_json::from_str( + r#"{ + "steps": [ + {"cmd": "cargo build", "parent_idx": null, "label": "build", + "cache": {"policy": "forever"}}, + {"cmd": "cargo test", "parent_idx": 0, "label": "test", + "cache": {"policy": "forever", "env_keys": ["RUST_VERSION"]}} + ], + "leaf_indices": [1] + }"#, + ) + .unwrap(); + + let g = lower_with_options(&chain, Some(&opts())).unwrap(); + assert_eq!( + cache_key_for(&g, "build").as_deref(), + Some("507e8361f96eac5eb350787dc29906644eb93456320ed7a19b877de9f3fa7456") + ); + assert_eq!( + cache_key_for(&g, "test").as_deref(), + Some("15dee13d2b211a8bbe2647526a4a0735a42af9861dc4fbc4462d224dae8e53d5") + ); +} + +#[test] +fn lower_without_options_leaves_keys_unresolved() { + let chain: RawStepChain = serde_json::from_str( + r#"{ + "steps": [ + {"cmd": "cargo test", "parent_idx": null, "label": "test", + "cache": {"policy": "forever"}} + ], + "leaf_indices": [0] + }"#, + ) + .unwrap(); + + let g = lower_with_options(&chain, None).unwrap(); + assert_eq!(cache_key_for(&g, "test"), None); +} + +#[test] +fn cached_step_with_uncached_parent_errors() { + let chain: RawStepChain = serde_json::from_str( + r#"{ + "steps": [ + {"cmd": "cargo build", "parent_idx": null, "label": "build"}, + {"cmd": "cargo test", "parent_idx": 0, "label": "test", + "cache": {"policy": "forever"}} + ], + "leaf_indices": [1] + }"#, + ) + .unwrap(); + + let err = lower_with_options(&chain, Some(&opts())).unwrap_err(); + assert!( + err.to_string().contains("no cached key"), + "unexpected error: {err}" + ); +} From b73fa7f064f9b502d31089957193a820e7141f98 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 14:47:44 -0700 Subject: [PATCH 7/9] feat: SubprocessPythonEngine now lowers step chains in Rust --- crates/hm-dsl-engine/src/python_engine.rs | 77 ++++++++++++------- crates/hm-dsl-engine/src/raw_envelope.rs | 48 +++++++++++- .../hm-dsl-engine/tests/python_engine_test.rs | 38 +++++++++ 3 files changed, 132 insertions(+), 31 deletions(-) diff --git a/crates/hm-dsl-engine/src/python_engine.rs b/crates/hm-dsl-engine/src/python_engine.rs index 6c386320..dd1d5022 100644 --- a/crates/hm-dsl-engine/src/python_engine.rs +++ b/crates/hm-dsl-engine/src/python_engine.rs @@ -1,11 +1,13 @@ +use std::collections::BTreeMap; use std::path::Path; use std::process::Stdio; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, anyhow, bail}; use async_trait::async_trait; use tracing::debug; use crate::bundled_sources; +use crate::raw_envelope::{FinalEnvelope, RawEnvelope, process_raw_envelope_with_options}; use crate::{DslEngine, PipelineMeta}; const LIST_PIPELINES_SCRIPT: &str = "\ @@ -37,27 +39,6 @@ for p in sorted(pathlib.Path('.hm').glob('*.py')): sys.stdout.write(hm.dump_registry_json()) "; -const RENDER_PIPELINE_SCRIPT: &str = "\ -import sys, json, pathlib, importlib.util -try: - import harmont as hm -except ImportError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -slug = sys.argv[1] -for p in sorted(pathlib.Path('.hm').glob('*.py')): - spec = importlib.util.spec_from_file_location(f'_harmont_{p.stem}', p) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) -envelope = json.loads(hm.dump_registry_json()) -match = next((p for p in envelope['pipelines'] if p['slug'] == slug), None) -if match is None: - avail = ', '.join(p['slug'] for p in envelope['pipelines']) or '(none)' - print(f'error: pipeline {slug!r} not found\\n -> available: {avail}', file=sys.stderr) - sys.exit(2) -print(json.dumps(match['definition'])) -"; - #[derive(Debug)] pub struct SubprocessPythonEngine { python_bin: std::path::PathBuf, @@ -108,6 +89,33 @@ impl SubprocessPythonEngine { String::from_utf8(output.stdout).context("python3 stdout is not valid UTF-8") } + + /// Run the Python discovery script, deserialize the raw step-chain + /// envelope, and lower every pipeline into the v0 IR in Rust. + /// + /// Cache keys are resolved here (not in Python) using the same inputs the + /// legacy Python resolver used: `pipeline_org` from `HM_PIPELINE_ORG` + /// (falling back to `"default"`), the current unix time, `project_dir` as + /// the `on_change` base path, and the process environment. + async fn run_and_process_envelope(&self, project_dir: &Path) -> Result { + let raw_json = self + .run_script(project_dir, REGISTRY_JSON_SCRIPT, &[]) + .await?; + let raw: RawEnvelope = + serde_json::from_str(&raw_json).context("parsing raw envelope from Python")?; + + let env: BTreeMap = std::env::vars().collect(); + let org = env + .get("HM_PIPELINE_ORG") + .cloned() + .unwrap_or_else(|| "default".to_owned()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .context("system clock is before the unix epoch")? + .as_secs(); + + process_raw_envelope_with_options(raw, &org, now, project_dir, &env) + } } #[async_trait] @@ -124,15 +132,28 @@ impl DslEngine for SubprocessPythonEngine { } async fn render_pipeline_json(&self, project_dir: &Path, slug: &str) -> Result { - self.run_script(project_dir, RENDER_PIPELINE_SCRIPT, &[slug]) - .await - .context("rendering pipeline via python3") + let envelope = self.run_and_process_envelope(project_dir).await?; + let entry = envelope + .pipelines + .iter() + .find(|p| p.slug == slug) + .ok_or_else(|| { + let avail: String = envelope + .pipelines + .iter() + .map(|p| p.slug.as_str()) + .collect::>() + .join(", "); + let avail = if avail.is_empty() { "(none)" } else { &avail }; + anyhow!("pipeline {slug:?} not found\n -> available: {avail}") + })?; + serde_json::to_string(&entry.definition) + .with_context(|| format!("serializing definition for pipeline {slug:?}")) } async fn registry_json(&self, project_dir: &Path) -> Result { - self.run_script(project_dir, REGISTRY_JSON_SCRIPT, &[]) - .await - .context("dumping pipeline registry via python3") + let envelope = self.run_and_process_envelope(project_dir).await?; + serde_json::to_string(&envelope).context("serializing lowered discovery envelope") } } diff --git a/crates/hm-dsl-engine/src/raw_envelope.rs b/crates/hm-dsl-engine/src/raw_envelope.rs index 9111e169..6bc98bc1 100644 --- a/crates/hm-dsl-engine/src/raw_envelope.rs +++ b/crates/hm-dsl-engine/src/raw_envelope.rs @@ -6,9 +6,13 @@ //! carries the canonical v0 [`hm_pipeline_ir::PipelineGraph`], ready for the //! backend's pipeline discovery to consume. +use std::collections::BTreeMap; +use std::path::Path; + use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; +use crate::keygen::LowerOptions; use crate::lower; use crate::step_chain::RawStepChain; @@ -59,7 +63,45 @@ pub fn process_raw_envelope(raw: RawEnvelope) -> Result { let pipelines = raw .pipelines .into_iter() - .map(process_entry) + .map(|entry| process_entry(entry, None)) + .collect::>>()?; + Ok(FinalEnvelope { + schema_version: raw.schema_version, + pipelines, + }) +} + +/// Lower every pipeline's step chain with cache-key resolution enabled. +/// +/// Each pipeline is lowered with its own [`LowerOptions`], carrying the +/// pipeline's slug so resolved cache keys are namespaced per pipeline — +/// byte-for-byte matching the Python resolver (`pipeline_slug = reg.slug`). +/// +/// # Errors +/// +/// Returns an error if any pipeline's step chain fails to lower, if +/// cache-key resolution fails (e.g. a missing `on_change` path), or if the +/// resulting graph cannot be serialized. +pub fn process_raw_envelope_with_options( + raw: RawEnvelope, + pipeline_org: &str, + now: u64, + base_path: &Path, + env: &BTreeMap, +) -> Result { + let pipelines = raw + .pipelines + .into_iter() + .map(|entry| { + let opts = LowerOptions { + pipeline_org: pipeline_org.to_owned(), + pipeline_slug: entry.slug.clone(), + now, + base_path: base_path.to_path_buf(), + env: env.clone(), + }; + process_entry(entry, Some(&opts)) + }) .collect::>>()?; Ok(FinalEnvelope { schema_version: raw.schema_version, @@ -67,8 +109,8 @@ pub fn process_raw_envelope(raw: RawEnvelope) -> Result { }) } -fn process_entry(entry: RawPipelineEntry) -> Result { - let graph = lower::lower(&entry.step_chain) +fn process_entry(entry: RawPipelineEntry, opts: Option<&LowerOptions>) -> Result { + let graph = lower::lower_with_options(&entry.step_chain, opts) .with_context(|| format!("failed to lower pipeline '{}'", entry.slug))?; let definition = serde_json::to_value(&graph) .with_context(|| format!("failed to serialize definition for pipeline '{}'", entry.slug))?; diff --git a/crates/hm-dsl-engine/tests/python_engine_test.rs b/crates/hm-dsl-engine/tests/python_engine_test.rs index 81c29184..98c78e94 100644 --- a/crates/hm-dsl-engine/tests/python_engine_test.rs +++ b/crates/hm-dsl-engine/tests/python_engine_test.rs @@ -74,3 +74,41 @@ def ci() -> hm.Step: assert_eq!(p["triggers"][0]["branches"][0], "main"); assert_eq!(p["definition"]["version"], "0"); } + +/// A cached step's `cache.key` is resolved by the Rust lowering pass after the +/// Python subprocess emits the raw step chain — proving cache-key resolution +/// moved off the Python side. +#[tokio::test] +async fn python_render_resolves_cache_keys_in_rust() { + if which::which("python3").is_err() { + eprintln!("skipping: python3 not on PATH"); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let harmont = dir.path().join(".hm"); + std::fs::create_dir_all(&harmont).unwrap(); + std::fs::write( + harmont.join("ci.py"), + r#"import harmont as hm + +@hm.pipeline('ci') +def ci() -> hm.Step: + return hm.scratch().sh('curl example.com | tar xz', label='fetch', cache=hm.forever()) +"#, + ) + .unwrap(); + + let engine = hm_dsl_engine::python_engine().unwrap(); + let json_str = engine.render_pipeline_json(dir.path(), "ci").await.unwrap(); + let v: serde_json::Value = serde_json::from_str(&json_str).unwrap(); + + let cache = &v["graph"]["nodes"][0]["step"]["cache"]; + assert_eq!(cache["policy"], "forever"); + let key = cache["key"].as_str().expect("cache.key must be resolved"); + assert_eq!(key.len(), 64, "outer key is a sha256 hex digest: {key}"); + assert!( + key.chars().all(|c| c.is_ascii_hexdigit()), + "cache key is not hex: {key}" + ); +} From 431861ebfd974e3065146eac68ff1cd97ee5607e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 15:08:48 -0700 Subject: [PATCH 8/9] chore: remove Python lowering code superseded by Rust implementation --- .../harmont-py/harmont/__init__.py | 60 +-- .../hm-dsl-engine/harmont-py/harmont/_keys.py | 121 ----- .../harmont-py/harmont/_pipeline.py | 259 ---------- .../harmont-py/harmont/_serialize.py | 64 ++- .../harmont-py/harmont/json_emit.py | 63 --- .../harmont-py/harmont/keygen.py | 172 ------- .../harmont-py/tests/test_cmake.py | 94 ++-- .../harmont-py/tests/test_e2e_fixtures.py | 171 ------- .../harmont-py/tests/test_elixir.py | 43 +- .../harmont-py/tests/test_examples_render.py | 23 +- .../hm-dsl-engine/harmont-py/tests/test_go.py | 36 +- .../harmont-py/tests/test_har_28_example.py | 16 +- .../hm-dsl-engine/harmont-py/tests/test_js.py | 7 +- .../harmont-py/tests/test_json_emit.py | 246 --------- .../harmont-py/tests/test_keygen.py | 467 ------------------ .../harmont-py/tests/test_keys.py | 97 ---- .../harmont-py/tests/test_pipeline.py | 64 --- .../tests/test_pipeline_fixtures.py | 18 +- .../tests/test_pipeline_lowering.py | 156 ------ .../harmont-py/tests/test_py_uv.py | 71 +-- .../harmont-py/tests/test_python.py | 45 +- .../harmont-py/tests/test_rust.py | 101 ++-- .../harmont-py/tests/test_setup.py | 24 +- .../tests/test_target_cross_module.py | 4 +- .../tests/test_toolchain_compose.py | 33 +- .../harmont-py/tests/test_zig.py | 36 +- .../harmont-py/tests/test_zig_toolchain.py | 43 +- crates/hm-dsl-engine/src/bundled_sources.rs | 10 +- crates/hm-pipeline-ir/tests/e2e_fixtures.rs | 151 ------ tests/e2e/fixtures/python/cmake-advanced.json | 124 ----- tests/e2e/fixtures/python/kitchen-sink.json | 204 -------- tests/e2e/fixtures/python/monorepo-ci.json | 328 ------------ tests/e2e/fixtures/python/rust-release.json | 136 ----- .../fixtures/python/zig-node-polyglot.json | 213 -------- 34 files changed, 325 insertions(+), 3375 deletions(-) delete mode 100644 crates/hm-dsl-engine/harmont-py/harmont/_keys.py delete mode 100644 crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py delete mode 100644 crates/hm-dsl-engine/harmont-py/harmont/json_emit.py delete mode 100644 crates/hm-dsl-engine/harmont-py/harmont/keygen.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_keygen.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_keys.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_pipeline.py delete mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py delete mode 100644 crates/hm-pipeline-ir/tests/e2e_fixtures.rs delete mode 100644 tests/e2e/fixtures/python/cmake-advanced.json delete mode 100644 tests/e2e/fixtures/python/kitchen-sink.json delete mode 100644 tests/e2e/fixtures/python/monorepo-ci.json delete mode 100644 tests/e2e/fixtures/python/rust-release.json delete mode 100644 tests/e2e/fixtures/python/zig-node-polyglot.json diff --git a/crates/hm-dsl-engine/harmont-py/harmont/__init__.py b/crates/hm-dsl-engine/harmont-py/harmont/__init__.py index 9448277f..554f1a15 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/__init__.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/__init__.py @@ -8,9 +8,6 @@ Step.fork(label=None) -> Step wait(*, continue_on_failure=False) -> Step - pipeline(leaves, *, env=None) -> dict (v0 IR) - pipeline_to_json(p, **kw) -> str - @pipeline(slug, ..., triggers=[...], allow_manual=True) -> decorator push(branch=..., tag=...) -> PushTrigger pull_request(branches=..., types=...) -> PullRequestTrigger @@ -18,10 +15,9 @@ Cache helpers: `ttl`, `on_change`, `forever`, `compose`. -``hm.pipeline`` is polymorphic. When called with a list of ``Step`` -objects it builds a v0 IR dict (the factory). When called with no -positionals or a string slug it returns a decorator that registers a -function as a CI pipeline (HAR-9). +``hm.pipeline`` is a decorator that registers a function as a CI pipeline +(HAR-9). Lowering to the v0 IR happens on the Rust side after the envelope +is emitted by ``dump_registry_json``. """ from __future__ import annotations @@ -36,8 +32,6 @@ from ._envelope import dump_registry_json from ._go import go from ._js import JsProject, js, ts -from ._pipeline import pipeline as _pipeline_factory -from ._pipeline import pipeline_to_json from ._python import python from ._rust import RustProject, rust from ._step import Step, scratch, wait @@ -62,46 +56,33 @@ def pipeline(*args: Any, **kwargs: Any) -> Any: - """Build a v0 IR dict or register a pipeline function. - - This function is polymorphic based on the type of its positional arguments. - - Factory form — first positional is a list/tuple of ``Step``s: - - pipeline([step1, step2, ...], env=None) -> dict - - Decorator form — no positionals or a string slug: + """Register a function as a CI pipeline (decorator form). @pipeline(slug=None, *, name=None, triggers=(), allow_manual=True, - env=None) + env=None, timeout=None) def my_pipeline() -> Step: ... - The discriminant is the type of the first positional argument: - a list or tuple routes to the factory path; anything else - (including no positionals) routes to the decorator path. + The decorated function returns the terminal ``Step`` (or a tuple/list of + leaves) of each branch. Lowering to the v0 IR happens on the Rust side + after ``dump_registry_json`` emits the envelope. Returns: - A v0 IR ``dict`` in factory form, or a decorator in decorator form. + A decorator that registers the wrapped function and returns it. Raises: - TypeError: When called with the legacy variadic ``Step`` form - (``pipeline(step)`` / ``pipeline(a, b)``). The factory now takes - a single list of leaves. + TypeError: When called with ``Step`` objects or a list of leaves — + the pre-CLI-45 factory form (``pipeline([step, ...])``) is gone; + annotate a function with ``@pipeline`` and return the leaves. """ - if args and isinstance(args[0], (list, tuple)): - return _pipeline_factory(args[0], **kwargs) - # Legacy form: leaves passed as positional Step args (pre-CLI-9 - # `pipeline(step)` / `pipeline(a, b)`). Without this guard the call would - # fall through to the decorator and fail far downstream with a cryptic - # AttributeError. Fail fast with the migration hint instead. - if args and all(isinstance(a, Step) for a in args): + if args and ( + isinstance(args[0], (list, tuple)) or all(isinstance(a, Step) for a in args) + ): msg = ( - "hm.pipeline() takes a single list of leaves, not variadic Step " - "arguments\n" - f" observed: {len(args)} positional Step " - f"argument{'s' if len(args) != 1 else ''}\n" - " → wrap the leaves in a list, e.g. " - "hm.pipeline([step]) or hm.pipeline([a, b])" + "hm.pipeline is a decorator, not a factory\n" + " → annotate a function and return the leaves, e.g.\n" + " @hm.pipeline('ci')\n" + " def ci():\n" + " return [step_a, step_b]" ) raise TypeError(msg) return _decorator.pipeline(*args, **kwargs) @@ -322,7 +303,6 @@ def group(steps: list[Step] | tuple[Step, ...]) -> tuple[Step, ...]: "js", "on_change", "pipeline", - "pipeline_to_json", "pr", "pull_request", "push", diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_keys.py b/crates/hm-dsl-engine/harmont-py/harmont/_keys.py deleted file mode 100644 index 0e440001..00000000 --- a/crates/hm-dsl-engine/harmont-py/harmont/_keys.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Key derivation for chain-DSL steps. - -Order of precedence per the design doc: - 1. explicit `key=` override on .sh() - 2. slugified label (when unique within the pipeline) - 3. stable 12-char hash of (parent_resolved_key, cmd, position) - -Collision policy: when two steps' label-slugs collide and neither -claimed the slug via explicit `key=`, both fall back to hash. An -explicit override always wins, even if it would collide with another -step's natural slug. -""" - -from __future__ import annotations - -import hashlib -import re -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Iterable - - from ._step import Step - -_EMOJI_SHORTCODE_RE = re.compile(r":[a-z0-9_+-]+:") -_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+") - - -def slugify_label(label: str) -> str: - """Lowercase, strip ``:emoji_codes:``, replace non-alnum runs with ``-``, - trim leading/trailing dashes. - - Slugs are ASCII-only by policy (matches Buildkite). Non-ASCII - letters are treated as separators: ``"Café Build"`` slugs to - ``"caf-build"`` and ``"构建"`` slugs to ``""``. Labels that reduce - to the empty string fall back to a hash key in ``resolve_keys``; - the user's label is preserved on the step's ``label`` field for - display, only the cross-reference key is hash-based. - """ - s = label.lower() - s = _EMOJI_SHORTCODE_RE.sub(" ", s) - s = _NON_ALNUM_RE.sub("-", s) - return s.strip("-") - - -def hash_key(parent_key: str, cmd: str, position: int) -> str: - """Stable 12-char SHA-256 prefix over (parent_key, cmd, position). - - Used as the fallback key when no usable slug is available.""" - h = hashlib.sha256() - h.update(parent_key.encode("utf-8")) - h.update(b"\x00") - h.update(cmd.encode("utf-8")) - h.update(b"\x00") - h.update(str(position).encode("utf-8")) - return h.hexdigest()[:12] - - -def resolve_keys(steps: Iterable[Step]) -> dict[int, str]: - """Resolve each Step's key. Returns ``{id(step): key}``. - - The ``id()`` indexing is deliberate: two structurally-equal Steps - that arose from independent fork branches must keep distinct keys, - and frozen-dataclass equality would conflate them. - """ - steps_list = list(steps) - - overrides: dict[int, str] = {} - # Natural slug per step (computed for every labeled step, even - # those with explicit overrides — see slug_counts below). - natural_slugs: dict[int, str] = {} - for s in steps_list: - if s.key_override is not None: - overrides[id(s)] = s.key_override - if s.label is not None: - slug = slugify_label(s.label) - if slug: - natural_slugs[id(s)] = slug - - # Reserve every override; any natural slug that matches a reserved - # override is a collision for the slug claimant. - reserved = set(overrides.values()) - - # Detect slug collisions across every labeled step — including those - # with explicit overrides. An override-bearing step still "claims" - # its natural slug for collision purposes, so a peer with the same - # label can't quietly take it. - slug_counts: dict[str, int] = {} - for slug in natural_slugs.values(): - slug_counts[slug] = slug_counts.get(slug, 0) + 1 - - # The slug pool that non-override steps may draw from: only steps - # without a `key=` override are eligible to receive their slug. - label_slugs: dict[int, str] = { - sid: slug for sid, slug in natural_slugs.items() if sid not in overrides - } - - keys: dict[int, str] = {} - for position, s in enumerate(steps_list): - sid = id(s) - if sid in overrides: - keys[sid] = overrides[sid] - continue - candidate_slug = label_slugs.get(sid) - if ( - candidate_slug is not None - and candidate_slug not in reserved - and slug_counts[candidate_slug] == 1 - ): - keys[sid] = candidate_slug - reserved.add(candidate_slug) - continue - # Fall back to hash. Parent resolved key may not be in `keys` - # yet; use the empty string as a sentinel — call sites that - # need the resolved parent_key pass it explicitly via the - # lowering pass (see pipeline.py). - parent_key = "" - if s.parent is not None and id(s.parent) in keys: - parent_key = keys[id(s.parent)] - keys[sid] = hash_key(parent_key, s.cmd or "", position) - return keys diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py b/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py deleted file mode 100644 index 38b25ea1..00000000 --- a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py +++ /dev/null @@ -1,259 +0,0 @@ -"""Pipeline factory + lowering pass. - -The factory walks back from each leaf via `Step.parent`, collects every -unique step (keyed by `id`, since structurally-equal forks must keep -distinct keys), topo-sorts by parent edges with a stable -leaf-then-DFS-pre tiebreaker, and lowers each step to the petgraph-serde -graph format matching the v0 IR schema. - -Use `pipeline_to_json` from `json_emit` to emit the wire-format string. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from ._duration import parse_duration -from ._keys import resolve_keys -from .cache import ( - CacheCompose, - CacheForever, - CacheNone, - CacheOnChange, - CachePolicy, - CacheTTL, -) - -if TYPE_CHECKING: - from ._step import Step - -# Across-the-board default image for imageless root steps. The SDK's -# toolchains assume an apt-capable base (apt-get), so ubuntu:24.04 is the -# universal default; child steps boot from their parent's snapshot and -# stay imageless. -DEFAULT_IMAGE = "ubuntu:24.04" - - -def pipeline( - leaves: list[Step] | tuple[Step, ...], - *, - env: dict[str, str] | None = None, - timeout: str | int | None = None, -) -> dict[str, Any]: - """Top-level factory. Returns a JSON-shaped dict (version "0"). - - Every imageless root command step (one with no ``builds_in`` parent - and no per-step ``image``) is stamped with ``DEFAULT_IMAGE`` - (``ubuntu:24.04``). Set a per-step ``image=`` on a step to override. - - ``timeout`` is a whole-build wall-clock budget (``"30m"``, ``"1h"``, - or an int number of seconds). When it elapses the build is killed and - fails as *timed out*, regardless of how far the step graph got. - """ - if not leaves: - msg = ( - "pipeline must have at least one leaf — " - "pass the terminal step(s) of each branch in the first argument" - ) - raise ValueError(msg) - out: dict[str, Any] = {"version": "0"} - if timeout is not None: - out["timeout_seconds"] = parse_duration(timeout) - out["graph"] = _lower_to_graph(list(leaves), env=env) - return out - - -def _lower_to_graph( - leaves: list[Step], - *, - env: dict[str, str] | None = None, -) -> dict[str, Any]: - """Walk back via `parent`, topo-sort, emit petgraph-serde graph dict. - - `scratch` and `fork` nodes carry no command and are not emitted as - graph nodes; they exist only to set the `parent` of their children. - Wait steps are not emitted as nodes — they are translated into - explicit ``depends_on`` edges. - """ - ordered = _topo_collect(leaves) - command_steps = [s for s in ordered if s.cmd is not None and not s.is_wait] - keys = resolve_keys(command_steps) - - # Assign integer node indices (dense, in emission order). - idx_by_id: dict[int, int] = {} - for i, s in enumerate(command_steps): - idx_by_id[id(s)] = i - - # Track which node indices have a builds_in parent (child steps stay image-less). - has_builds_in_parent: set[int] = set() - - nodes: list[dict[str, Any]] = [] - edges: list[list[Any]] = [] - - # Collect all command-step indices emitted before each wait barrier. - # When we encounter a wait, every step after the wait gets a - # depends_on edge from every step before the wait. - pre_wait_indices: list[int] = [] - # Pending depends_on sources (from the most recent wait barrier). - pending_depends_on: list[int] = [] - - for s in ordered: - if s.is_wait: - # All command-step indices emitted so far (after the last wait) - # become sources for depends_on edges to subsequent steps. - pending_depends_on = list(pre_wait_indices) - pre_wait_indices = [] - continue - - if s.cmd is None: - # scratch or fork — passthrough, not emitted. - continue - - node_idx = idx_by_id[id(s)] - step_key = keys[id(s)] - - # Build the CommandStep dict (no "type" or "builds_in" fields). - step_dict: dict[str, Any] = { - "key": step_key, - "cmd": s.cmd, - } - if s.label is not None: - step_dict["label"] = s.label - if s.cache is not None: - step_dict["cache"] = _cache_to_dict(s.cache) - if s.timeout_seconds is not None: - step_dict["timeout_seconds"] = s.timeout_seconds - if s.image is not None: - step_dict["image"] = s.image - if s.runner is not None: - step_dict["runner"] = s.runner - if s.runner_args is not None: - step_dict["runner_args"] = s.runner_args - - # Baseline env for non-interactive operation inside VMs/containers. - merged_env: dict[str, str] = { - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb", - } - if env: - merged_env.update(env) - if s.env: - merged_env.update(s.env) - - nodes.append({"step": step_dict, "env": merged_env}) - - # builds_in edge from parent. - parent_key = _resolved_parent_key(s, keys) - if parent_key is not None: - parent_idx = _find_idx_by_key(parent_key, command_steps, keys, idx_by_id) - edges.append([parent_idx, node_idx, "builds_in"]) - has_builds_in_parent.add(node_idx) - - # depends_on edges from pre-wait steps. - edges.extend([dep_idx, node_idx, "depends_on"] for dep_idx in pending_depends_on) - - pre_wait_indices.append(node_idx) - - # Stamp the default image on every root command step that lacks an - # explicit one. Root steps boot from an image tag (not a parent - # snapshot); child steps inherit the parent's committed snapshot and - # must stay image-less. - for i, node in enumerate(nodes): - if i not in has_builds_in_parent and "image" not in node["step"]: - node["step"]["image"] = DEFAULT_IMAGE - - return { - "nodes": nodes, - "node_holes": [], - "edge_property": "directed", - "edges": edges, - } - - -def _find_idx_by_key( - key: str, - command_steps: list[Step], - keys: dict[int, str], - idx_by_id: dict[int, int], -) -> int: - """Return the node index for the step with the given resolved key.""" - for s in command_steps: - if keys[id(s)] == key: - return idx_by_id[id(s)] - msg = f"BUG: no step with key {key!r}" - raise KeyError(msg) - - -def _topo_collect(leaves: list[Step]) -> list[Step]: - """Collect every Step reachable from `leaves` via `parent`, return them - in parent-before-child order. Tiebreak by leaf order, then DFS-pre on - each leaf chain (deterministic). Wait steps are inserted in their - leaf-tuple position.""" - seen: set[int] = set() - ordered: list[Step] = [] - - for leaf in leaves: - if leaf.is_wait: - ordered.append(leaf) - continue - chain: list[Step] = [] - node: Step | None = leaf - while node is not None: - if id(node) in seen: - break - chain.append(node) - node = node.parent - # chain is leaf -> root order; reverse for parent-first. - for s in reversed(chain): - if id(s) in seen: - continue - seen.add(id(s)) - ordered.append(s) - return ordered - - -def _resolved_parent_key(s: Step, keys: dict[int, str]) -> str | None: - """Walk back through scratch/fork nodes to the nearest emitted ancestor.""" - node = s.parent - while node is not None: - if node.cmd is not None and not node.is_wait: - return keys[id(node)] - node = node.parent - return None - - -def _cache_to_dict(policy: CachePolicy) -> dict[str, Any]: - """Render a CachePolicy to its JSON-shape dict. - - Cache key resolution happens in keygen.resolve_pipeline_keys after - the pipeline structure is built. - """ - if isinstance(policy, CacheNone): - return {"policy": "none"} - if isinstance(policy, CacheForever): - return {"policy": "forever", "env_keys": list(policy.env_keys)} - if isinstance(policy, CacheTTL): - return { - "policy": "ttl", - "duration_seconds": int(policy.duration.total_seconds()), - "env_keys": list(policy.env_keys), - } - if isinstance(policy, CacheOnChange): - return {"policy": "on_change", "paths": list(policy.paths)} - if isinstance(policy, CacheCompose): - return { - "policy": "compose", - "sub_policies": [_cache_to_dict(p) for p in policy.policies], - } - msg = f"unknown CachePolicy: {type(policy).__name__}" - raise TypeError(msg) - - -from .json_emit import pipeline_to_json as _pipeline_to_json # noqa: E402 - - -def pipeline_to_json(p: dict[str, Any], **kw: Any) -> str: - """Convenience re-export so callers can do - ``harmont.pipeline_to_json(pipeline(...))`` without importing - `json_emit` directly. See `json_emit.pipeline_to_json` for kwargs.""" - return _pipeline_to_json(p, **kw) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py b/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py index ec187672..3bbd86bc 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_serialize.py @@ -17,7 +17,14 @@ from typing import TYPE_CHECKING, Any from ._duration import parse_duration -from ._pipeline import _cache_to_dict, _topo_collect +from .cache import ( + CacheCompose, + CacheForever, + CacheNone, + CacheOnChange, + CachePolicy, + CacheTTL, +) if TYPE_CHECKING: from ._step import Step @@ -90,3 +97,58 @@ def _serialize_step(step: Step, idx_by_id: dict[int, int]) -> dict[str, Any]: if step.key_override is not None: d["key_override"] = step.key_override return d + + +def _topo_collect(leaves: list[Step]) -> list[Step]: + """Collect every Step reachable from `leaves` via `parent`, return them + in parent-before-child order. Tiebreak by leaf order, then DFS-pre on + each leaf chain (deterministic). Wait steps are inserted in their + leaf-tuple position.""" + seen: set[int] = set() + ordered: list[Step] = [] + + for leaf in leaves: + if leaf.is_wait: + ordered.append(leaf) + continue + chain: list[Step] = [] + node: Step | None = leaf + while node is not None: + if id(node) in seen: + break + chain.append(node) + node = node.parent + # chain is leaf -> root order; reverse for parent-first. + for s in reversed(chain): + if id(s) in seen: + continue + seen.add(id(s)) + ordered.append(s) + return ordered + + +def _cache_to_dict(policy: CachePolicy) -> dict[str, Any]: + """Render a CachePolicy to its JSON-shape dict. + + Cache key resolution happens on the Rust side after the pipeline + structure is built. + """ + if isinstance(policy, CacheNone): + return {"policy": "none"} + if isinstance(policy, CacheForever): + return {"policy": "forever", "env_keys": list(policy.env_keys)} + if isinstance(policy, CacheTTL): + return { + "policy": "ttl", + "duration_seconds": int(policy.duration.total_seconds()), + "env_keys": list(policy.env_keys), + } + if isinstance(policy, CacheOnChange): + return {"policy": "on_change", "paths": list(policy.paths)} + if isinstance(policy, CacheCompose): + return { + "policy": "compose", + "sub_policies": [_cache_to_dict(p) for p in policy.policies], + } + msg = f"unknown CachePolicy: {type(policy).__name__}" + raise TypeError(msg) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/json_emit.py b/crates/hm-dsl-engine/harmont-py/harmont/json_emit.py deleted file mode 100644 index 755129d9..00000000 --- a/crates/hm-dsl-engine/harmont-py/harmont/json_emit.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Render a chain-DSL pipeline dict to the v0 IR JSON string. - -The wire format uses petgraph-serde graph encoding: nodes carry -CommandStep dicts and edges encode ``builds_in`` / ``depends_on`` -relationships. - -Cache keys are resolved in keygen.resolve_pipeline_keys before -serialization, so the emitted JSON includes `cache.key` for every -node whose policy is not 'none'. -""" - -from __future__ import annotations - -import copy -import json -import os -import time -from pathlib import Path -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from collections.abc import Mapping - -from .keygen import resolve_pipeline_keys - - -def pipeline_to_json( - p: dict[str, Any], - *, - pipeline_org: str | None = None, - pipeline_slug: str | None = None, - now: int | None = None, - base_path: Path | None = None, - env: Mapping[str, str] | None = None, -) -> str: - """Render the pipeline dict (as returned by `pipeline(...)`) to JSON. - - Resolves cache keys before serialization. Defaults mirror the - environment hooks of the old Scheme renderer: - pipeline_org <- env["HM_PIPELINE_ORG"] or "default" - pipeline_slug <- env["HM_PIPELINE_SLUG"] or "default" - now <- int(time.time()) - base_path <- Path.cwd() - env <- os.environ - """ - env_map: Mapping[str, str] = env if env is not None else os.environ - org = pipeline_org if pipeline_org is not None else env_map.get("HM_PIPELINE_ORG", "default") - slug = ( - pipeline_slug if pipeline_slug is not None else env_map.get("HM_PIPELINE_SLUG", "default") - ) - render_now = now if now is not None else int(time.time()) - bp = base_path if base_path is not None else Path.cwd() - - body = copy.deepcopy(p) - resolve_pipeline_keys( - body.get("graph", {}), - pipeline_org=org, - pipeline_slug=slug, - now=render_now, - base_path=bp, - env=env_map, - ) - return json.dumps(body, ensure_ascii=False, separators=(", ", ": ")) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/keygen.py b/crates/hm-dsl-engine/harmont-py/harmont/keygen.py deleted file mode 100644 index f7285c33..00000000 --- a/crates/hm-dsl-engine/harmont-py/harmont/keygen.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Cache-key resolver. - -Direct port of cidsl/lisp/src/harmont_macros.scm (resolve-cache-key -and helpers). Output bytes MUST match the Scheme version so cached -snapshots persisted before the Scheme removal remain reachable. - -Algorithm (pre-image of the outer sha256): - - pipeline_org NUL pipeline_slug NUL step_key NUL - parent_resolved_key NUL policy_resolution - -policy_resolution branches: - none -> "none" (no key emitted) - forever -> "forever-" + sha256(cmd NUL env_subset) - ttl -> "ttl-N-" + sha256(cmd NUL env_subset) N = now // duration - on_change -> "sha-" + sha256(concat(file_hash(p) NUL for p in sorted)) - compose -> "compose-" + sha256(concat(resolve(sub) or "none")) - -The Scheme `cache-when` policy is removed (see HAR-16) — it required a -Scheme sandbox that no longer exists. -""" - -from __future__ import annotations - -import hashlib -from pathlib import Path # noqa: TC003 used at runtime in _path_hash -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from collections.abc import Mapping - -NUL = "\x00" - - -def resolve_pipeline_keys( - graph: dict[str, Any], - *, - pipeline_org: str, - pipeline_slug: str, - now: int, - base_path: Path, - env: Mapping[str, str], -) -> dict[str, Any]: - """Walk graph nodes in order. For every node whose cache policy is not - 'none', compute a deterministic sha256 cache key and inject it into - that node's step ``cache`` dict as ``cache["key"]``. Returns the - same graph dict (mutated in place -- callers may rely on identity).""" - nodes = graph.get("nodes", []) - edges = graph.get("edges", []) - - # Build parent key map from builds_in edges. - key_by_idx: dict[int, str] = {i: n["step"]["key"] for i, n in enumerate(nodes)} - parent_key_map: dict[str, str] = {} - for src, dst, kind in edges: - if kind == "builds_in": - parent_key_map[key_by_idx[dst]] = key_by_idx[src] - - resolved: dict[str, str] = {} - for node in nodes: - step = node["step"] - cache = step.get("cache") - if not cache or cache["policy"] == "none": - continue - cmd = step.get("cmd", "") - parent = parent_key_map.get(step["key"]) - parent_resolved = _lookup_parent(parent, resolved) - policy_res = _resolve_policy(cache, cmd, now, base_path, env) - key = _sha256_hex( - pipeline_org - + NUL - + pipeline_slug - + NUL - + step["key"] - + NUL - + parent_resolved - + NUL - + policy_res - ) - cache["key"] = key - resolved[step["key"]] = key - return graph - - -def _lookup_parent(parent: str | None, resolved: dict[str, str]) -> str: - if parent is None: - return "scratch" - key = resolved.get(parent) - if key is None: - msg = ( - f"step references builds_in {parent!r} which has no cached " - f"key (parent must be defined upstream and cached)" - ) - raise ValueError(msg) - return key - - -def _resolve_policy( - policy: dict[str, Any], - cmd: str, - now: int, - base_path: Path, - env: Mapping[str, str], -) -> str: - kind = policy["policy"] - if kind == "none": - return "none" - if kind == "forever": - env_keys = policy.get("env_keys", []) - return "forever-" + _sha256_hex(cmd + NUL + _env_subset(env_keys, env)) - if kind == "ttl": - duration = policy["duration_seconds"] - bucket = now // duration - env_keys = policy.get("env_keys", []) - return "ttl-" + str(bucket) + "-" + _sha256_hex(cmd + NUL + _env_subset(env_keys, env)) - if kind == "on_change": - resolved: list[Path] = [] - for p in sorted(policy["paths"]): - if any(c in p for c in ("*", "?", "[")): - resolved.extend(sorted(base_path.glob(p))) - else: - full = base_path / p - if full.exists(): - resolved.append(full) - pre = "".join(_path_hash(r) + NUL for r in resolved) - return "sha-" + _sha256_hex(pre) - if kind == "compose": - subs = policy["sub_policies"] - parts = [ - _resolve_policy(sub, cmd, now, base_path, env) if sub["policy"] != "none" else "none" - for sub in subs - ] - return "compose-" + _sha256_hex("".join(parts)) - msg = f"resolve-policy-key: unknown policy {kind!r}" - raise ValueError(msg) - - -def _env_subset(env_keys: list[str], env: Mapping[str, str]) -> str: - sorted_keys = sorted(env_keys) - return "".join(k + "=" + env.get(k, "") + NUL for k in sorted_keys) - - -def _path_hash(path: Path) -> str: - """Hash a path's content for an `on_change` cache key. - - Files: hash the bytes. - - Directories: walk recursively in sorted order and fold each file's - POSIX-style relative path + content into one SHA-256 stream. Empty - directories hash to the empty stream's digest, which is stable. - - Missing paths fail loudly: ``on_change`` is a build-time invariant - and a typo should not silently weaken the cache key. - """ - if path.is_file(): - with path.open("rb") as fp: - return hashlib.sha256(fp.read()).hexdigest() - if path.is_dir(): - h = hashlib.sha256() - files = sorted(p for p in path.rglob("*") if p.is_file()) - for child in files: - rel = child.relative_to(path).as_posix() - h.update(rel.encode("utf-8")) - h.update(b"\x00") - h.update(child.read_bytes()) - h.update(b"\x00") - return h.hexdigest() - msg = f"on_change path does not exist: {path}" - raise FileNotFoundError(msg) - - -def _sha256_hex(s: str) -> str: - return hashlib.sha256(s.encode("utf-8")).hexdigest() diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py b/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py index 6615498c..a883af22 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py @@ -5,10 +5,12 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] # --------------------------------------------------------------------------- @@ -19,8 +21,7 @@ def _cmds(p: dict) -> list[str]: class TestCMakeToolchain: def test_default_toolchain_installs_cmake_ninja_ccache(self): tc = hm.cmake() - p = hm.pipeline([tc.installed]) - cmds = _cmds(p) + cmds = _cmds([tc.installed]) apt_cmd = next(c for c in cmds if "apt-get install" in c) assert "cmake" in apt_cmd assert "ninja-build" in apt_cmd @@ -28,15 +29,13 @@ def test_default_toolchain_installs_cmake_ninja_ccache(self): def test_clang_18_compiler_installs_clang_18(self): tc = hm.cmake(compiler="clang-18") - p = hm.pipeline([tc.installed]) - cmds = _cmds(p) + cmds = _cmds([tc.installed]) apt_cmd = next(c for c in cmds if "apt-get install" in c) assert "clang-18" in apt_cmd def test_gcc_14_compiler_installs_gcc_14_and_gpp_14(self): tc = hm.cmake(compiler="gcc-14") - p = hm.pipeline([tc.installed]) - cmds = _cmds(p) + cmds = _cmds([tc.installed]) apt_cmd = next(c for c in cmds if "apt-get install" in c) assert "gcc-14" in apt_cmd assert "g++-14" in apt_cmd @@ -48,8 +47,7 @@ def test_invalid_compiler_raises_valueerror(self): def test_ccache_false_omits_ccache_from_apt_and_flags(self): tc = hm.cmake(ccache=False) proj = tc.project(path=".") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) apt_cmd = next(c for c in cmds if "apt-get install" in c) assert "ccache" not in apt_cmd configure_cmd = next(c for c in cmds if "cmake -S" in c) @@ -60,8 +58,7 @@ def test_toolchain_shared_across_projects_single_apt_install(self): tc = hm.cmake() proj1 = tc.project(path="svc1") proj2 = tc.project(path="svc2") - p = hm.pipeline([proj1.built, proj2.built]) - cmds = _cmds(p) + cmds = _cmds([proj1.built, proj2.built]) apt_installs = [c for c in cmds if "apt-get install" in c] assert len(apt_installs) == 1 @@ -74,15 +71,13 @@ def test_toolchain_shared_across_projects_single_apt_install(self): class TestCMakeProject: def test_build_produces_configure_and_build_commands(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) assert any("cmake -S . -B build" in c for c in cmds) assert any("cmake --build" in c for c in cmds) def test_warmup_uses_relative_build_dir_after_cd(self): proj = hm.cmake(path="infra/agent") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) warmup = next(c for c in cmds if "cmake -S . -B build" in c) assert "cd infra/agent" in warmup assert "cmake --build build " in warmup @@ -90,95 +85,82 @@ def test_warmup_uses_relative_build_dir_after_cd(self): def test_uses_ninja_generator_by_default(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "-G Ninja" in configure_cmd def test_no_build_type_by_default(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "CMAKE_BUILD_TYPE" not in configure_cmd def test_defines_cmake_build_type(self): proj = hm.cmake(path="svc", defines={"CMAKE_BUILD_TYPE": "Debug"}) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "CMAKE_BUILD_TYPE=Debug" in configure_cmd def test_defines_produces_d_flags(self): proj = hm.cmake(path="svc", defines={"BUILD_TESTING": "ON"}) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "-DBUILD_TESTING=ON" in configure_cmd def test_defines_build_shared_libs(self): proj = hm.cmake(path="svc", defines={"BUILD_SHARED_LIBS": "ON"}) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "-DBUILD_SHARED_LIBS=ON" in configure_cmd def test_defines_cmake_cxx_standard(self): proj = hm.cmake(path="svc", defines={"CMAKE_CXX_STANDARD": "20"}) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "-DCMAKE_CXX_STANDARD=20" in configure_cmd def test_preset_produces_preset_flag_and_no_build_type(self): proj = hm.cmake(path="svc", preset="ci-linux") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "--preset" in c) assert "--preset ci-linux" in configure_cmd assert "CMAKE_BUILD_TYPE" not in configure_cmd def test_ccache_true_adds_compiler_launcher_flags(self): proj = hm.cmake(path="svc", ccache=True) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) configure_cmd = next(c for c in cmds if "cmake -S" in c) assert "-DCMAKE_C_COMPILER_LAUNCHER=ccache" in configure_cmd assert "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" in configure_cmd def test_test_produces_ctest_with_output_on_failure_and_parallel(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.test()]) - cmds = _cmds(p) + cmds = _cmds([proj.test()]) test_cmd = next(c for c in cmds if "ctest" in c) assert "--output-on-failure" in test_cmd assert "--parallel" in test_cmd def test_test_includes_incremental_build(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.test()]) - cmds = _cmds(p) + cmds = _cmds([proj.test()]) test_cmd = next(c for c in cmds if "ctest" in c) assert "cmake --build" in test_cmd def test_test_uses_absolute_path_for_standalone_step(self): proj = hm.cmake(path="infra/agent") - p = hm.pipeline([proj.test()]) - cmds = _cmds(p) + cmds = _cmds([proj.test()]) test_cmd = next(c for c in cmds if "ctest" in c) assert "cmake --build infra/agent/build" in test_cmd def test_install_with_prefix(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.install(prefix="/usr/local")]) - cmds = _cmds(p) + cmds = _cmds([proj.install(prefix="/usr/local")]) install_cmd = next(c for c in cmds if "cmake --install" in c) assert "--prefix /usr/local" in install_cmd def test_fmt_runs_clang_format_dry_run(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.fmt()]) - cmds = _cmds(p) + cmds = _cmds([proj.fmt()]) fmt_cmd = next(c for c in cmds if "xargs clang-format" in c) assert "--dry-run --Werror" in fmt_cmd assert "-not -path './build/*'" in fmt_cmd @@ -190,8 +172,7 @@ def test_fmt_parent_is_toolchain_installed(self): def test_lint_runs_run_clang_tidy(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.lint()]) - cmds = _cmds(p) + cmds = _cmds([proj.lint()]) assert any("run-clang-tidy" in c for c in cmds) def test_lint_parent_is_built(self): @@ -201,8 +182,7 @@ def test_lint_parent_is_built(self): def test_package_runs_cpack(self): proj = hm.cmake(path="svc") - p = hm.pipeline([proj.package()]) - cmds = _cmds(p) + cmds = _cmds([proj.package()]) assert any("cpack" in c for c in cmds) @@ -214,8 +194,7 @@ def test_package_runs_cpack(self): class TestCMakeVcpkg: def test_deps_vcpkg_produces_bootstrap_command(self): proj = hm.cmake(path="svc", deps="vcpkg") - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) assert any("bootstrap-vcpkg" in c for c in cmds) def test_invalid_deps_raises_valueerror(self): @@ -224,10 +203,9 @@ def test_invalid_deps_raises_valueerror(self): def test_vcpkg_step_has_on_change_cache_policy(self): proj = hm.cmake(path="svc", deps="vcpkg") - p = hm.pipeline([proj.built]) - nodes = p["graph"]["nodes"] - vcpkg_node = next(n for n in nodes if "bootstrap-vcpkg" in n["step"]["cmd"]) - assert vcpkg_node["step"]["cache"]["policy"] == "on_change" + steps = serialize_step_chain([proj.built])["steps"] + vcpkg_step = next(s for s in steps if "bootstrap-vcpkg" in (s.get("cmd") or "")) + assert vcpkg_step["cache"]["policy"] == "on_change" # --------------------------------------------------------------------------- @@ -237,18 +215,15 @@ def test_vcpkg_step_has_on_change_cache_policy(self): class TestCMakeBareForm: def test_bare_build_produces_cmake_build(self): - p = hm.pipeline([hm.cmake.build()]) - cmds = _cmds(p) + cmds = _cmds([hm.cmake.build()]) assert any("cmake --build" in c for c in cmds) def test_bare_test_produces_ctest(self): - p = hm.pipeline([hm.cmake.test()]) - cmds = _cmds(p) + cmds = _cmds([hm.cmake.test()]) assert any("ctest" in c for c in cmds) def test_bare_fmt_produces_clang_format(self): - p = hm.pipeline([hm.cmake.fmt()]) - cmds = _cmds(p) + cmds = _cmds([hm.cmake.fmt()]) assert any("clang-format" in c for c in cmds) @@ -284,6 +259,5 @@ class TestCMakeWithBase: def test_providing_base_skips_apt_install(self): base = hm.scratch().sh("custom base", label="base") proj = hm.cmake(path="svc", base=base) - p = hm.pipeline([proj.built]) - cmds = _cmds(p) + cmds = _cmds([proj.built]) assert not any("apt-get install" in c for c in cmds) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py b/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py deleted file mode 100644 index 2d106491..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py +++ /dev/null @@ -1,171 +0,0 @@ -"""E2E fixture generation + validation. - -Renders 4 complex pipeline scenarios to v0 IR JSON and writes -committed fixtures for Rust deserialization tests. - -Regenerate: UPDATE_E2E_FIXTURES=1 pytest tests/test_e2e_fixtures.py -v -""" - -from __future__ import annotations - -import json -import os -from datetime import timedelta -from pathlib import Path - -import pytest - -import harmont as hm -from harmont._cmake import cmake -from harmont._go import go -from harmont._js import js -from harmont._python import python as python_tc -from harmont._rust import rust -from harmont._zig import zig - -REPO_ROOT = Path(__file__).resolve().parents[4] -FIXTURES_DIR = REPO_ROOT / "tests" / "e2e" / "fixtures" / "python" - - -def _render(ir: dict) -> str: - return json.dumps(ir, indent=2, sort_keys=True, ensure_ascii=False) - - -def _assert_fixture(name: str, ir: dict) -> None: - rendered = _render(ir) - fixture_path = FIXTURES_DIR / f"{name}.json" - - if os.environ.get("UPDATE_E2E_FIXTURES"): - fixture_path.write_text(rendered + "\n") - return - - assert fixture_path.exists(), ( - f"Fixture {fixture_path} missing — run with UPDATE_E2E_FIXTURES=1" - ) - expected = json.loads(fixture_path.read_text()) - actual = json.loads(rendered) - assert actual == expected, f"Fixture drift for {name}. Regenerate with UPDATE_E2E_FIXTURES=1" - - -def _build_monorepo_ci() -> dict: - go_project = go(path="services/api") - py_project = python_tc(path="services/ml") - web_project = js.project(path="web") - - return hm.pipeline( - [ - go_project.build(), - go_project.test(), - go_project.vet(), - py_project.test(), - py_project.lint(), - py_project.typecheck(), - web_project.run("build"), - web_project.run("test"), - web_project.run("lint"), - ], - env={"CI": "true"}, - ) - - -def _build_rust_release() -> dict: - project = rust.toolchain(path=".") - - return hm.pipeline( - [project.build(), project.test(), project.clippy(), project.fmt(), project.doc()], - env={"CI": "true"}, - ) - - -def _build_zig_node_polyglot() -> dict: - base = hm.sh( - "apt-get update && apt-get install -y --no-install-recommends " - "curl ca-certificates xz-utils", - label=":apt: base", - cache=hm.ttl(timedelta(days=1)), - image="ubuntu:24.04", - ) - zig_tc = zig(base=base) - proj_a = zig_tc.project(path="zig-a") - proj_b = zig_tc.project(path="zig-b") - web = js.project(path="web", base=base) - - return hm.pipeline( - [ - proj_a.build(), - proj_a.test(), - proj_b.build(), - proj_b.test(), - web.run("build"), - web.run("test"), - web.run("lint"), - ], - env={"CI": "true"}, - ) - - -def _build_kitchen_sink() -> dict: - c_project = cmake(path="infra/agent") - py_web = python_tc(path="services/web") - - return hm.pipeline( - [ - c_project.build(), - c_project.test(), - c_project.fmt(), - py_web.test(), - py_web.lint(), - ], - env={"CI": "true"}, - ) - - -def _build_cmake_advanced() -> dict: - project = cmake( - path=".", - compiler="clang-18", - defines={ - "CMAKE_BUILD_TYPE": "Release", - "CMAKE_CXX_STANDARD": "20", - }, - ) - return hm.pipeline( - [project.test(), project.lint(), project.fmt()], - env={"CI": "true"}, - ) - - -SCENARIOS = { - "monorepo-ci": _build_monorepo_ci, - "rust-release": _build_rust_release, - "zig-node-polyglot": _build_zig_node_polyglot, - "kitchen-sink": _build_kitchen_sink, - "cmake-advanced": _build_cmake_advanced, -} - - -@pytest.mark.parametrize("name", SCENARIOS.keys()) -def test_e2e_fixture(name: str) -> None: - ir = SCENARIOS[name]() - - assert ir["version"] == "0" - assert len(ir["graph"]["nodes"]) > 0 - # Root steps (no builds_in parent) should carry the ubuntu:24.04 default image. - nodes = ir["graph"]["nodes"] - edges = ir["graph"]["edges"] - child_idxs = {e[1] for e in edges if e[2] == "builds_in"} - roots = [n for i, n in enumerate(nodes) if i not in child_idxs] - assert all(n["step"].get("image") == "ubuntu:24.04" for n in roots) - assert ir["graph"]["edge_property"] == "directed" - - for node in ir["graph"]["nodes"]: - assert "key" in node["step"] - assert "cmd" in node["step"] - assert isinstance(node["env"], dict) - - for src, dst, kind in ir["graph"]["edges"]: - assert kind in ("builds_in", "depends_on") - assert src < len(ir["graph"]["nodes"]) - assert dst < len(ir["graph"]["nodes"]) - - _assert_fixture(name, ir) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py b/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py index ea042abd..0c86887a 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py @@ -5,24 +5,26 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) def test_elixir_object_form_full_chain(): ex = hm.elixir(path="apps/api") - p = hm.pipeline([ex.compile()]) - cmds = _cmds(p) + cmds = _cmds([ex.compile()]) assert any("apt-get install" in c for c in cmds) assert any("erlang" in c.lower() for c in cmds) assert any("elixir" in c.lower() for c in cmds) @@ -31,10 +33,7 @@ def test_elixir_object_form_full_chain(): def test_elixir_actions_share_install_step(): ex = hm.elixir(path=".") - p = hm.pipeline( - [ex.compile(), ex.test(), ex.format(), ex.credo()], - ) - cmds = _cmds(p) + cmds = _cmds([ex.compile(), ex.test(), ex.format(), ex.credo()]) assert len([c for c in cmds if "mix deps.get" in c]) == 1 assert any("mix compile --warnings-as-errors" in c for c in cmds) assert any("mix test" in c for c in cmds) @@ -44,17 +43,15 @@ def test_elixir_actions_share_install_step(): def test_elixir_install_cache_forever(): ex = hm.elixir(path=".") - p = hm.pipeline([ex.compile()]) - erlang = _step_by_substring(p, "erlang") + erlang = _step_by_substring([ex.compile()], "erlang") assert erlang["cache"]["policy"] == "forever" - elixir_step = _step_by_substring(p, "elixir --version") + elixir_step = _step_by_substring([ex.compile()], "elixir --version") assert elixir_step["cache"]["policy"] == "forever" def test_elixir_version_in_install_cmd(): ex = hm.elixir(elixir_version="1.18.3", otp_version="27.3.3") - p = hm.pipeline([ex.compile()]) - elixir_step = _step_by_substring(p, "elixir-otp") + elixir_step = _step_by_substring([ex.compile()], "elixir-otp") assert "1.18.3" in elixir_step["cmd"] assert "27" in elixir_step["cmd"] @@ -70,8 +67,7 @@ def test_elixir_invalid_otp_version_rejected(): def test_elixir_bare_form_actions(): - p = hm.pipeline([hm.elixir.compile(), hm.elixir.test(), hm.elixir.format()]) - cmds = _cmds(p) + cmds = _cmds([hm.elixir.compile(), hm.elixir.test(), hm.elixir.format()]) assert any("mix compile" in c for c in cmds) assert any("mix test" in c for c in cmds) assert any("mix format" in c for c in cmds) @@ -95,10 +91,8 @@ def test_elixir_plt_cached_on_lock(): step = ex.plt() assert "mix dialyzer --plt" in (step.cmd or "") assert step.label == ":ex: plt" - p = hm.pipeline([step]) - plt_ir = next( - n["step"] for n in p["graph"]["nodes"] if "dialyzer --plt" in (n["step"].get("cmd") or "") - ) + steps = serialize_step_chain([step])["steps"] + plt_ir = next(s for s in steps if "dialyzer --plt" in (s.get("cmd") or "")) assert plt_ir["cache"]["policy"] == "on_change" assert "./mix.lock" in plt_ir["cache"]["paths"] @@ -114,8 +108,7 @@ def test_elixir_dialyzer_chains_through_plt(): def test_elixir_with_base_skips_apt(): base = hm.scratch().sh("custom base", label="base") ex = hm.elixir(path=".", base=base) - p = hm.pipeline([ex.compile()]) - cmds = _cmds(p) + cmds = _cmds([ex.compile()]) assert not any("apt-get update && apt-get install -y" in c for c in cmds) assert any("custom base" in c for c in cmds) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_examples_render.py b/crates/hm-dsl-engine/harmont-py/tests/test_examples_render.py index 5442a9ae..64010767 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_examples_render.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_examples_render.py @@ -40,7 +40,7 @@ def _example_dirs() -> list[pathlib.Path]: @pytest.mark.parametrize("example_dir", _example_dirs(), ids=EXAMPLE_IDS) -def test_example_renders_to_v0_ir( +def test_example_renders_to_step_chain( example_dir: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: import harmont as hm @@ -59,17 +59,12 @@ def test_example_renders_to_v0_ir( f"{example_dir.name}: no 'ci' pipeline registered; " f"got slugs {[p['slug'] for p in envelope['pipelines']]}" ) - definition = ci_pipeline["definition"] - assert definition["version"] == "0" - assert definition.get("graph", {}).get("nodes"), ( - f"{example_dir.name}: ci pipeline has no nodes" - ) - nodes = definition.get("graph", {}).get("nodes", []) - edges = definition.get("graph", {}).get("edges", []) - child_idxs = {e[1] for e in edges if e[2] == "builds_in"} - roots = [n for i, n in enumerate(nodes) if i not in child_idxs] - assert roots, f"{example_dir.name}: ci pipeline has no root steps" - assert all("image" in n["step"] for n in roots), ( - f"{example_dir.name}: a root step is missing an image — the lowering " - f"should stamp the ubuntu:24.04 default on every imageless root" + # The envelope now carries the raw step chain; lowering to the v0 IR + # (env layering, key resolution, default-image stamp) happens Rust-side. + step_chain = ci_pipeline["step_chain"] + steps = step_chain["steps"] + assert steps, f"{example_dir.name}: ci pipeline has no steps" + assert step_chain["leaf_indices"], f"{example_dir.name}: ci pipeline has no leaves" + assert any(s.get("cmd") for s in steps), ( + f"{example_dir.name}: ci pipeline has no command steps" ) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_go.py b/crates/hm-dsl-engine/harmont-py/tests/test_go.py index 5b9cdf14..56d89144 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_go.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_go.py @@ -5,24 +5,26 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) def test_go_object_form_full_chain(): go = hm.go(path="svc") - p = hm.pipeline([go.build()]) - cmds = _cmds(p) + cmds = _cmds([go.build()]) assert any("apt-get install" in c for c in cmds) assert any("go.dev/dl/" in c for c in cmds) assert any("cd svc && go build ./..." in c for c in cmds) @@ -30,8 +32,7 @@ def test_go_object_form_full_chain(): def test_go_actions_share_install_step(): go = hm.go(path="svc") - p = hm.pipeline([go.build(), go.test(), go.vet(), go.fmt()]) - cmds = _cmds(p) + cmds = _cmds([go.build(), go.test(), go.vet(), go.fmt()]) assert len([c for c in cmds if "go.dev/dl/" in c]) == 1 assert any("go build ./..." in c for c in cmds) assert any("go test ./..." in c for c in cmds) @@ -41,15 +42,13 @@ def test_go_actions_share_install_step(): def test_go_install_cache_forever(): go = hm.go(path=".") - p = hm.pipeline([go.build()]) - install = _step_by_substring(p, "go.dev/dl/") + install = _step_by_substring([go.build()], "go.dev/dl/") assert install["cache"]["policy"] == "forever" def test_go_version_in_install_cmd(): go = hm.go(path=".", version="1.23.2") - p = hm.pipeline([go.build()]) - install = _step_by_substring(p, "go.dev/dl/") + install = _step_by_substring([go.build()], "go.dev/dl/") assert "go1.23.2" in install["cmd"] @@ -59,8 +58,7 @@ def test_go_invalid_version_rejected(): def test_go_bare_form_actions(): - p = hm.pipeline([hm.go.build(), hm.go.test(), hm.go.vet(), hm.go.fmt()]) - cmds = _cmds(p) + cmds = _cmds([hm.go.build(), hm.go.test(), hm.go.vet(), hm.go.fmt()]) assert any("go build" in c for c in cmds) assert any("go test" in c for c in cmds) assert any("go vet" in c for c in cmds) @@ -78,8 +76,7 @@ def test_go_action_labels_auto_generated(): def test_go_with_base_skips_apt(): base = hm.scratch().sh("custom base", label="base") go = hm.go(path="svc", base=base) - p = hm.pipeline([go.build()]) - cmds = _cmds(p) + cmds = _cmds([go.build()]) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) @@ -87,5 +84,4 @@ def test_go_with_base_skips_apt(): def test_go_installed_escape_hatch_chains(): go = hm.go(path="svc") custom = go.installed.sh("cd svc && go generate ./...", label=":go: gen") - p = hm.pipeline([custom]) - assert any("go generate" in c for c in _cmds(p)) + assert any("go generate" in c for c in _cmds([custom])) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py b/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py index 2e65fbcb..b18c10c2 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py @@ -24,8 +24,8 @@ def _reset(tmp_path, monkeypatch): clear_target_names() -def _graph_nodes(definition): - return definition["graph"]["nodes"] +def _steps(step_chain): + return step_chain["steps"] def test_har_28_example_renders(): @@ -56,16 +56,16 @@ def ci(): out = json.loads(hm.dump_registry_json()) p = out["pipelines"][0] - nodes = _graph_nodes(p["definition"]) + steps = _steps(p["step_chain"]) - cmds = [n["step"].get("cmd") for n in nodes] + cmds = [s.get("cmd") for s in steps] assert any("pytest -v" in (c or "") for c in cmds) assert any("go build" in (c or "") for c in cmds) assert any("npm" in (c or "") for c in cmds) # apt-base used by the venv chain appears exactly once (memoized). - apt_update_nodes = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] - assert len(apt_update_nodes) == 1 + apt_update_steps = [s for s in steps if s.get("cmd") == "apt-get update"] + assert len(apt_update_steps) == 1 def test_har_28_cwd_kwarg_renders_to_cd_prefix(): @@ -74,6 +74,6 @@ def ci(): return hm.sh("pytest -v", cwd="cidsl/py") out = json.loads(hm.dump_registry_json()) - nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"]["cmd"] for n in nodes] + steps = _steps(out["pipelines"][0]["step_chain"]) + cmds = [s.get("cmd") for s in steps] assert "cd cidsl/py && pytest -v" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_js.py b/crates/hm-dsl-engine/harmont-py/tests/test_js.py index dd7f2797..db0c2e3f 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_js.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_js.py @@ -8,6 +8,7 @@ import harmont as hm from harmont._js import JsProject, js, ts +from harmont._serialize import serialize_step_chain # --------------------------------------------------------------------------- # Factory defaults @@ -273,9 +274,9 @@ def test_default_label(runtime: str, expected: str) -> None: ) def test_pipeline_ir(opts: dict) -> None: p = js.project(**opts) - ir = hm.pipeline([p.run("test"), p.run("lint")]) - assert ir["version"] == "0" - assert len(ir["graph"]["nodes"]) >= 4 + chain = serialize_step_chain([p.run("test"), p.run("lint")]) + cmds = [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] + assert len(cmds) >= 4 # --------------------------------------------------------------------------- diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py b/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py deleted file mode 100644 index 08167f99..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py +++ /dev/null @@ -1,246 +0,0 @@ -"""JSON emitter -- v0 IR output shape goldens. - -The wire format uses petgraph-serde graph encoding. Cache keys are -resolved at render time and embedded in cache.key.""" - -from __future__ import annotations - -import json -from datetime import timedelta -from pathlib import Path - -import harmont as hm -from harmont import ( - forever, - on_change, - pipeline, - scratch, - ttl, - wait, -) -from harmont.json_emit import pipeline_to_json - - -def _emit(p, **kw): - kw.setdefault("env", {}) - return json.loads(pipeline_to_json(p, now=0, base_path=Path("/tmp"), **kw)) # noqa: S108 - - -def _nodes(out): - return out["graph"]["nodes"] - - -def _edges(out): - return out["graph"]["edges"] - - -def _step_by_key(out, key): - for n in _nodes(out): - if n["step"]["key"] == key: - return n["step"] - msg = f"no node with key {key!r}" - raise AssertionError(msg) - - -def _node_by_key(out, key): - for n in _nodes(out): - if n["step"]["key"] == key: - return n - msg = f"no node with key {key!r}" - raise AssertionError(msg) - - -def _builds_in_parent_key(out, child_key): - """Return the parent key for a child_key via builds_in edges, or None.""" - key_by_idx = {i: n["step"]["key"] for i, n in enumerate(_nodes(out))} - idx_by_key = {v: k for k, v in key_by_idx.items()} - child_idx = idx_by_key[child_key] - for src, dst, kind in _edges(out): - if kind == "builds_in" and dst == child_idx: - return key_by_idx[src] - return None - - -def test_minimal_command(): - p = pipeline([scratch().sh("echo hi", label="hello")]) - out = _emit(p) - assert out["version"] == "0" - assert len(_nodes(out)) == 1 - step = _nodes(out)[0]["step"] - assert step["key"] == "hello" - assert step["label"] == "hello" - assert step["cmd"] == "echo hi" - # No "type" or "builds_in" field on step dicts. - assert "type" not in step - assert "builds_in" not in step - # No builds_in edges for a root step. - assert _builds_in_parent_key(out, "hello") is None - - -def test_chain_parent_key_in_builds_in_edge(): - a = scratch().sh("install", label="install") - b = a.sh("build", label="build") - out = _emit(pipeline([b])) - assert _builds_in_parent_key(out, "install") is None - assert _builds_in_parent_key(out, "build") == "install" - - -def test_wait_step_becomes_depends_on_edges(): - out = _emit(pipeline([scratch().sh("a", label="a"), wait()])) - # Wait produces no nodes; only the command step "a" is present. - # (No post-wait steps in this case, so no depends_on edges either.) - assert len(_nodes(out)) == 1 - assert _nodes(out)[0]["step"]["key"] == "a" - - -def test_wait_emits_depends_on_edges(): - a = scratch().sh("a", label="a") - b = scratch().sh("b", label="b") - out = _emit(pipeline([a, wait(), b])) - keys = [n["step"]["key"] for n in _nodes(out)] - idx_a = keys.index("a") - idx_b = keys.index("b") - depends_on = [(s, d) for s, d, k in _edges(out) if k == "depends_on"] - assert (idx_a, idx_b) in depends_on - - -def test_pipeline_env_merged_into_node_env(): - out = _emit(pipeline([scratch().sh("a", label="a")], env={"CI": "true"})) - assert _nodes(out)[0]["env"]["CI"] == "true" - assert _nodes(out)[0]["env"]["DEBIAN_FRONTEND"] == "noninteractive" - - -def test_imageless_root_emitted_with_ubuntu_default(): - out = _emit(pipeline([scratch().sh("a", label="a")])) - assert _nodes(out)[0]["step"]["image"] == "ubuntu:24.04" - assert "default_image" not in out - - -def test_cache_ttl_resolves_key(): - p = pipeline( - [scratch().sh("apt-get install -y curl", label="apt", cache=ttl(timedelta(days=1)))] - ) - out = _emit(p) - s = _nodes(out)[0]["step"] - assert s["cache"]["policy"] == "ttl" - assert s["cache"]["duration_seconds"] == 86400 - assert isinstance(s["cache"]["key"], str) - assert len(s["cache"]["key"]) == 64 - - -def test_cache_forever_with_env_keys_emitted(): - out = _emit( - pipeline([scratch().sh("x", label="x", cache=forever(env_keys=("FOO", "BAR")))]), - env={"FOO": "1", "BAR": "2"}, - ) - s = _nodes(out)[0]["step"] - assert s["cache"]["policy"] == "forever" - assert s["cache"]["env_keys"] == ["FOO", "BAR"] - assert "key" in s["cache"] - - -def test_cache_on_change_paths_round_trip(tmp_path): - (tmp_path / "a.txt").write_bytes(b"contents") - (tmp_path / "b.txt").write_bytes(b"other") - out = json.loads( - pipeline_to_json( - pipeline([scratch().sh("make", label="m", cache=on_change("a.txt", "b.txt"))]), - now=0, - base_path=tmp_path, - env={}, - ) - ) - s = _nodes(out)[0]["step"] - assert s["cache"]["policy"] == "on_change" - assert s["cache"]["paths"] == ["a.txt", "b.txt"] - assert "key" in s["cache"] - - -def test_no_optional_fields_when_not_set(): - out = _emit(pipeline([scratch().sh("x", label="x")])) - s = _nodes(out)[0]["step"] - # Root imageless steps now receive ubuntu:24.04 automatically. - assert s.get("image") == "ubuntu:24.04" - assert "timeout_seconds" not in s - assert "cache" not in s - - -def test_timeout_seconds_emitted_when_set(): - out = _emit(pipeline([hm.timeout(300, scratch().sh("x", label="x"))])) - assert _nodes(out)[0]["step"]["timeout_seconds"] == 300 - - -def test_image_emitted_when_set(): - out = _emit(pipeline([scratch().sh("x", label="x", image="alpine:3.19")])) - assert _nodes(out)[0]["step"]["image"] == "alpine:3.19" - - -def test_command_emits_runner_and_runner_args(): - out = _emit( - pipeline( - [ - scratch().sh( - "cargo test", - label="t", - image="rust:1.82", - runner="freestyle", - runner_args={"region": "us"}, - ) - ] - ) - ) - step = _nodes(out)[0]["step"] - assert step["runner"] == "freestyle" - assert step["runner_args"] == {"region": "us"} - - -def test_command_omits_runner_when_unset(): - out = _emit(pipeline([scratch().sh("echo hi", label="hi")])) - step = _nodes(out)[0]["step"] - assert "runner" not in step - assert "runner_args" not in step - - -def test_multi_leaf_pipeline_emits_all_command_steps(): - a = scratch().sh("a", label="a") - b = scratch().sh("b", label="b") - out = _emit(pipeline([a, b])) - keys = sorted(n["step"]["key"] for n in _nodes(out)) - assert keys == ["a", "b"] - - -def test_pipeline_org_and_slug_threaded_through_to_cache_key(): - """Different (org, slug) pairs produce different cache keys for the - same step. Mirrors the namespacing in harmont_macros.scm.""" - p = pipeline([scratch().sh("x", label="x", cache=forever())]) - k1 = json.loads( - pipeline_to_json( - p, - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - pipeline_org="acme", - pipeline_slug="api", - ) - )["graph"]["nodes"][0]["step"]["cache"]["key"] - k2 = json.loads( - pipeline_to_json( - p, - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - pipeline_org="acme", - pipeline_slug="web", - ) - )["graph"]["nodes"][0]["step"]["cache"]["key"] - assert k1 != k2 - - -def test_pipeline_timeout_emitted_as_top_level_seconds(): - out = _emit(pipeline([scratch().sh("x", label="x")], timeout="30m")) - assert out["timeout_seconds"] == 1800 - - -def test_pipeline_timeout_absent_when_unset(): - out = _emit(pipeline([scratch().sh("x", label="x")])) - assert "timeout_seconds" not in out diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py b/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py deleted file mode 100644 index 38a30e92..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Cache-key resolver -- direct ports of the Scheme algorithm in -harmont_macros.scm. Keys must be byte-identical to what harmont-eval -produced pre-removal, so existing cached snapshots remain reachable.""" - -from __future__ import annotations - -import hashlib -import tempfile -from pathlib import Path - -import pytest - -from harmont.keygen import resolve_pipeline_keys - - -def _sha256_hex(s: str) -> str: - return hashlib.sha256(s.encode("utf-8")).hexdigest() - - -NUL = "\x00" - - -def _make_graph(nodes, edges=None): - """Build a minimal graph dict for keygen tests.""" - return { - "nodes": nodes, - "node_holes": [], - "edge_property": "directed", - "edges": edges or [], - } - - -def test_none_policy_emits_no_key(): - graph = _make_graph( - [ - { - "step": {"key": "a", "cmd": "echo", "cache": {"policy": "none"}}, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - assert "key" not in out["nodes"][0]["step"]["cache"] - - -def test_forever_policy_key_matches_scheme_formula(): - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "echo hi", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - inner = _sha256_hex("echo hi" + NUL + "") - policy_res = "forever-" + inner - expected = _sha256_hex( - "default" + NUL + "default" + NUL + "a" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_ttl_policy_key_includes_bucket(): - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "x", - "cache": {"policy": "ttl", "duration_seconds": 3600, "env_keys": []}, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=7200, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - inner = _sha256_hex("x" + NUL + "") - policy_res = "ttl-2-" + inner - expected = _sha256_hex( - "default" + NUL + "default" + NUL + "a" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_on_change_reads_file_contents(): - with tempfile.TemporaryDirectory() as d: - f = Path(d) / "file.txt" - f.write_bytes(b"hello") - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "make", - "cache": {"policy": "on_change", "paths": ["file.txt"]}, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path(d), - env={}, - ) - file_hash = hashlib.sha256(b"hello").hexdigest() - inner = _sha256_hex(file_hash + NUL) - policy_res = "sha-" + inner - expected = _sha256_hex( - "default" + NUL + "default" + NUL + "a" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_on_change_handles_directory_paths(): - """A directory path in ``on_change`` hashes every file inside, - sorted, with its relative path included in the stream. Two builds - of the same tree produce the same key; touching a file under the - directory flips the key.""" - with tempfile.TemporaryDirectory() as d: - root = Path(d) - sub = root / "dir" - sub.mkdir() - (sub / "a.txt").write_bytes(b"alpha") - (sub / "b.txt").write_bytes(b"beta") - - graph = _make_graph( - [ - { - "step": { - "key": "s", - "cmd": "make", - "cache": {"policy": "on_change", "paths": ["dir/"]}, - }, - "env": {}, - }, - ] - ) - out1 = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=root, - env={}, - ) - key1 = out1["nodes"][0]["step"]["cache"]["key"] - - # Same tree -> same key. - graph2 = _make_graph( - [ - { - "step": { - "key": "s", - "cmd": "make", - "cache": {"policy": "on_change", "paths": ["dir/"]}, - }, - "env": {}, - }, - ] - ) - out_again = resolve_pipeline_keys( - graph2, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=root, - env={}, - ) - assert out_again["nodes"][0]["step"]["cache"]["key"] == key1 - - # Modify a file -> key changes. - (sub / "a.txt").write_bytes(b"alpha2") - graph3 = _make_graph( - [ - { - "step": { - "key": "s", - "cmd": "make", - "cache": {"policy": "on_change", "paths": ["dir/"]}, - }, - "env": {}, - }, - ] - ) - out2 = resolve_pipeline_keys( - graph3, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=root, - env={}, - ) - assert out2["nodes"][0]["step"]["cache"]["key"] != key1 - - -def test_on_change_missing_path_skipped(): - with tempfile.TemporaryDirectory() as d: - graph = _make_graph( - [ - { - "step": { - "key": "s", - "cmd": "make", - "cache": {"policy": "on_change", "paths": ["nope/"]}, - }, - "env": {}, - }, - ] - ) - resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path(d), - env={}, - ) - assert graph["nodes"][0]["step"]["cache"]["key"] is not None - - -def test_env_keys_are_sorted_and_picked_up(): - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "echo", - "cache": {"policy": "forever", "env_keys": ["BAR", "FOO"]}, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={"FOO": "1", "BAR": "2"}, - ) - env_str = "BAR=2" + NUL + "FOO=1" + NUL - inner = _sha256_hex("echo" + NUL + env_str) - policy_res = "forever-" + inner - expected = _sha256_hex( - "default" + NUL + "default" + NUL + "a" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_parent_key_chains_through_resolved_cache_keys(): - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "x", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - { - "step": { - "key": "b", - "cmd": "y", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - ], - edges=[[0, 1, "builds_in"]], - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - parent_key = out["nodes"][0]["step"]["cache"]["key"] - inner_b = _sha256_hex("y" + NUL + "") - policy_res = "forever-" + inner_b - expected_b = _sha256_hex( - "default" + NUL + "default" + NUL + "b" + NUL + parent_key + NUL + policy_res - ) - assert out["nodes"][1]["step"]["cache"]["key"] == expected_b - - -def test_compose_concatenates_subpolicies(): - graph = _make_graph( - [ - { - "step": { - "key": "a", - "cmd": "z", - "cache": { - "policy": "compose", - "sub_policies": [ - {"policy": "forever", "env_keys": []}, - {"policy": "none"}, - ], - }, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - forever_inner = _sha256_hex("z" + NUL + "") - sub1 = "forever-" + forever_inner - sub2 = "none" - inner = _sha256_hex(sub1 + sub2) - policy_res = "compose-" + inner - expected = _sha256_hex( - "default" + NUL + "default" + NUL + "a" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_parent_without_cache_is_planerror(): - graph = _make_graph( - [ - { - "step": {"key": "a", "cmd": "x"}, - "env": {}, - }, - { - "step": { - "key": "b", - "cmd": "y", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - ], - edges=[[0, 1, "builds_in"]], - ) - with pytest.raises(ValueError, match="builds_in 'a' which has no cached key"): - resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path("/tmp"), # noqa: S108 - env={}, - ) - - -def test_golden_hash_cross_sdk_reference_pipeline(): - """Golden hash fixture: single-step pipeline. - - Must produce the EXACT same cache key as the TypeScript SDK's - ``golden hash: cross-SDK reference pipeline`` test in keygen.test.ts. - """ - graph = _make_graph( - [ - { - "step": { - "key": "build", - "cmd": "make build", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - ] - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="acme", - pipeline_slug="ci", - now=1000000, - base_path=Path("/nonexistent"), - env={}, - ) - policy_res = "forever-" + _sha256_hex("make build" + NUL) - expected = _sha256_hex( - "acme" + NUL + "ci" + NUL + "build" + NUL + "scratch" + NUL + policy_res - ) - assert out["nodes"][0]["step"]["cache"]["key"] == expected - - -def test_golden_hash_cross_sdk_chained_pipeline(): - """Golden hash fixture: two-step chained pipeline. - - Must produce the EXACT same cache keys as the TypeScript SDK's - ``golden hash: cross-SDK chained pipeline`` test in keygen.test.ts. - """ - graph = _make_graph( - [ - { - "step": { - "key": "setup", - "cmd": "apt-get update && apt-get install -y gcc", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - { - "step": { - "key": "compile", - "cmd": "gcc -o main main.c", - "cache": {"policy": "forever", "env_keys": []}, - }, - "env": {}, - }, - ], - edges=[[0, 1, "builds_in"]], - ) - out = resolve_pipeline_keys( - graph, - pipeline_org="acme", - pipeline_slug="ci", - now=1000000, - base_path=Path("/nonexistent"), - env={}, - ) - - parent_policy_res = "forever-" + _sha256_hex("apt-get update && apt-get install -y gcc" + NUL) - parent_key = _sha256_hex( - "acme" + NUL + "ci" + NUL + "setup" + NUL + "scratch" + NUL + parent_policy_res - ) - - child_policy_res = "forever-" + _sha256_hex("gcc -o main main.c" + NUL) - child_key = _sha256_hex( - "acme" + NUL + "ci" + NUL + "compile" + NUL + parent_key + NUL + child_policy_res - ) - - assert out["nodes"][0]["step"]["cache"]["key"] == parent_key - assert out["nodes"][1]["step"]["cache"]["key"] == child_key diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_keys.py b/crates/hm-dsl-engine/harmont-py/tests/test_keys.py deleted file mode 100644 index 1007d265..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_keys.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Key derivation: slug from label, hash fallback, collision resolution.""" - -from __future__ import annotations - -from harmont._keys import hash_key, resolve_keys, slugify_label -from harmont._step import scratch - - -def test_slugify_strips_emoji_shortcodes(): - assert slugify_label(":rust: api build") == "api-build" - - -def test_slugify_lowercases_and_dashes_non_alnum(): - assert slugify_label("API Build (Test)") == "api-build-test" - - -def test_slugify_collapses_runs_of_dashes(): - assert slugify_label("foo -- bar") == "foo-bar" - - -def test_slugify_trims_leading_trailing_dashes(): - assert slugify_label(":fire: !!! foo !!!") == "foo" - - -def test_slugify_empty_returns_empty_string(): - assert slugify_label(":fire:") == "" - assert slugify_label("") == "" - - -def test_slugify_drops_non_ascii_letters(): - assert slugify_label("Café Build") == "caf-build" - - -def test_slugify_all_non_ascii_returns_empty_string(): - assert slugify_label("构建") == "" - - -def test_resolve_keys_falls_back_to_hash_for_non_ascii_only_label(): - s = scratch().sh("make", label="构建") - keys = resolve_keys([s]) - assert len(keys[id(s)]) == 12 # hash, since slug is empty - - -def test_hash_key_is_deterministic_12_hex_chars(): - h1 = hash_key("parent-key", "make build", 0) - h2 = hash_key("parent-key", "make build", 0) - assert h1 == h2 - assert len(h1) == 12 - assert all(c in "0123456789abcdef" for c in h1) - - -def test_hash_key_changes_with_inputs(): - a = hash_key("p", "make", 0) - b = hash_key("p", "make", 1) - c = hash_key("p", "test", 0) - d = hash_key("q", "make", 0) - assert len({a, b, c, d}) == 4 - - -def test_resolve_keys_uses_explicit_override(): - s = scratch().sh("make", key="my-key") - keys = resolve_keys([s]) - assert keys[id(s)] == "my-key" - - -def test_resolve_keys_uses_label_slug_when_unique(): - s = scratch().sh("make", label=":rust: build") - keys = resolve_keys([s]) - assert keys[id(s)] == "build" - - -def test_resolve_keys_falls_back_to_hash_when_label_collides(): - a = scratch().sh("make a", label=":rust: build") - b = scratch().sh("make b", label=":rust: build") - keys = resolve_keys([a, b]) - # Both colliding labels fall through to hash-derived keys. - assert keys[id(a)] != "build" - assert keys[id(b)] != "build" - assert len(keys[id(a)]) == 12 - assert keys[id(a)] != keys[id(b)] - - -def test_resolve_keys_falls_back_to_hash_when_no_label(): - s = scratch().sh("make") - keys = resolve_keys([s]) - assert len(keys[id(s)]) == 12 - - -def test_resolve_keys_explicit_override_wins_even_under_collision(): - a = scratch().sh("make a", label=":rust: build", key="explicit-a") - b = scratch().sh("make b", label=":rust: build") - keys = resolve_keys([a, b]) - assert keys[id(a)] == "explicit-a" - # `b` had a label that would have been "build", but `a` claimed - # "build" via override, so `b` falls to hash. - assert keys[id(b)] != "build" - assert len(keys[id(b)]) == 12 diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline.py deleted file mode 100644 index d4259b8d..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline.py +++ /dev/null @@ -1,64 +0,0 @@ -"""High-level pipeline-factory tests. Lowering details live in -test_pipeline_lowering.py; this file only covers the public factory.""" - -from __future__ import annotations - -import pytest - -from harmont import pipeline, scratch - - -def test_pipeline_returns_v2_dict(): - p = pipeline([scratch().sh("echo", label="echo")]) - assert p["version"] == "0" - assert isinstance(p["graph"], dict) - assert len(p["graph"]["nodes"]) == 1 - - -def test_pipeline_factory_rejects_no_leaves(): - # `harmont.pipeline` (re-exported) is a polymorphic facade: no-arg - # call routes to the @hm.pipeline decorator path. The factory's - # "at least one leaf" guard is tested via the submodule directly. - from harmont._pipeline import pipeline as _factory - - with pytest.raises(ValueError, match="at least one leaf"): - _factory([]) - - -def test_pipeline_rejects_legacy_single_step_form(): - # Pre-CLI-9 `pipeline(step)` must fail fast with the migration hint, not - # silently route to the @hm.pipeline decorator and blow up downstream. - step = scratch().sh("echo", label="echo") - with pytest.raises(TypeError, match="single list of leaves") as exc: - pipeline(step) - assert "hm.pipeline([step])" in str(exc.value) - - -def test_pipeline_rejects_legacy_variadic_step_form(): - a = scratch().sh("a", label="a") - b = scratch().sh("b", label="b") - with pytest.raises(TypeError, match="single list of leaves") as exc: - pipeline(a, b) - assert "hm.pipeline([a, b])" in str(exc.value) - - -def test_imageless_root_gets_ubuntu_default(): - p = pipeline([scratch().sh("echo hi", label="a")]) - nodes = p["graph"]["nodes"] - assert nodes[0]["step"]["image"] == "ubuntu:24.04" - # No top-level default_image key is emitted anymore. - assert "default_image" not in p - - -def test_explicit_root_image_is_preserved(): - p = pipeline([scratch().sh("echo hi", label="a", image="alpine:3.20")]) - assert p["graph"]["nodes"][0]["step"]["image"] == "alpine:3.20" - - -def test_child_step_stays_imageless(): - root = scratch().sh("echo p", label="p") - child = root.sh("echo c", label="c") - p = pipeline([child]) - nodes = {n["step"]["key"]: n["step"] for n in p["graph"]["nodes"]} - # parent (root) gets the default; child boots from parent snapshot. - assert "image" not in nodes["c"] diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py index 037827b2..0f60e529 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py @@ -20,8 +20,8 @@ def _reset(): clear_target_cache() -def _graph_nodes(definition): - return definition["graph"]["nodes"] +def _steps(step_chain): + return step_chain["steps"] def test_zero_param_pipeline_still_works(): @@ -30,8 +30,8 @@ def ci() -> hm.Step: return hm.sh("echo hi") out = json.loads(hm.dump_registry_json()) - nodes = _graph_nodes(out["pipelines"][0]["definition"]) - assert any(n["step"].get("cmd") == "echo hi" for n in nodes) + steps = _steps(out["pipelines"][0]["step_chain"]) + assert any(s.get("cmd") == "echo hi" for s in steps) def test_pipeline_receives_target_as_param(): @@ -44,8 +44,8 @@ def ci(apt_base: hm.Target[hm.Step]) -> hm.Step: return apt_base.sh("smoke") out = json.loads(hm.dump_registry_json()) - nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"].get("cmd") for n in nodes] + steps = _steps(out["pipelines"][0]["step_chain"]) + cmds = [s.get("cmd") for s in steps] assert "apt-get update" in cmds assert "smoke" in cmds @@ -71,10 +71,10 @@ def ci( return (api, py_test) out = json.loads(hm.dump_registry_json()) - nodes = _graph_nodes(out["pipelines"][0]["definition"]) - apt = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] + steps = _steps(out["pipelines"][0]["step_chain"]) + apt = [s for s in steps if s.get("cmd") == "apt-get update"] assert len(apt) == 1 # apt_base deduped via target memoization - cmds = sorted(n["step"].get("cmd") for n in nodes) + cmds = sorted(s.get("cmd") for s in steps if s.get("cmd") is not None) assert "cabal build" in cmds assert "pytest" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py deleted file mode 100644 index a02a55ad..00000000 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Lowering: walk leaves back to scratch, topo-sort, emit graph-format dicts. - -The lowering pass returns an intermediate Python dict (the petgraph-serde -graph shape the JSON IR will have). This test asserts on that -intermediate graph structure. -""" - -from __future__ import annotations - -import pytest - -import harmont as hm -from harmont._pipeline import _lower_to_graph, pipeline -from harmont._step import scratch, wait - - -def _nodes(graph: dict) -> list[dict]: - return graph["nodes"] - - -def _edges(graph: dict) -> list[list]: - return graph["edges"] - - -def _step_keys(graph: dict) -> list[str]: - return [n["step"]["key"] for n in graph["nodes"]] - - -def _builds_in_edges(graph: dict) -> list[tuple[int, int]]: - return [(src, dst) for src, dst, kind in graph["edges"] if kind == "builds_in"] - - -def _depends_on_edges(graph: dict) -> list[tuple[int, int]]: - return [(src, dst) for src, dst, kind in graph["edges"] if kind == "depends_on"] - - -def _parent_key_map(graph: dict) -> dict[str, str | None]: - """Return {child_key: parent_key} for builds_in edges.""" - key_by_idx = {i: n["step"]["key"] for i, n in enumerate(graph["nodes"])} - result: dict[str, str | None] = {} - # Start with all keys having no parent. - for n in graph["nodes"]: - result[n["step"]["key"]] = None - for src, dst, kind in graph["edges"]: - if kind == "builds_in": - result[key_by_idx[dst]] = key_by_idx[src] - return result - - -def test_single_chain_emits_three_command_nodes_in_parent_order(): - a = scratch().sh("step a", label="a") - b = a.sh("step b", label="b") - c = b.sh("step c", label="c") - graph = _lower_to_graph([c]) - assert _step_keys(graph) == ["a", "b", "c"] - parents = _parent_key_map(graph) - assert parents["a"] is None - assert parents["b"] == "a" - assert parents["c"] == "b" - - -def test_fork_node_is_not_emitted_children_inherit_grandparent(): - base = scratch().sh("install", label="install") - branch = base.fork(label="branch-a") - leaf = branch.sh("test", label="test") - graph = _lower_to_graph([leaf]) - keys = _step_keys(graph) - parents = _parent_key_map(graph) - assert keys == ["install", "test"] - assert parents["install"] is None - assert parents["test"] == "install" - - -def test_two_branches_share_parent_key(): - base = scratch().sh("install", label="install") - a = base.fork(label="a").sh("test-a", label="test-a") - b = base.fork(label="b").sh("test-b", label="test-b") - graph = _lower_to_graph([a, b]) - parents = _parent_key_map(graph) - assert parents["test-a"] == "install" - assert parents["test-b"] == "install" - - -def test_wait_step_emitted_as_depends_on_edges(): - a = scratch().sh("a", label="a") - b = scratch().sh("b", label="b") - c = scratch().sh("c", label="c") - graph = _lower_to_graph([a, b, wait(), c]) - keys = _step_keys(graph) - assert "a" in keys - assert "b" in keys - assert "c" in keys - # c should have depends_on edges from a and b. - depends_on = _depends_on_edges(graph) - idx_a = keys.index("a") - idx_b = keys.index("b") - idx_c = keys.index("c") - assert (idx_a, idx_c) in depends_on - assert (idx_b, idx_c) in depends_on - - -def test_command_includes_label_env_timeout_when_set(): - s = hm.timeout( - 600, - scratch().sh("make", label="build", env={"CI": "true"}), - ) - graph = _lower_to_graph([s]) - node = graph["nodes"][0] - assert node["step"]["label"] == "build" - assert node["env"]["CI"] == "true" - assert node["env"]["DEBIAN_FRONTEND"] == "noninteractive" - assert node["step"]["timeout_seconds"] == 600 - - -def test_command_omits_optional_fields_when_unset(): - s = scratch().sh("make") - graph = _lower_to_graph([s]) - step = graph["nodes"][0]["step"] - # Required fields present. - assert "key" in step - assert "cmd" in step - # No "type" or "builds_in" fields in the new format. - assert "type" not in step - assert "builds_in" not in step - # Optional fields omitted (not None) when unset. - assert "label" not in step - assert "timeout_seconds" not in step - assert "cache" not in step - - -def test_pipeline_factory_collects_reachable_via_parent(): - base = scratch().sh("install", label="install") - leaf_a = base.fork(label="a").sh("test-a", label="test-a") - leaf_b = base.fork(label="b").sh("test-b", label="test-b") - p = pipeline([leaf_a, leaf_b], env={"CI": "true"}) - keys = _step_keys(p["graph"]) - assert set(keys) == {"install", "test-a", "test-b"} - # Pipeline-level env is merged into every node. - for node in p["graph"]["nodes"]: - assert "CI" in node["env"] - assert p["version"] == "0" - - -def test_pipeline_with_no_leaves_raises(): - with pytest.raises(ValueError, match="at least one leaf"): - pipeline([]) - - -def test_dedup_when_step_reachable_from_multiple_leaves(): - base = scratch().sh("install", label="install") - a = base.sh("a", label="a") - b = base.sh("b", label="b") - p = pipeline([a, b]) - keys = _step_keys(p["graph"]) - # `install` appears once even though it's reachable from both leaves. - assert keys.count("install") == 1 diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py b/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py index 919c99fd..d95851bf 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py @@ -5,17 +5,20 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain from harmont.cache import CacheOnChange -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -26,8 +29,7 @@ def _step_by_substring(p: dict, needle: str) -> dict: class TestUvObjectForm: def test_full_chain(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.test()]) - cmds = _cmds(p) + cmds = _cmds([proj.test()]) assert any("apt-get install" in c for c in cmds) assert any("astral.sh/uv/install.sh" in c for c in cmds) assert any("cd svc && uv sync" in c for c in cmds) @@ -35,10 +37,7 @@ def test_full_chain(self): def test_shared_install(self): proj = hm.py.uv(path="svc") - p = hm.pipeline( - [proj.test(), proj.lint(), proj.fmt(), proj.typecheck()], - ) - cmds = _cmds(p) + cmds = _cmds([proj.test(), proj.lint(), proj.fmt(), proj.typecheck()]) assert len([c for c in cmds if "astral.sh/uv/install.sh" in c]) == 1 assert len([c for c in cmds if "apt-get install" in c]) == 1 assert any("uv run pytest" in c for c in cmds) @@ -48,16 +47,14 @@ def test_shared_install(self): def test_sync_cached_on_change(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.test()]) - sync = _step_by_substring(p, "uv sync") + sync = _step_by_substring([proj.test()], "uv sync") assert sync["cache"]["policy"] == "on_change" assert "svc/uv.lock" in sync["cache"]["paths"] assert "svc/pyproject.toml" in sync["cache"]["paths"] def test_install_cache_forever(self): proj = hm.py.uv(path=".") - p = hm.pipeline([proj.test()]) - install = _step_by_substring(p, "astral.sh/uv/install.sh") + install = _step_by_substring([proj.test()], "astral.sh/uv/install.sh") assert install["cache"]["policy"] == "forever" @@ -101,8 +98,7 @@ def test_cache_forwarded(self): def test_run_command(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.run("flask run --port 8080")]) - cmds = _cmds(p) + cmds = _cmds([proj.run("flask run --port 8080")]) assert any("cd svc && uv run flask run --port 8080" in c for c in cmds) def test_run_auto_label_uses_first_word(self): @@ -111,20 +107,17 @@ def test_run_auto_label_uses_first_word(self): def test_build_command(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.build()]) - cmds = _cmds(p) + cmds = _cmds([proj.build()]) assert any("cd svc && uv build" in c for c in cmds) def test_lock_check_command(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.lock_check()]) - cmds = _cmds(p) + cmds = _cmds([proj.lock_check()]) assert any("cd svc && uv lock --check" in c for c in cmds) def test_publish_command(self): proj = hm.py.uv(path="svc") - p = hm.pipeline([proj.publish()]) - cmds = _cmds(p) + cmds = _cmds([proj.publish()]) assert any("cd svc && uv publish" in c for c in cmds) @@ -134,15 +127,13 @@ def test_publish_command(self): class TestUvChainSetup: def test_image_emitted_on_apt_step(self): proj = hm.py.uv(path=".", image="ubuntu:24.04") - p = hm.pipeline([proj.test()]) - apt = _step_by_substring(p, "apt-get install") + apt = _step_by_substring([proj.test()], "apt-get install") assert apt.get("image") == "ubuntu:24.04" def test_base_skips_apt(self): base = hm.scratch().sh("custom base", label="base") proj = hm.py.uv(path="svc", base=base) - p = hm.pipeline([proj.test()]) - cmds = _cmds(p) + cmds = _cmds([proj.test()]) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) assert any("astral.sh/uv/install.sh" in c for c in cmds) @@ -153,8 +144,7 @@ def test_installed_escape_hatch(self): "cd svc && uv run python -m mytool", label=":python: custom", ) - p = hm.pipeline([custom]) - cmds = _cmds(p) + cmds = _cmds([custom]) assert any("mytool" in c for c in cmds) @@ -164,8 +154,7 @@ def test_installed_escape_hatch(self): class TestUvVersionValidation: def test_pinned_version(self): proj = hm.py.uv(path=".", version="0.4.18") - p = hm.pipeline([proj.test()]) - install = _step_by_substring(p, "astral.sh/uv/install.sh") + install = _step_by_substring([proj.test()], "astral.sh/uv/install.sh") assert "UV_VERSION=0.4.18" in install["cmd"] def test_invalid_version_rejected(self): @@ -178,31 +167,25 @@ def test_invalid_version_rejected(self): class TestUvBareForm: def test_bare_test(self): - p = hm.pipeline([hm.py.uv.test()]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.test()]) assert any("cd . && uv run pytest" in c for c in cmds) def test_bare_lint(self): - p = hm.pipeline([hm.py.uv.lint()]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.lint()]) assert any("cd . && uv run ruff check" in c for c in cmds) def test_bare_fmt(self): - p = hm.pipeline([hm.py.uv.fmt()]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.fmt()]) assert any("cd . && uv run ruff format --check" in c for c in cmds) def test_bare_typecheck(self): - p = hm.pipeline([hm.py.uv.typecheck()]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.typecheck()]) assert any("cd . && uv run ty check" in c for c in cmds) def test_bare_run(self): - p = hm.pipeline([hm.py.uv.run("serve")]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.run("serve")]) assert any("cd . && uv run serve" in c for c in cmds) def test_bare_build(self): - p = hm.pipeline([hm.py.uv.build()]) - cmds = _cmds(p) + cmds = _cmds([hm.py.uv.build()]) assert any("cd . && uv build" in c for c in cmds) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_python.py b/crates/hm-dsl-engine/harmont-py/tests/test_python.py index cace93a0..eaee4721 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_python.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_python.py @@ -5,25 +5,27 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain from harmont.cache import CacheOnChange -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) def test_python_object_form_full_chain(): py = hm.python(path="svc") - p = hm.pipeline([py.test()]) - cmds = _cmds(p) + cmds = _cmds([py.test()]) assert any("apt-get install" in c for c in cmds) assert any("astral.sh/uv/install.sh" in c for c in cmds) assert any("cd svc && uv sync" in c for c in cmds) @@ -32,8 +34,7 @@ def test_python_object_form_full_chain(): def test_python_actions_share_install_step(): py = hm.python(path="svc") - p = hm.pipeline([py.test(), py.lint(), py.fmt(), py.typecheck()]) - cmds = _cmds(p) + cmds = _cmds([py.test(), py.lint(), py.fmt(), py.typecheck()]) assert len([c for c in cmds if "astral.sh/uv/install.sh" in c]) == 1 assert len([c for c in cmds if "apt-get install" in c]) == 1 assert any("uv run pytest" in c for c in cmds) @@ -44,8 +45,7 @@ def test_python_actions_share_install_step(): def test_python_sync_cached_on_change_of_lockfile(): py = hm.python(path="svc") - p = hm.pipeline([py.test()]) - sync = _step_by_substring(p, "uv sync") + sync = _step_by_substring([py.test()], "uv sync") assert sync["cache"]["policy"] == "on_change" assert "svc/uv.lock" in sync["cache"]["paths"] assert "svc/pyproject.toml" in sync["cache"]["paths"] @@ -53,20 +53,17 @@ def test_python_sync_cached_on_change_of_lockfile(): def test_python_install_cache_forever(): py = hm.python(path=".") - p = hm.pipeline([py.test()]) - install = _step_by_substring(p, "astral.sh/uv/install.sh") + install = _step_by_substring([py.test()], "astral.sh/uv/install.sh") assert install["cache"]["policy"] == "forever" def test_python_bare_form_test(): - p = hm.pipeline([hm.python.test()]) - cmds = _cmds(p) + cmds = _cmds([hm.python.test()]) assert any("cd . && uv run pytest" in c for c in cmds) def test_python_bare_form_all_actions(): - p = hm.pipeline([hm.python.test(), hm.python.lint(), hm.python.fmt(), hm.python.typecheck()]) - cmds = _cmds(p) + cmds = _cmds([hm.python.test(), hm.python.lint(), hm.python.fmt(), hm.python.typecheck()]) assert any("pytest" in c for c in cmds) assert any("ruff check" in c for c in cmds) assert any("ruff format --check" in c for c in cmds) @@ -112,16 +109,14 @@ def test_python_action_cache_forwarded(): def test_python_image_emitted_on_apt_step(): py = hm.python(path=".", image="ubuntu:24.04") - p = hm.pipeline([py.test()]) - apt = _step_by_substring(p, "apt-get install") + apt = _step_by_substring([py.test()], "apt-get install") assert apt.get("image") == "ubuntu:24.04" def test_python_with_base_skips_apt(): base = hm.scratch().sh("custom base", label="base") py = hm.python(path="svc", base=base) - p = hm.pipeline([py.test()]) - cmds = _cmds(p) + cmds = _cmds([py.test()]) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) assert any("astral.sh/uv/install.sh" in c for c in cmds) @@ -133,15 +128,13 @@ def test_python_installed_escape_hatch_chains(): "cd svc && uv run python -m mytool", label=":python: custom", ) - p = hm.pipeline([custom]) - cmds = _cmds(p) + cmds = _cmds([custom]) assert any("mytool" in c for c in cmds) def test_python_uv_version_in_install_cmd(): py = hm.python(path=".", uv_version="0.4.18") - p = hm.pipeline([py.test()]) - install = _step_by_substring(p, "astral.sh/uv/install.sh") + install = _step_by_substring([py.test()], "astral.sh/uv/install.sh") assert "UV_VERSION=0.4.18" in install["cmd"] diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_rust.py b/crates/hm-dsl-engine/harmont-py/tests/test_rust.py index 819d32e3..72ed537c 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_rust.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_rust.py @@ -2,24 +2,28 @@ from __future__ import annotations -import tempfile -from pathlib import Path +from typing import TYPE_CHECKING import pytest import harmont as hm +from harmont._serialize import serialize_step_chain from harmont.cache import CacheOnChange -from harmont.keygen import resolve_pipeline_keys +if TYPE_CHECKING: + from harmont._step import Step -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list[Step]) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] + +def _step_by_substring(leaves: list[Step], needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -30,18 +34,14 @@ def _step_by_substring(p: dict, needle: str) -> dict: class TestRustToolchain: def test_full_chain(self): tc = hm.rust.toolchain(path="cli") - p = hm.pipeline([tc.build()]) - cmds = _cmds(p) + cmds = _cmds([tc.build()]) assert any("apt-get install" in c for c in cmds) assert any("sh.rustup.rs" in c for c in cmds) assert any("cd cli && cargo build" in c for c in cmds) def test_actions_share_install_step(self): tc = hm.rust.toolchain(path="cli") - p = hm.pipeline( - [tc.build(), tc.test(), tc.clippy(), tc.fmt(), tc.doc()], - ) - cmds = _cmds(p) + cmds = _cmds([tc.build(), tc.test(), tc.clippy(), tc.fmt(), tc.doc()]) assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 assert len([c for c in cmds if "apt-get install" in c]) == 1 @@ -57,27 +57,23 @@ def test_test_release(self): def test_rustup_cache_forever(self): tc = hm.rust.toolchain(path="cli") - p = hm.pipeline([tc.build()]) - rustup = _step_by_substring(p, "sh.rustup.rs") + rustup = _step_by_substring([tc.build()], "sh.rustup.rs") assert rustup["cache"]["policy"] == "forever" def test_default_components(self): tc = hm.rust.toolchain(path=".") - p = hm.pipeline([tc.build()]) - rustup = _step_by_substring(p, "sh.rustup.rs") + rustup = _step_by_substring([tc.build()], "sh.rustup.rs") assert "--component clippy,rustfmt" in rustup["cmd"] def test_components_override(self): tc = hm.rust.toolchain(path=".", components=("clippy",)) - p = hm.pipeline([tc.build()]) - rustup = _step_by_substring(p, "sh.rustup.rs") + rustup = _step_by_substring([tc.build()], "sh.rustup.rs") assert "--component clippy" in rustup["cmd"] assert "rustfmt" not in rustup["cmd"] def test_version_in_rustup_cmd(self): tc = hm.rust.toolchain(path=".", version="1.81.0") - p = hm.pipeline([tc.build()]) - rustup = _step_by_substring(p, "sh.rustup.rs") + rustup = _step_by_substring([tc.build()], "sh.rustup.rs") assert "--default-toolchain 1.81.0" in rustup["cmd"] def test_invalid_version_rejected(self): @@ -90,8 +86,7 @@ def test_installed_escape_hatch(self): "cd cli && cargo build --release --features foo", label=":rust: custom", ) - p = hm.pipeline([custom]) - cmds = _cmds(p) + cmds = _cmds([custom]) assert any("--features foo" in c for c in cmds) def test_action_labels(self): @@ -114,15 +109,13 @@ def test_action_cache_forwarded(self): def test_image_emitted_on_apt_step(self): tc = hm.rust.toolchain(path=".", image="alpine:3.20") - p = hm.pipeline([tc.build()]) - apt = _step_by_substring(p, "apt-get install") + apt = _step_by_substring([tc.build()], "apt-get install") assert apt.get("image") == "alpine:3.20" def test_with_base_skips_apt(self): base = hm.scratch().sh("custom base", label="base") tc = hm.rust.toolchain(path="cli", base=base) - p = hm.pipeline([tc.build()]) - cmds = _cmds(p) + cmds = _cmds([tc.build()]) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) assert any("sh.rustup.rs" in c for c in cmds) @@ -154,8 +147,7 @@ def test_warmup_in_pipeline(self): ". $HOME/.cargo/env && cd cli && cargo test --workspace --locked", label=":rust: test", ) - p = hm.pipeline([t, tc.fmt()]) - cmds = _cmds(p) + cmds = _cmds([t, tc.fmt()]) assert any("cargo build --workspace --tests --locked" in c for c in cmds) assert any("cargo test --workspace --locked" in c for c in cmds) assert any("cargo fmt" in c for c in cmds) @@ -259,8 +251,7 @@ def test_feature_powerset_default(self): def test_feature_powerset_installs_cargo_hack(self): tc = hm.rust.toolchain(path="cli") s = tc.feature_powerset() - p = hm.pipeline([s]) - cmds = _cmds(p) + cmds = _cmds([s]) assert any("cargo install cargo-hack --locked" in c for c in cmds) def test_feature_powerset_each_feature(self): @@ -332,38 +323,6 @@ def test_project_has_all_methods(self): assert proj.clippy().cmd is not None assert proj.fmt().cmd is not None - def test_empty_path_throws_error_is_caught(self): - proj = hm.rust.project(path="") - graph = { - "nodes": [ - { - "step": { - "key": "a", - "cmd": "cargo build", - "cache": { - "policy": "on_change", - "paths": list(proj.warmup.cache.paths), - }, - }, - "env": {}, - }, - ], - "node_holes": [], - "edge_property": "directed", - "edges": [], - } - - with tempfile.TemporaryDirectory() as d: - res = resolve_pipeline_keys( - graph, - pipeline_org="default", - pipeline_slug="default", - now=0, - base_path=Path(d), - env={}, - ) - assert res["nodes"][0]["step"]["cache"]["key"] is not None - def test_warmup_implicit_cache_on_change(self): proj = hm.rust.project(path="cli") assert proj.warmup.cache == CacheOnChange( @@ -429,8 +388,7 @@ def test_toolchain_escape_hatch(self): def test_with_base_skips_apt(self): base = hm.scratch().sh("custom base", label="base") proj = hm.rust.project(path="cli", base=base) - p = hm.pipeline([proj.test(), proj.clippy(), proj.fmt()]) - cmds = _cmds(p) + cmds = _cmds([proj.test(), proj.clippy(), proj.fmt()]) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) @@ -443,8 +401,7 @@ def test_labels(self): def test_pipeline_ir(self): proj = hm.rust.project(path="cli") - p = hm.pipeline([proj.test(), proj.clippy(), proj.fmt()]) - cmds = _cmds(p) + cmds = _cmds([proj.test(), proj.clippy(), proj.fmt()]) assert any("cargo build --workspace --tests --locked" in c for c in cmds) assert any("cargo test --workspace --locked" in c for c in cmds) assert any("cargo clippy" in c for c in cmds) @@ -454,8 +411,7 @@ def test_pipeline_ir(self): def test_version_forwarded(self): proj = hm.rust.project(path=".", version="1.81.0") - p = hm.pipeline([proj.test()]) - rustup = _step_by_substring(p, "sh.rustup.rs") + rustup = _step_by_substring([proj.test()], "sh.rustup.rs") assert "--default-toolchain 1.81.0" in rustup["cmd"] def test_test_packages(self): @@ -514,8 +470,7 @@ def test_doc_exclude(self): def test_ci_in_pipeline(self): proj = hm.rust.project(path="cli") - p = hm.pipeline(list(proj.ci())) - cmds = _cmds(p) + cmds = _cmds(list(proj.ci())) assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 def test_feature_powerset_delegates(self): diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_setup.py b/crates/hm-dsl-engine/harmont-py/tests/test_setup.py index 9d835f77..00ef28ba 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_setup.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_setup.py @@ -4,11 +4,10 @@ from __future__ import annotations -import json - import pytest import harmont as hm +from harmont._serialize import serialize_step_chain # (label, factory) for every toolchain object that owns an `installed` chain. # Each factory returns an object exposing `.installed` and `.setup()`. @@ -24,13 +23,10 @@ ] -def _render_keys_and_edges(leaf: hm.Step) -> tuple[dict, list]: - """Render a one-leaf pipeline and return (nodes-by-index-key, edges).""" - doc = json.loads(hm.pipeline_to_json(hm.pipeline([leaf]))) - graph = doc["graph"] - keys = [n["step"]["key"] for n in graph["nodes"]] - cmds = [n["step"].get("cmd") for n in graph["nodes"]] - return {"keys": keys, "cmds": cmds}, graph["edges"] +def _cmds(leaf: hm.Step) -> list[str | None]: + """Serialize a one-leaf chain and return its per-step commands.""" + chain = serialize_step_chain([leaf]) + return [s.get("cmd") for s in chain["steps"]] @pytest.mark.parametrize( @@ -48,12 +44,12 @@ def test_setup_advances_install_chain(label: str, factory) -> None: assert type(advanced) is type(proj) # The setup command renders, as an ancestor of the install cursor. - info, _edges = _render_keys_and_edges(advanced.installed) - assert any(c and "__SETUP_MARKER__" in c for c in info["cmds"]), info + cmds = _cmds(advanced.installed) + assert any(c and "__SETUP_MARKER__" in c for c in cmds), cmds def test_setup_is_chainable() -> None: proj = hm.elixir(path=".").setup("echo __ONE__").setup("echo __TWO__") - info, _edges = _render_keys_and_edges(proj.installed) - assert any(c and "__ONE__" in c for c in info["cmds"]) - assert any(c and "__TWO__" in c for c in info["cmds"]) + cmds = _cmds(proj.installed) + assert any(c and "__ONE__" in c for c in cmds) + assert any(c and "__TWO__" in c for c in cmds) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py b/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py index ac5fcd90..6a10662b 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py @@ -41,8 +41,8 @@ def ci(py_test: hm.Target[hm.Step]) -> hm.Step: return py_test out = json.loads(hm.dump_registry_json()) - nodes = out["pipelines"][0]["definition"]["graph"]["nodes"] - cmds = sorted(n["step"].get("cmd") for n in nodes) + steps = out["pipelines"][0]["step_chain"]["steps"] + cmds = sorted(s.get("cmd") for s in steps if s.get("cmd") is not None) assert "apt-get update" in cmds assert "cd cidsl/py && pytest -v" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py b/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py index 7e5d6167..f64914d0 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py @@ -3,18 +3,19 @@ from __future__ import annotations import harmont as hm +from harmont._serialize import serialize_step_chain -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] def test_stack_npm_on_spec_step(): """spec -> node install -> npm ci -> codegen. Used by dogfood.""" spec = hm.scratch().sh("make openapi", label=":lock: spec") node = hm.js.project(path="app/codegen", base=spec) - p = hm.pipeline([node.install()]) - cmds = _cmds(p) + cmds = _cmds([node.install()]) assert any("make openapi" in c for c in cmds) assert any("deb.nodesource.com" in c for c in cmds) assert any("npm ci" in c for c in cmds) @@ -37,7 +38,7 @@ def test_deterministic_emission(): def build() -> dict: rust = hm.rust.toolchain(path="cli") - return hm.pipeline([rust.build(), rust.test()]) + return serialize_step_chain([rust.build(), rust.test()]) assert build() == build() @@ -47,17 +48,17 @@ def test_mixed_pipeline_compiles(): rust = hm.rust.toolchain(path="cli") node = hm.js.project(path="app/codegen") go = hm.go(path="services/api") - p = hm.pipeline( + chain = serialize_step_chain( [rust.test(), rust.clippy(), node.install(), go.build(), go.test()], ) - assert p["version"] == "0" - assert len(p["graph"]["nodes"]) > 0 + assert len(chain["steps"]) > 0 -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -77,10 +78,7 @@ def test_apt_base_shared_across_toolchains(): ) rust = hm.rust.toolchain(path=".", base=base) py = hm.py.uv(path="dsls/harmont-py", base=base) - p = hm.pipeline( - [rust.build(), py.test()], - ) - cmds = _cmds(p) + cmds = _cmds([rust.build(), py.test()]) assert len([c for c in cmds if "apt-get install" in c]) == 1 assert any("sh.rustup.rs" in c for c in cmds) assert any("uv" in c for c in cmds) @@ -94,8 +92,7 @@ def test_apt_base_default_label(): def test_apt_base_custom_image(): base = hm.apt_base(packages=("curl",), image="debian:bookworm") rust = hm.rust.toolchain(path=".", base=base) - p = hm.pipeline([rust.build()]) - apt_step = _step_by_substring(p, "apt-get install") + apt_step = _step_by_substring([rust.build()], "apt-get install") assert apt_step.get("image") == "debian:bookworm" diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_zig.py b/crates/hm-dsl-engine/harmont-py/tests/test_zig.py index 015c347f..c4870840 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_zig.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_zig.py @@ -5,31 +5,32 @@ import pytest import harmont as hm +from harmont._serialize import serialize_step_chain -def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] +def _cmds(leaves: list) -> list[str]: + chain = serialize_step_chain(list(leaves)) + return [s["cmd"] for s in chain["steps"] if s.get("cmd") is not None] -def _step_by_substring(p: dict, needle: str) -> dict: - for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): - return n["step"] +def _step_by_substring(leaves: list, needle: str) -> dict: + chain = serialize_step_chain(list(leaves)) + for s in chain["steps"]: + if needle in (s.get("cmd") or ""): + return s raise AssertionError(needle) def test_zig_object_form_full_chain(): z = hm.zig(path="svc") - p = hm.pipeline([z.build()]) - cmds = _cmds(p) + cmds = _cmds([z.build()]) assert any("ziglang.org" in c for c in cmds) assert any("cd svc && zig build" in c for c in cmds) def test_zig_actions_share_install(): z = hm.zig(path="svc") - p = hm.pipeline([z.build(), z.test(), z.fmt()]) - cmds = _cmds(p) + cmds = _cmds([z.build(), z.test(), z.fmt()]) assert len([c for c in cmds if "ziglang.org" in c]) == 1 assert any("zig build test" in c for c in cmds) assert any("zig fmt --check ." in c for c in cmds) @@ -37,8 +38,7 @@ def test_zig_actions_share_install(): def test_zig_version_in_install_cmd(): z = hm.zig(path=".", version="0.14.1") - p = hm.pipeline([z.build()]) - install = _step_by_substring(p, "ziglang.org") + install = _step_by_substring([z.build()], "ziglang.org") assert "0.14.1" in install["cmd"] @@ -55,8 +55,7 @@ def test_zig_action_labels_auto_generated(): def test_zig_bare_form_actions(): - p = hm.pipeline([hm.zig.build(), hm.zig.test(), hm.zig.fmt()]) - cmds = _cmds(p) + cmds = _cmds([hm.zig.build(), hm.zig.test(), hm.zig.fmt()]) assert any("zig build" in c for c in cmds) assert any("zig fmt --check ." in c for c in cmds) @@ -64,21 +63,18 @@ def test_zig_bare_form_actions(): def test_zig_old_version_uses_old_url_format(): """Versions < 0.14.1 use zig-linux-x86_64-{v} format.""" z = hm.zig(path=".", version="0.13.0") - p = hm.pipeline([z.build()]) - install = _step_by_substring(p, "ziglang.org") + install = _step_by_substring([z.build()], "ziglang.org") assert "zig-linux-x86_64-0.13.0" in install["cmd"] def test_zig_new_version_uses_new_url_format(): """Versions >= 0.14.1 use zig-x86_64-linux-{v} format.""" z = hm.zig(path=".", version="0.14.1") - p = hm.pipeline([z.build()]) - install = _step_by_substring(p, "ziglang.org") + install = _step_by_substring([z.build()], "ziglang.org") assert "zig-x86_64-linux-0.14.1" in install["cmd"] def test_zig_with_base_skips_apt(): base = hm.scratch().sh("custom base", label="base") z = hm.zig(path="svc", base=base) - p = hm.pipeline([z.build()]) - assert not any("apt-get install" in c for c in _cmds(p)) + assert not any("apt-get install" in c for c in _cmds([z.build()])) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_zig_toolchain.py b/crates/hm-dsl-engine/harmont-py/tests/test_zig_toolchain.py index 7ea94f36..60959773 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_zig_toolchain.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_zig_toolchain.py @@ -66,30 +66,31 @@ def ci( return (lib_a.build(), lib_b.build()) envelope = json.loads(hm.dump_registry_json()) - definition = envelope["pipelines"][0]["definition"] - nodes = definition["graph"]["nodes"] - edges = definition["graph"]["edges"] + step_chain = envelope["pipelines"][0]["step_chain"] + steps = step_chain["steps"] - zig_installs = [n for n in nodes if n["step"].get("label") == ":zig: install"] + zig_installs = [ + i for i, s in enumerate(steps) if s.get("label") == ":zig: install" + ] assert len(zig_installs) == 1, ( - f"expected exactly one :zig: install node, got {[n['step']['key'] for n in zig_installs]}" + f"expected exactly one :zig: install step, got {len(zig_installs)}" ) - - install_key = zig_installs[0]["step"]["key"] - lib_a_build = next(n for n in nodes if "lib-a" in (n["step"].get("label") or "")) - lib_b_build = next(n for n in nodes if "lib-b" in (n["step"].get("label") or "")) - - # Verify builds_in edges connect install to both builds. - key_by_idx = {i: n["step"]["key"] for i, n in enumerate(nodes)} - idx_by_key = {v: k for k, v in key_by_idx.items()} - - install_idx = idx_by_key[install_key] - lib_a_idx = idx_by_key[lib_a_build["step"]["key"]] - lib_b_idx = idx_by_key[lib_b_build["step"]["key"]] - - builds_in_edges = [(s, d) for s, d, k in edges if k == "builds_in"] - assert (install_idx, lib_a_idx) in builds_in_edges - assert (install_idx, lib_b_idx) in builds_in_edges + install_idx = zig_installs[0] + + lib_a_idx = next(i for i, s in enumerate(steps) if "lib-a" in (s.get("label") or "")) + lib_b_idx = next(i for i, s in enumerate(steps) if "lib-b" in (s.get("label") or "")) + + # Both builds descend from the single shared install via parent_idx. + def _ancestors(idx: int) -> list[int]: + chain: list[int] = [] + cur: int | None = idx + while cur is not None: + chain.append(cur) + cur = steps[cur].get("parent_idx") + return chain + + assert install_idx in _ancestors(lib_a_idx) + assert install_idx in _ancestors(lib_b_idx) reg.clear_registry() targets.clear_target_cache() diff --git a/crates/hm-dsl-engine/src/bundled_sources.rs b/crates/hm-dsl-engine/src/bundled_sources.rs index ece9285f..8c10fdb4 100644 --- a/crates/hm-dsl-engine/src/bundled_sources.rs +++ b/crates/hm-dsl-engine/src/bundled_sources.rs @@ -24,10 +24,10 @@ mod tests { } #[test] - fn harmont_py_contains_pipeline() { - // The pipeline module is private (underscore-prefixed); the public - // surface is re-exported from `__init__.py`. - assert!(HARMONT_PY.get_file("_pipeline.py").is_some()); + fn harmont_py_contains_serialize() { + // The step-chain serializer is private (underscore-prefixed); the + // public surface is re-exported from `__init__.py`. + assert!(HARMONT_PY.get_file("_serialize.py").is_some()); } #[test] @@ -41,6 +41,6 @@ mod tests { let target = tmp.path().join("harmont"); extract_to(&HARMONT_PY, &target).expect("extract"); assert!(target.join("__init__.py").exists()); - assert!(target.join("_pipeline.py").exists()); + assert!(target.join("_serialize.py").exists()); } } diff --git a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs b/crates/hm-pipeline-ir/tests/e2e_fixtures.rs deleted file mode 100644 index 2ea71319..00000000 --- a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs +++ /dev/null @@ -1,151 +0,0 @@ -#![allow( - clippy::cargo_common_metadata, - clippy::multiple_crate_versions, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic -)] - -use std::collections::BTreeSet; -use std::fs; -use std::path::PathBuf; - -use daggy::petgraph::visit::{EdgeRef, IntoNodeReferences}; -use hm_pipeline_ir::{EdgeKind, PipelineGraph}; - -const SCENARIOS: &[&str] = &[ - "monorepo-ci", - "rust-release", - "zig-node-polyglot", - "kitchen-sink", -]; - -fn fixtures_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/e2e/fixtures") -} - -fn load_fixture(scenario: &str) -> PipelineGraph { - let path = fixtures_dir() - .join("python") - .join(format!("{scenario}.json")); - let bytes = fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); - serde_json::from_slice(&bytes).unwrap_or_else(|e| panic!("parse py/{scenario}: {e}")) -} - -fn step_labels(g: &PipelineGraph) -> BTreeSet { - g.dag() - .graph() - .node_references() - .filter_map(|(_, t)| t.step.label.clone()) - .collect() -} - -fn edge_kinds(g: &PipelineGraph) -> (usize, usize) { - let mut builds_in = 0usize; - let mut depends_on = 0usize; - for e in g.dag().graph().edge_references() { - match e.weight() { - EdgeKind::BuildsIn => builds_in += 1, - EdgeKind::DependsOn => depends_on += 1, - } - } - (builds_in, depends_on) -} - -#[test] -fn python_monorepo_ci() { - let g = load_fixture("monorepo-ci"); - assert_eq!(g.default_image(), None); - assert!(g.node_count() >= 15, "nodes: {}", g.node_count()); - let labels = step_labels(&g); - assert!(labels.iter().any(|l| l.contains("go"))); - assert!( - labels - .iter() - .any(|l| l.contains("python") || l.contains("uv")) - ); - assert!( - labels - .iter() - .any(|l| l.contains("node") || l.contains("npm")) - ); -} - -#[test] -fn python_rust_release() { - let g = load_fixture("rust-release"); - assert_eq!(g.default_image(), None); - assert!(g.node_count() >= 5, "nodes: {}", g.node_count()); - let labels = step_labels(&g); - assert!(labels.iter().any(|l| l.contains("rust"))); - // The DSL now injects image on each imageless root step directly. - let apt_base = g - .dag() - .graph() - .node_references() - .find(|(_, t)| t.step.key == "apt-base") - .map(|(_, t)| t.step.image.as_deref()); - assert_eq!( - apt_base, - Some(Some("ubuntu:24.04")), - "root step apt-base must carry explicit image" - ); -} - -#[test] -fn python_zig_node_polyglot() { - let g = load_fixture("zig-node-polyglot"); - assert_eq!(g.default_image(), None); - assert!(g.node_count() >= 10, "nodes: {}", g.node_count()); - let labels = step_labels(&g); - assert!(labels.iter().any(|l| l.contains("zig"))); - assert!( - labels - .iter() - .any(|l| l.contains("node") || l.contains("npm")) - ); -} - -#[test] -fn python_kitchen_sink() { - let g = load_fixture("kitchen-sink"); - assert_eq!(g.default_image(), None); - assert!(g.node_count() >= 10, "nodes: {}", g.node_count()); - let labels = step_labels(&g); - assert!(labels.iter().any(|l| l.contains("python"))); - assert!( - labels - .iter() - .any(|l| l.contains("cmake") || l.contains(":c:")) - ); - for (_, t) in g.dag().graph().node_references() { - assert!( - t.env.contains_key("CI"), - "node {} missing CI env", - t.step.key - ); - } -} - -#[test] -fn all_fixtures_have_valid_structure() { - for scenario in SCENARIOS { - let g = load_fixture(scenario); - - for (_, t) in g.dag().graph().node_references() { - assert!(!t.step.key.is_empty(), "py/{scenario}: empty key"); - assert!( - !t.step.cmd.is_empty(), - "py/{scenario}: empty cmd for {}", - t.step.key, - ); - } - - let (bi, dep) = edge_kinds(&g); - assert!(bi + dep > 0, "py/{scenario}: no edges"); - - for e in g.dag().graph().edge_references() { - assert_ne!(e.source(), e.target(), "py/{scenario}: self-loop"); - } - } -} diff --git a/tests/e2e/fixtures/python/cmake-advanced.json b/tests/e2e/fixtures/python/cmake-advanced.json deleted file mode 100644 index b1ccf9b3..00000000 --- a/tests/e2e/fixtures/python/cmake-advanced.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "graph": { - "edge_property": "directed", - "edges": [ - [ - 0, - 1, - "builds_in" - ], - [ - 1, - 2, - "builds_in" - ], - [ - 2, - 3, - "builds_in" - ], - [ - 2, - 4, - "builds_in" - ], - [ - 1, - 5, - "builds_in" - ] - ], - "node_holes": [], - "nodes": [ - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy clang-18 lld-18", - "image": "ubuntu:24.04", - "key": "apt-base", - "label": ":cmake: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "cmake --version && ninja --version && ccache --version && clang-18 --version", - "key": "verify", - "label": ":cmake: verify" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "CMakeLists.txt" - ], - "policy": "on_change" - }, - "cmd": "cd . && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 && cmake --build build --parallel $(nproc)", - "key": "build", - "label": ":cmake: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cmake --build ./build --parallel $(nproc) && ctest --test-dir ./build --output-on-failure --parallel $(nproc)", - "key": "test", - "label": ":cmake: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd . && run-clang-tidy -p build", - "key": "lint", - "label": ":cmake: lint" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd . && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", - "key": "fmt", - "label": ":cmake: fmt" - } - } - ] - }, - "version": "0" -} diff --git a/tests/e2e/fixtures/python/kitchen-sink.json b/tests/e2e/fixtures/python/kitchen-sink.json deleted file mode 100644 index e1fcb4ea..00000000 --- a/tests/e2e/fixtures/python/kitchen-sink.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "graph": { - "edge_property": "directed", - "edges": [ - [ - 0, - 1, - "builds_in" - ], - [ - 1, - 2, - "builds_in" - ], - [ - 2, - 3, - "builds_in" - ], - [ - 1, - 4, - "builds_in" - ], - [ - 5, - 6, - "builds_in" - ], - [ - 6, - 7, - "builds_in" - ], - [ - 7, - 8, - "builds_in" - ], - [ - 7, - 9, - "builds_in" - ] - ], - "node_holes": [], - "nodes": [ - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy", - "image": "ubuntu:24.04", - "key": "fdcc2fd62363", - "label": ":cmake: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "cmake --version && ninja --version && ccache --version", - "key": "verify", - "label": ":cmake: verify" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "infra/agent/CMakeLists.txt" - ], - "policy": "on_change" - }, - "cmd": "cd infra/agent && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && cmake --build build --parallel $(nproc)", - "key": "build", - "label": ":cmake: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cmake --build infra/agent/build --parallel $(nproc) && ctest --test-dir infra/agent/build --output-on-failure --parallel $(nproc)", - "key": "431f35b84318", - "label": ":cmake: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd infra/agent && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", - "key": "fmt", - "label": ":cmake: fmt" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", - "image": "ubuntu:24.04", - "key": "c8d9fda86ff3", - "label": ":python: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", - "key": "uv-install", - "label": ":python: uv-install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "services/web/uv.lock", - "services/web/pyproject.toml" - ], - "policy": "on_change" - }, - "cmd": "cd services/web && uv sync --all-extras", - "key": "uv-sync", - "label": ":python: uv-sync" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/web && uv run pytest", - "key": "239c4926568b", - "label": ":python: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/web && uv run ruff check .", - "key": "lint", - "label": ":python: lint" - } - } - ] - }, - "version": "0" -} diff --git a/tests/e2e/fixtures/python/monorepo-ci.json b/tests/e2e/fixtures/python/monorepo-ci.json deleted file mode 100644 index ec0ef385..00000000 --- a/tests/e2e/fixtures/python/monorepo-ci.json +++ /dev/null @@ -1,328 +0,0 @@ -{ - "graph": { - "edge_property": "directed", - "edges": [ - [ - 0, - 1, - "builds_in" - ], - [ - 1, - 2, - "builds_in" - ], - [ - 1, - 3, - "builds_in" - ], - [ - 1, - 4, - "builds_in" - ], - [ - 5, - 6, - "builds_in" - ], - [ - 6, - 7, - "builds_in" - ], - [ - 7, - 8, - "builds_in" - ], - [ - 7, - 9, - "builds_in" - ], - [ - 7, - 10, - "builds_in" - ], - [ - 11, - 12, - "builds_in" - ], - [ - 12, - 13, - "builds_in" - ], - [ - 13, - 14, - "builds_in" - ], - [ - 13, - 15, - "builds_in" - ], - [ - 13, - 16, - "builds_in" - ] - ], - "node_holes": [], - "nodes": [ - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates git", - "image": "ubuntu:24.04", - "key": "334b29e96b76", - "label": ":go: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -fsSL https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /tmp/go.tgz && rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt && go version", - "key": "e0b494124562", - "label": ":go: install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/api && go build ./...", - "key": "6f9493b7219f", - "label": ":go: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/api && go test ./...", - "key": "1ad6d86b2c0a", - "label": ":go: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/api && go vet ./...", - "key": "vet", - "label": ":go: vet" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", - "image": "ubuntu:24.04", - "key": "c8d9fda86ff3", - "label": ":python: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", - "key": "uv-install", - "label": ":python: uv-install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "services/ml/uv.lock", - "services/ml/pyproject.toml" - ], - "policy": "on_change" - }, - "cmd": "cd services/ml && uv sync --all-extras", - "key": "uv-sync", - "label": ":python: uv-sync" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/ml && uv run pytest", - "key": "847020e744bc", - "label": ":python: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/ml && uv run ruff check .", - "key": "6c48498afb84", - "label": ":python: lint" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd services/ml && uv run ty check .", - "key": "typecheck", - "label": ":python: typecheck" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates", - "image": "ubuntu:24.04", - "key": "3c2cfedcad46", - "label": ":node: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", - "key": "ed974519b390", - "label": ":node: install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "web/package-lock.json" - ], - "policy": "on_change" - }, - "cmd": "cd web && npm ci", - "key": "deps", - "label": ":node: deps" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run build", - "key": "a94a0f84e711", - "label": ":node: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run test", - "key": "d2438adde70d", - "label": ":node: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run lint", - "key": "74c52c9e5ef6", - "label": ":node: lint" - } - } - ] - }, - "version": "0" -} diff --git a/tests/e2e/fixtures/python/rust-release.json b/tests/e2e/fixtures/python/rust-release.json deleted file mode 100644 index 4f9cc69d..00000000 --- a/tests/e2e/fixtures/python/rust-release.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "graph": { - "edge_property": "directed", - "edges": [ - [ - 0, - 1, - "builds_in" - ], - [ - 1, - 2, - "builds_in" - ], - [ - 1, - 3, - "builds_in" - ], - [ - 1, - 4, - "builds_in" - ], - [ - 1, - 5, - "builds_in" - ], - [ - 1, - 6, - "builds_in" - ] - ], - "node_holes": [], - "nodes": [ - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev", - "image": "ubuntu:24.04", - "key": "apt-base", - "label": ":rust: apt-base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --component clippy,rustfmt && . $HOME/.cargo/env && rustc --version && cargo --version", - "key": "rustup", - "label": ":rust: rustup" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo build --locked", - "key": "build", - "label": ":rust: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo test --locked", - "key": "test", - "label": ":rust: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo clippy --all-targets --locked -- -D warnings", - "key": "clippy", - "label": ":rust: clippy" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo fmt --all --check", - "key": "fmt", - "label": ":rust: fmt" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "RUSTDOCFLAGS": "-D warnings", - "TERM": "dumb" - }, - "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo doc --no-deps --locked", - "key": "doc", - "label": ":rust: doc" - } - } - ] - }, - "version": "0" -} diff --git a/tests/e2e/fixtures/python/zig-node-polyglot.json b/tests/e2e/fixtures/python/zig-node-polyglot.json deleted file mode 100644 index 0ddbebff..00000000 --- a/tests/e2e/fixtures/python/zig-node-polyglot.json +++ /dev/null @@ -1,213 +0,0 @@ -{ - "graph": { - "edge_property": "directed", - "edges": [ - [ - 0, - 1, - "builds_in" - ], - [ - 1, - 2, - "builds_in" - ], - [ - 1, - 3, - "builds_in" - ], - [ - 1, - 4, - "builds_in" - ], - [ - 1, - 5, - "builds_in" - ], - [ - 0, - 6, - "builds_in" - ], - [ - 6, - 7, - "builds_in" - ], - [ - 7, - 8, - "builds_in" - ], - [ - 7, - 9, - "builds_in" - ], - [ - 7, - 10, - "builds_in" - ] - ], - "node_holes": [], - "nodes": [ - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "duration_seconds": 86400, - "env_keys": [], - "policy": "ttl" - }, - "cmd": "apt-get update && apt-get install -y --no-install-recommends curl ca-certificates xz-utils", - "image": "ubuntu:24.04", - "key": "base", - "label": ":apt: base" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -fsSL https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz -o /tmp/zig.tar.xz && rm -rf /usr/local/zig && mkdir -p /usr/local/zig && tar -xJf /tmp/zig.tar.xz -C /usr/local/zig --strip-components=1 && ln -sf /usr/local/zig/zig /usr/local/bin/zig && zig version", - "key": "3083a531d11a", - "label": ":zig: install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd zig-a && zig build", - "key": "zig-a-build", - "label": ":zig: zig-a build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd zig-a && zig build test", - "key": "zig-a-test", - "label": ":zig: zig-a test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd zig-b && zig build", - "key": "zig-b-build", - "label": ":zig: zig-b build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd zig-b && zig build test", - "key": "zig-b-test", - "label": ":zig: zig-b test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "env_keys": [], - "policy": "forever" - }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", - "key": "a8f0d9d99460", - "label": ":node: install" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cache": { - "paths": [ - "web/package-lock.json" - ], - "policy": "on_change" - }, - "cmd": "cd web && npm ci", - "key": "deps", - "label": ":node: deps" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run build", - "key": "build", - "label": ":node: build" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run test", - "key": "test", - "label": ":node: test" - } - }, - { - "env": { - "CI": "true", - "DEBIAN_FRONTEND": "noninteractive", - "TERM": "dumb" - }, - "step": { - "cmd": "cd web && npm run lint", - "key": "lint", - "label": ":node: lint" - } - } - ] - }, - "version": "0" -} From 86a4b1e91ba4b612fbd7f8a84255fa8475b9157f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 13 Jul 2026 15:11:47 -0700 Subject: [PATCH 9/9] docs: update docstrings for decorator-only pipeline API Docstring examples updated from removed hm.pipeline([...]) factory form to current decorator-based API. References to deleted modules removed. Note: docs-site MDX regeneration needed (make docs-generate from simci root). --- crates/hm-dsl-engine/harmont-py/harmont/_cmake.py | 3 ++- crates/hm-dsl-engine/harmont-py/harmont/_elixir.py | 6 ++++-- crates/hm-dsl-engine/harmont-py/harmont/_go.py | 3 ++- crates/hm-dsl-engine/harmont-py/harmont/_js.py | 3 ++- crates/hm-dsl-engine/harmont-py/harmont/_python.py | 3 ++- crates/hm-dsl-engine/harmont-py/harmont/_rust.py | 3 ++- crates/hm-dsl-engine/harmont-py/harmont/_step.py | 2 +- crates/hm-dsl-engine/harmont-py/harmont/_zig.py | 8 +++++--- crates/hm-dsl-engine/harmont-py/harmont/py/uv.py | 4 ++-- 9 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_cmake.py b/crates/hm-dsl-engine/harmont-py/harmont/_cmake.py index 04c092fa..3fdd19b6 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_cmake.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_cmake.py @@ -453,7 +453,8 @@ def __call__( Examples: >>> import harmont as hm >>> proj = hm.cmake(path=".", compiler="gcc-14") - >>> hm.pipeline([proj.build(), proj.test()]) + >>> proj.build() + >>> proj.test() """ tc = _make_toolchain( compiler=compiler, diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_elixir.py b/crates/hm-dsl-engine/harmont-py/harmont/_elixir.py index cadd7b83..ffa3647c 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_elixir.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_elixir.py @@ -92,7 +92,8 @@ def setup( Examples: >>> import harmont as hm >>> proj = hm.elixir(path="elixir").setup("mix proto.gen") - >>> hm.pipeline([proj.compile(), proj.test()]) + >>> proj.compile() + >>> proj.test() """ return advance_install(self, cmd, cwd=cwd, label=label, cache=cache, env=env) @@ -266,7 +267,8 @@ def __call__( Examples: >>> import harmont as hm >>> proj = hm.elixir(elixir_version="1.18.3") - >>> hm.pipeline([proj.compile(), proj.test()]) + >>> proj.compile() + >>> proj.test() """ return _make_elixir( path=path, diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_go.py b/crates/hm-dsl-engine/harmont-py/harmont/_go.py index 1fa1aaa3..e2dd87ba 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_go.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_go.py @@ -143,7 +143,8 @@ def __call__( Examples: >>> import harmont as hm >>> tc = hm.go(version="1.23.2") - >>> hm.pipeline([tc.test(), tc.vet()]) + >>> tc.test() + >>> tc.vet() """ return _make_go(path=path, version=version, image=image, base=base) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_js.py b/crates/hm-dsl-engine/harmont-py/harmont/_js.py index 1b1e4aaa..c489be0a 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_js.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_js.py @@ -302,7 +302,8 @@ def project( Examples: >>> import harmont as hm >>> proj = hm.js.project(path="web", runtime="bun") - >>> hm.pipeline([proj.run("test"), proj.run("lint")]) + >>> proj.run("test") + >>> proj.run("lint") """ return _make_js(path=path, pm=pm, runtime=runtime, version=version, image=image, base=base) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_python.py b/crates/hm-dsl-engine/harmont-py/harmont/_python.py index 0177338f..9aba0e31 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_python.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_python.py @@ -188,7 +188,8 @@ def __call__( Examples: >>> import harmont as hm >>> tc = hm.python(path="services/api") - >>> hm.pipeline([tc.test(), tc.lint()]) + >>> tc.test() + >>> tc.lint() """ return _make_python(path=path, uv_version=uv_version, image=image, base=base) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_rust.py b/crates/hm-dsl-engine/harmont-py/harmont/_rust.py index f69dc159..06d66a0e 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_rust.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_rust.py @@ -798,7 +798,8 @@ def toolchain( Examples: >>> import harmont as hm >>> tc = hm.rust.toolchain(version="1.81.0") - >>> hm.pipeline([tc.test(), tc.clippy()]) + >>> tc.test() + >>> tc.clippy() """ return _make_rust( path=path, diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_step.py b/crates/hm-dsl-engine/harmont-py/harmont/_step.py index 2fb8e090..509d0d66 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_step.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_step.py @@ -170,6 +170,6 @@ def wait(*, continue_on_failure: bool = False) -> Step: Examples: >>> import harmont as hm - >>> p = hm.pipeline([hm.sh("make build"), hm.wait(), hm.sh("make deploy")]) + >>> barrier = hm.wait() """ return Step(is_wait=True, continue_on_failure=continue_on_failure) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_zig.py b/crates/hm-dsl-engine/harmont-py/harmont/_zig.py index de455948..bcf5998e 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_zig.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_zig.py @@ -146,7 +146,7 @@ def setup( Examples: >>> import harmont as hm >>> tc = hm.zig().setup("zig build gen") - >>> hm.pipeline([tc.project("lib-a").test()]) + >>> tc.project("lib-a").test() """ return advance_install(self, cmd, cwd=cwd, label=label, cache=cache, env=env) @@ -165,7 +165,8 @@ def project(self, path: str = ".") -> ZigProject: >>> tc = hm.zig(version="0.14.1") >>> lib = tc.project("lib-a") >>> app = tc.project("app") - >>> hm.pipeline([lib.test(), app.test()]) + >>> lib.test() + >>> app.test() """ return ZigProject(path=path, installed=self.installed) @@ -250,7 +251,8 @@ def __call__( Examples: >>> import harmont as hm >>> proj = hm.zig(path=".", version="0.14.1") - >>> hm.pipeline([proj.build(), proj.test()]) + >>> proj.build() + >>> proj.test() """ toolchain = _make_toolchain(version=version, image=image, base=base) if path is None: diff --git a/crates/hm-dsl-engine/harmont-py/harmont/py/uv.py b/crates/hm-dsl-engine/harmont-py/harmont/py/uv.py index b4e81785..847e6839 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/py/uv.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/py/uv.py @@ -187,8 +187,8 @@ def __call__( Examples: >>> import harmont.py as hmpy >>> proj = hmpy.uv(path="services/api") - >>> import harmont as hm - >>> hm.pipeline([proj.test(), proj.lint()]) + >>> proj.test() + >>> proj.lint() """ return _make_uv(path=path, version=version, image=image, base=base)