From bf6f51c9b60c74539a92404589080a24efc54957 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:57:30 -0700 Subject: [PATCH 01/13] feat: EXPR and WRAP_ACTIONS v0 model parity via Rust bindings Bind the v0 Python model to the openjd-rs crates (openjd-expr, openjd-model) and bring the 2023-09 model to RFC 0007 (EXPR) and RFC 0008 (WRAP_ACTIONS) parity: - Rust-backed openjd.expr module (PyO3): expression parsing/evaluation, typed symbol tables (build_symbol_table), FormatString resolve_value, RangeExpr/IntRange, path mapping, and evaluation profiles with the spec's operation/memory limits. - WRAP_ACTIONS: onWrapEnvEnter/onWrapTaskRun/onWrapEnvExit environment actions with all-or-nothing validation, per-hook WrappedAction.* / WrappedEnv.Name / WrappedStep.Name template-variable injection, and create_job instantiation coverage. - EXPR: script-level let bindings on step and environment scripts, typed parameter definitions, creation-scope validation of variable references (including cancelation-mode format strings), spec string coercion and value constraints, timeout format-string scope checks. - Operation-limit error expectations pinned to the evaluator's pre-charged allocation counts (openjd-rs #272/#276). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- Cargo.lock | 22 +- Cargo.toml | 11 + rust-bindings/Cargo.toml | 10 +- rust-bindings/src/model/job.rs | 15 +- rust-bindings/src/model/template_types.rs | 45 +- rust-bindings/src/sessions/session.rs | 11 +- src/openjd/model/_create_job.py | 72 ++- .../model/_format_strings/_expression.py | 21 + .../model/_format_strings/_format_string.py | 36 +- src/openjd/model/_format_strings/_nodes.py | 48 +- src/openjd/model/_internal/_create_job.py | 61 ++- .../_variable_reference_validation.py | 26 +- src/openjd/model/_symbol_table.py | 15 + src/openjd/model/_types.py | 48 +- src/openjd/model/v2023_09/__init__.py | 2 + src/openjd/model/v2023_09/_model.py | 487 +++++++++++++++--- test/openjd/expr/test_operation_limit.py | 24 +- .../model_v0/_internal/test_create_job.py | 2 +- test/openjd/model_v0/v2023_09/test_action.py | 19 +- 19 files changed, 848 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 969252ff..5ca22e43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,9 +405,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "0.0.3" +version = "0.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50ba32164f9e098d5da618776a32afbb32270adcbe3d3d006107dae11e37c91" +checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d" dependencies = [ "arraydeque", "smallvec", @@ -706,9 +706,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c9407d110e2b7a769d163314883d50e5333fc697bb9bb8ace1daeb48b5a58cd" +version = "0.2.0" dependencies = [ "regex", "regex-syntax", @@ -723,9 +721,7 @@ dependencies = [ [[package]] name = "openjd-model" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe8867627c211bc704c9a8d94f16b32bc4b184b96560ae1ed2af24522965677" +version = "0.3.1" dependencies = [ "indexmap", "openjd-expr", @@ -755,9 +751,7 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6afe6534a84e47e19ef834d78bf9715e578202b343454f258f2792e08fe503" +version = "0.3.2" dependencies = [ "bitflags", "futures-util", @@ -1271,9 +1265,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "0.0.27" +version = "0.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5897b4c3faadadd35fdb6689f015641f3bc481d5adaaac56231ea15aeb243db3" +checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" dependencies = [ "ahash", "annotate-snippets", @@ -1283,7 +1277,7 @@ dependencies = [ "granit-parser", "nohash-hasher", "num-traits", - "serde", + "serde_core", "smallvec", "zmij", ] diff --git a/Cargo.toml b/Cargo.toml index 8ca83b1e..85d76da4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,14 @@ [workspace] members = ["rust-bindings"] resolver = "2" + +# ── Local-development overrides (opt-in) ────────────────────────────── +# Redirect the openjd-* crates to a sibling ~/openjd-rs checkout so the +# bindings build against the in-flight Rust workspace instead of crates.io. +# Cargo only honors [patch] at the workspace root, so it lives here rather +# than in rust-bindings/Cargo.toml. Paths are relative to this file. +# Re-comment before committing so CI/published wheels resolve from crates.io. +[patch.crates-io] +openjd-expr = { path = "../openjd-rs/crates/openjd-expr" } +openjd-model = { path = "../openjd-rs/crates/openjd-model" } +openjd-sessions = { path = "../openjd-rs/crates/openjd-sessions" } diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 035d1958..1c514322 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.1.2" -openjd-model = "0.3.0" -openjd-sessions = "0.3.1" +openjd-expr = "0.2.0" +openjd-model = "0.3.1" +openjd-sessions = "0.3.2" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" @@ -52,6 +52,10 @@ windows = { version = "0.62", features = [ # wheel is built against the crates.io versions, which is what the # Cargo.lock file pins. # +# NOTE: the [patch.crates-io] block that redirects these to a sibling +# ~/openjd-rs checkout must live at the *workspace root* Cargo.toml — cargo +# ignores `[patch]` in non-root package manifests. See ../Cargo.toml. +# # [patch.crates-io] # openjd-expr = { path = "../../openjd-rs/crates/openjd-expr" } # openjd-model = { path = "../../openjd-rs/crates/openjd-model" } diff --git a/rust-bindings/src/model/job.rs b/rust-bindings/src/model/job.rs index bb7c13ba..c956e3f1 100644 --- a/rust-bindings/src/model/job.rs +++ b/rust-bindings/src/model/job.rs @@ -557,7 +557,7 @@ impl PyEmbeddedFile { fn new( name: String, r#type: String, - filename: Option, + filename: Option, data: Option, runnable: Option, #[allow(non_snake_case)] endOfLine: Option, @@ -586,7 +586,9 @@ impl PyEmbeddedFile { inner: job::EmbeddedFile { name, file_type, - filename: filename.map(|f| f.inner), + // Plain string per the 2023-09 schema: the field is not + // @fmtstring, so "{{ }}" sequences are literal text. + filename, data: data.map(|d| d.inner), runnable, end_of_line, @@ -607,7 +609,7 @@ impl PyEmbeddedFile { #[getter] fn filename(&self) -> Option { - self.inner.filename.as_ref().map(|f| f.raw().to_string()) + self.inner.filename.clone() } #[getter] @@ -1085,6 +1087,9 @@ impl PyCancelationMode { match &self.inner { job::CancelationMode::Terminate => "TERMINATE", job::CancelationMode::NotifyThenTerminate { .. } => "NOTIFY_THEN_TERMINATE", + // The raw format string; the mode decision is deferred to run + // time. Matches the openjd-rs serialization of DeferredMode. + job::CancelationMode::DeferredMode { mode, .. } => mode.raw(), } } @@ -1093,6 +1098,10 @@ impl PyCancelationMode { match &self.inner { job::CancelationMode::NotifyThenTerminate { notify_period_in_seconds, + } + | job::CancelationMode::DeferredMode { + notify_period_in_seconds, + .. } => notify_period_in_seconds .as_ref() .map(|t| t.raw().to_string()), diff --git a/rust-bindings/src/model/template_types.rs b/rust-bindings/src/model/template_types.rs index 3486c7d0..bd8ccd74 100644 --- a/rust-bindings/src/model/template_types.rs +++ b/rust-bindings/src/model/template_types.rs @@ -159,6 +159,18 @@ impl PyCancelationMode { "NOTIFY_THEN_TERMINATE" => CancelationMode::NotifyThenTerminate { notify_period_in_seconds: notify_period_in_seconds.map(|fs| fs.inner), }, + // A format-string mode (FEATURE_BUNDLE_1) defers the + // TERMINATE-vs-NOTIFY_THEN_TERMINATE decision to run time. + // Mirrors the openjd-rs serde impl, which routes any mode + // containing "{{" to CancelationMode::DeferredMode. + other if other.contains("{{") => CancelationMode::DeferredMode { + mode: openjd_expr::FormatString::new(other).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "invalid cancelation mode format string: {e}" + )) + })?, + notify_period_in_seconds: notify_period_in_seconds.map(|fs| fs.inner), + }, other => { return Err(pyo3::exceptions::PyValueError::new_err(format!( "unknown cancelation mode: {other}" @@ -173,6 +185,9 @@ impl PyCancelationMode { match &self.inner { CancelationMode::Terminate => "TERMINATE", CancelationMode::NotifyThenTerminate { .. } => "NOTIFY_THEN_TERMINATE", + // The raw format string; the mode decision is deferred to run + // time. Matches the openjd-rs serialization of DeferredMode. + CancelationMode::DeferredMode { mode, .. } => mode.raw(), } } @@ -182,6 +197,10 @@ impl PyCancelationMode { CancelationMode::Terminate => None, CancelationMode::NotifyThenTerminate { notify_period_in_seconds, + } + | CancelationMode::DeferredMode { + notify_period_in_seconds, + .. } => notify_period_in_seconds .as_ref() .map(|fs| PyFormatString { inner: fs.clone() }), @@ -208,6 +227,19 @@ impl PyCancelationMode { "CancelationMode(mode='NOTIFY_THEN_TERMINATE', notify_period_in_seconds={raw})" ) } + CancelationMode::DeferredMode { + mode, + notify_period_in_seconds, + } => { + let raw = notify_period_in_seconds + .as_ref() + .map(|fs| format!("{:?}", fs.raw())) + .unwrap_or_else(|| "None".to_string()); + format!( + "CancelationMode(mode={:?}, notify_period_in_seconds={raw})", + mode.raw() + ) + } } } @@ -524,7 +556,7 @@ impl PyEmbeddedFile { fn new( name: String, r#type: &str, - filename: Option, + filename: Option, data: Option, runnable: Option, end_of_line: Option<&str>, @@ -533,7 +565,9 @@ impl PyEmbeddedFile { inner: EmbeddedFile { name, file_type: parse_file_type(r#type)?, - filename: filename.map(|fs| fs.inner), + // Plain string per the 2023-09 schema: the field is not + // @fmtstring, so "{{ }}" sequences are literal text. + filename, data: data.map(|fs| fs.inner), runnable, end_of_line: end_of_line.map(parse_eol).transpose()?, @@ -553,11 +587,8 @@ impl PyEmbeddedFile { } #[getter] - fn filename(&self) -> Option { - self.inner - .filename - .as_ref() - .map(|fs| PyFormatString { inner: fs.clone() }) + fn filename(&self) -> Option { + self.inner.filename.clone() } #[getter] diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index f4056878..649a3470 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -540,19 +540,20 @@ impl PySession { /// Run a task. Non-blocking — spawns the onRun action on a background thread. /// - /// `step_name` is surfaced as `WrappedStep.Name` to a wrapping environment's - /// `onWrapTaskRun` hook (RFC 0008). - #[pyo3(signature = (*, step_script, step_name="", task_parameter_values=None, resolved_symtab=None, os_env_vars=None))] + /// `step_name` is surfaced as `WrappedStep.Name` to a wrapping + /// environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty + /// string when the caller does not supply one. + #[pyo3(signature = (*, step_script, step_name=None, task_parameter_values=None, resolved_symtab=None, os_env_vars=None))] fn run_task( &self, step_script: &PyStepScript, - step_name: &str, + step_name: Option, task_parameter_values: Option<&Bound<'_, PyDict>>, resolved_symtab: Option<&crate::expr::PySerializedSymbolTable>, os_env_vars: Option>, ) -> PyResult<()> { let script = step_script.inner.clone(); - let step_name = step_name.to_owned(); + let step_name = step_name.unwrap_or_default(); let task_params = match task_parameter_values { Some(d) => Some(extract_task_parameter_values(d)?), None => None, diff --git a/src/openjd/model/_create_job.py b/src/openjd/model/_create_job.py index 4f8ca38d..acee41c2 100644 --- a/src/openjd/model/_create_job.py +++ b/src/openjd/model/_create_job.py @@ -7,6 +7,7 @@ from pydantic import ValidationError from ._errors import CompatibilityError, DecodeValidationError +from ._format_strings import FormatStringError from ._symbol_table import SymbolTable from ._internal import instantiate_model from ._merge_job_parameter import merge_job_parameter_definitions @@ -33,6 +34,20 @@ _LEGACY_SCALAR_TYPE_NAMES = frozenset({"STRING", "INT", "FLOAT", "PATH"}) +def _is_uri(value: str) -> bool: + """Whether ``value`` is a URI (``scheme://...`` with an RFC 3986 scheme). + Mirrors openjd-rs's ``uri_path::is_uri``: the scheme must start with an + ASCII letter and contain only ASCII alphanumerics, ``+``, ``.``, or ``-``. + """ + scheme_end = value.find("://") + if scheme_end <= 0: + return False + scheme = value[:scheme_end] + if not scheme[0].isascii() or not scheme[0].isalpha(): + return False + return all(c.isascii() and (c.isalnum() or c in "+.-") for c in scheme) + + # ======================================================================= # ================ Preprocessing Job Parameters ========================= # ======================================================================= @@ -67,6 +82,7 @@ def _collect_defaults_2023_09( job_template_dir: Path, current_working_dir: Path, allow_job_template_dir_walk_up: bool, + allow_uri_path_values: bool = False, ) -> JobParameterValues: if not allow_job_template_dir_walk_up and not job_template_dir.is_absolute(): raise ValueError( @@ -89,6 +105,20 @@ def _collect_defaults_2023_09( ) continue default = str(param.default) + # RFC 0006 (EXPR): URI-form PATH defaults ("s3://...") are + # preserved verbatim — they are not filesystem paths, so the + # template-dir join and walk-up enforcement do not apply. + # Mirrors openjd-rs's preprocess_job_parameters URI handling. + if ( + param.type.name == "PATH" + and allow_uri_path_values + and default != "" + and _is_uri(default) + ): + return_value[param.name] = ParameterValue( + type=ParameterValueType(param.type), value=default + ) + continue # Make PATH defaults relative to job_template_dir, and # enforce the `allow_job_template_dir_walk_up` parameter request. if param.type.name == "PATH" and default != "": @@ -126,8 +156,14 @@ def _collect_defaults_2023_09( type=ParameterValueType(param.type), value=value ) continue - # Join any provided relative PATH parameter value with the current_working_directory (except the empty value "") - if param.type.name == "PATH" and value != "" and not Path(value).is_absolute(): + # Join any provided relative PATH parameter value with the current_working_directory + # (except the empty value "" and, under EXPR, URI values which are preserved verbatim) + if ( + param.type.name == "PATH" + and value != "" + and not (allow_uri_path_values and _is_uri(str(value))) + and not Path(value).is_absolute() + ): value = str(current_working_dir / value) return_value[param.name] = ParameterValue( type=ParameterValueType(param.type), value=str(value) @@ -265,6 +301,12 @@ def preprocess_job_parameters( job_template_dir, current_working_dir, allow_job_template_dir_walk_up, + # RFC 0006: URI-form PATH values are only meaningful with + # the EXPR extension (path ops and mapping understand + # URIs). Matches the openjd-rs CLI, which enables URI + # path values for EXPR templates. + allow_uri_path_values="EXPR" + in (getattr(job_template, "extensions", None) or []), ) _check_2023_09(parameterDefinitions, return_value) else: @@ -342,10 +384,16 @@ def create_job( if job_template.specificationVersion == TemplateSpecificationVersion.JOBTEMPLATE_v2023_09: from .v2023_09 import ValueReferenceConstants as ValueReferenceConstants_2023_09 - # EXPR-extension typed params (BOOL / RANGE_EXPR / LIST[*]) carry native - # values; record their OpenJD type so the typed symbol-table builder - # coerces them to the right ExprType during expression evaluation. The - # original scalar types keep their existing string-based handling. + # Record each parameter's OpenJD type so the typed symbol-table + # builder coerces its (stringly carried) value to the right ExprType + # during EXPR expression evaluation — matching openjd-rs, whose + # create_job symbol table carries typed ExprValues for every + # parameter (an INT param is an int, so `{{ Param.X + 3 }}` works). + # STRING needs no coercion, and PATH stays string-typed in template + # scope: openjd-rs excludes Param.* for PATH here (host-context only) + # and seeds RawParam.* for PATH as a plain string. + # The typing only affects the EXPR evaluation path; non-EXPR + # templates keep their existing string-based interpolation. expr_types: dict[str, str] = {} for name, param in all_job_parameter_values.items(): prefix = ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value @@ -353,10 +401,20 @@ def create_job( if param.type != "PATH": symtab[f"{prefix}.{name}"] = all_job_parameter_values[name].value symtab[f"{raw_prefix}.{name}"] = all_job_parameter_values[name].value - if param.type.name not in _LEGACY_SCALAR_TYPE_NAMES: + if param.type.name not in ("STRING", "PATH"): expr_types[f"{prefix}.{name}"] = param.type.value expr_types[f"{raw_prefix}.{name}"] = param.type.value symtab.expr_types.update(expr_types) + # RFC 0007 §7.3.1 (EXPR): Job.Name is the job's resolved name, + # available to the job's steps and environments. Seeded before + # instantiation so step fields (including step-level `let` bindings) + # can reference it — mirroring openjd-rs's instantiate_job, which + # resolves the name first and seeds it into the symbol table. + if "EXPR" in (getattr(job_template, "extensions", None) or []): + try: + symtab["Job.Name"] = job_template.name.resolve(symtab=symtab) + except FormatStringError as exc: + raise DecodeValidationError(f"Failed to resolve the job's name: {exc}") else: raise NotImplementedError( f"Spec version {job_template.specificationVersion} not implemented." diff --git a/src/openjd/model/_format_strings/_expression.py b/src/openjd/model/_format_strings/_expression.py index 7968eff9..844dc7c0 100644 --- a/src/openjd/model/_format_strings/_expression.py +++ b/src/openjd/model/_format_strings/_expression.py @@ -86,6 +86,27 @@ def evaluate( raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}") + def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> Any: + """Evaluate the expression, preferring the engine's typed value form. + + For EXPR-backed expressions this returns the engine ``ExprValue``, + which keeps the result's EXPR type (path/int/float/list/...) and its + string-rendering fidelity when re-seeded into a symbol table — the + session runtime's ``let`` bindings (RFC 0007) rely on this. Legacy + (non-EXPR) expressions return their native value, identical to + :meth:`evaluate`. + + Raises: + ExpressionError: If the expression could not be evaluated. + """ + evaluate_value = getattr(self._expresion_tree, "evaluate_value", None) + if evaluate_value is None: + return self.evaluate(symtab=symtab, path_format=path_format) + try: + return evaluate_value(symtab=symtab, path_format=path_format) + except ValueError as exc: + raise ExpressionError(f"Expression failed validation: {str(exc)}") + def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> str: """Evaluate the expression and coerce the result to the string form that is substituted into the surrounding format string. diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index 5ee6348c..f43244d2 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -1,7 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. from dataclasses import dataclass -from typing import Optional, Union +from typing import Any, Optional, Union from .._errors import ExpressionError, TokenError from .._symbol_table import SymbolTable @@ -128,6 +128,40 @@ def resolve(self, *, symtab: SymbolTable, path_format: Optional[object] = None) return "".join(resolved_list) + def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = None) -> Any: + """Typed resolution of this format string (RFC 0005/0006). + + When the format string is a single whole-field ``{{ ... }}`` EXPR + expression (only whitespace outside the braces), returns the engine's + typed value (an ``ExprValue``), preserving lists, null, and numeric + types — callers such as the session runner's argument resolution use + this for RFC 0005 §1.3.2 list flattening and null skipping, mirroring + openjd-rs's ``FormatString::resolve_with``. In every other case + (multi-segment strings, legacy non-EXPR expressions) the result is the + ordinary resolved string, identical to :meth:`resolve`. + + Raises: + FormatStringError: if the expression cannot be evaluated. + """ + expressions = self.expressions + if len(expressions) == 1: + info = expressions[0] + expression = info.expression + prefix = self.original_value[: info.start_pos] + suffix = self.original_value[info.end_pos :] + if expression is not None and not prefix.strip() and not suffix.strip(): + try: + return expression.evaluate_value(symtab=symtab, path_format=path_format) + except ExpressionError as exc: + raise FormatStringError( + string=self.original_value, + start=info.start_pos, + end=info.end_pos, + expr=expression.expr, + details=str(exc), + ) + return self.resolve(symtab=symtab, path_format=path_format) + def _preprocess( self, *, context: ModelParsingContextInterface ) -> list[Union[str, ExpressionInfo]]: diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 0bcb5f3a..5e29afa9 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -66,8 +66,17 @@ def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> st EXPR-backed nodes override this to use the engine's own spec-defined coercion (RFC 0005), so e.g. ``true``/``false``/``null`` and lists render per the specification rather than as Python reprs. + + A ``None`` value interpolates as the empty string, matching the EXPR + engine's null rendering (RFC 0005) — relevant for nullable injected + symbols such as ``WrappedAction.Cancelation.NotifyPeriodInSeconds`` + (RFC 0008 follow-up), which is ``None`` when no notify period + applies. """ - return str(self.evaluate(symtab=symtab, path_format=path_format)) + value = self.evaluate(symtab=symtab, path_format=path_format) + if value is None: + return "" + return str(value) @abstractmethod def __repr__(self) -> str: # pragma: no cover @@ -187,7 +196,15 @@ def validate_symbol_refs(self, *, symbols: set[str], symbol_types: Any = None) - # accessed_symbols are the free variable references in full dotted # spelling (e.g. "Param.X"); local let-bound names are excluded by the # engine. Compare against the set of symbols visible at this location. - missing = accessed - symbols + # A reference is defined when any dotted prefix of it is a defined + # symbol: the remaining segments are property/method access on that + # symbol's value (e.g. `work_dir.name` on a `let`-bound path), whose + # type-correctness is checked by the typed validation above when the + # types are known, and at evaluation time otherwise — mirroring the + # Rust engine's treatment of unknown-typed symbols. + from ._expr_support import longest_defined_prefix + + missing = {name for name in accessed if longest_defined_prefix(name, symbols) is None} if missing: raise _missing_symbol_error(sorted(missing)[0], symbols) self._check_comprehension_shadowing(symbols) @@ -212,6 +229,7 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: """ from ._expr_support import ( ExprProfile, + HostContext, map_eval_error, symtab_to_expr_values, ) @@ -219,16 +237,36 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: values = symtab_to_expr_values( symtab, types=symtab.expr_types or None, path_format=path_format ) + profile = ExprProfile.current() + # A symbol table carrying host-context path mapping rules (session + # scope) enables host-context functions such as apply_path_mapping, + # applying those rules — mirroring openjd-rs's session-scope + # HostContext::WithRules. None means template scope (no host context). + host_rules = getattr(symtab, "expr_host_rules", None) + if host_rules is not None: + profile = profile.with_host_context(HostContext.with_rules(host_rules)) try: - return self._parsed.evaluate( - values=values, profile=ExprProfile.current(), path_format=path_format - ) + return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error raise map_eval_error(exc) def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: return self._evaluate_raw(symtab=symtab, path_format=path_format).item() + def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + """Evaluate and return the engine's ``ExprValue`` (not unwrapped to a + native Python value). + + Callers that re-seed the result into a symbol table (e.g. the session + runtime's ``let`` bindings, RFC 0007) should use this form: the typed + symbol-table builder passes an ``ExprValue`` through unchanged, so the + bound value keeps its EXPR type (a path stays a path for property + access) and its rendering fidelity (a float's declared trailing zeros + are preserved), matching the Rust runtime's natively typed symbol + table. + """ + return self._evaluate_raw(symtab=symtab, path_format=path_format) + def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str: # ``str(ExprValue)`` applies the engine's RFC 0005 format-string # coercion (e.g. ``true``/``false``, double-quoted list items, preserved diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index 4583035a..59b36d76 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -11,7 +11,41 @@ from .._format_strings import FormatString from .._types import OpenJDModel -__all__ = ("instantiate_model",) +__all__ = ("instantiate_model", "resolve_whole_field_typed_list") + + +def resolve_whole_field_typed_list(value: FormatString, symtab: SymbolTable) -> Any: + """RFC 0006 typed whole-field resolution to a native list, or ``None``. + + When ``value`` is a single whole-field ``{{ ... }}`` expression (only + whitespace outside the expression) that evaluates to a list, return the + native Python list. Any other shape or result — multi-segment format + strings, scalar results, or evaluation errors — returns ``None`` so the + caller falls back to ordinary string resolution, mirroring the + typed-then-string fallback in openjd-rs's range instantiation. + """ + expressions = value.expressions + if len(expressions) != 1: + return None + info = expressions[0] + expression = info.expression + if expression is None: + return None + original = value.original_value + if original[: info.start_pos].strip() or original[info.end_pos :].strip(): + return None + try: + result = expression.evaluate_value(symtab=symtab) + except Exception: # noqa: BLE001 - fall back to string resolution + return None + # EXPR-backed evaluation returns the engine's ExprValue; unwrap lists. + result_type = getattr(result, "type", None) + if result_type is not None and str(result_type).startswith("list["): + return result.item() + if isinstance(result, list): + return result + return None + # Cache for TypeAdapter instances _type_adapter_cache: Dict[int, TypeAdapter] = {} @@ -92,6 +126,15 @@ def instantiate_model( # noqa: C901 errors = list[InitErrorDetails]() instantiated_fields = dict[str, Any]() + # Extend the symbol table for this model's subtree if defined (e.g. a + # step's Step.Name and step-level EXPR `let` bindings). This runs before + # the transform: StepTemplate's syntax-sugar transform folds step-level + # `let` bindings into the script (their runtime channel), so the original + # model is the one that still carries them for create_job-time fields + # (parameter space, host requirements). + if model._job_creation_metadata.extends_symtab is not None: + symtab = model._job_creation_metadata.extends_symtab(model, symtab) + # Apply pre-transform if defined if model._job_creation_metadata.transform is not None: model = model._job_creation_metadata.transform(model) @@ -103,7 +146,7 @@ def instantiate_model( # noqa: C901 if create_as_metadata.model is not None: target_model = create_as_metadata.model elif create_as_metadata.callable is not None: - target_model = create_as_metadata.callable(model) + target_model = create_as_metadata.callable(model, symtab) for field_name in model.__class__.model_fields.keys(): if field_name in model._job_creation_metadata.exclude_fields: @@ -124,6 +167,7 @@ def instantiate_model( # noqa: C901 with capture_validation_errors(output_errors=errors, loc=(field_name,), input=field_value): # Instantiate and resolve format string expressions needs_resolve = field_name in model._job_creation_metadata.resolve_fields + typed_resolve = field_name in model._job_creation_metadata.typed_resolve_fields if isinstance(field_value, list): if field_name in model._job_creation_metadata.reshape_field_to_dict: key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] @@ -137,7 +181,9 @@ def instantiate_model( # noqa: C901 elif isinstance(field_value, dict): instantiated = _instantiate_dict_field(field_value, symtab, needs_resolve) else: - instantiated = _instantiate_noncollection_value(field_value, symtab, needs_resolve) + instantiated = _instantiate_noncollection_value( + field_value, symtab, needs_resolve, typed_resolve=typed_resolve + ) # Validate as the target field type using cached TypeAdapter type_adapter = get_type_adapter(target_field_type) @@ -164,6 +210,8 @@ def _instantiate_noncollection_value( value: Any, symtab: SymbolTable, needs_resolve: bool, + *, + typed_resolve: bool = False, ) -> Any: """Instantiate a single value that must not be a collection type (list, dict, etc). @@ -172,10 +220,17 @@ def _instantiate_noncollection_value( value (Any): Value to process. symtab (SymbolTable): Symbol table for format string value lookups. needs_resolve (bool): Whether to resolve the value as a format string. + typed_resolve (bool): Whether to first attempt RFC 0006 typed + whole-field resolution to a native list (task-parameter ``range`` + fields), falling back to string resolution. """ if isinstance(value, OpenJDModel): return instantiate_model(value, symtab) elif isinstance(value, FormatString) and needs_resolve: + if typed_resolve: + typed_value = resolve_whole_field_typed_list(value, symtab) + if typed_value is not None: + return typed_value value = value.resolve(symtab=symtab) return value diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index 7298899a..10ad7863 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -412,11 +412,25 @@ def _validate_model_template_variable_references( # Add in all of the symbols passed down from the parent. validation_symbols.update_self(symbols) + # Per-field extra symbols (e.g. the RFC 0008 WrappedAction.* variables + # within their wrap hook's action). Added at TEMPLATE scope so they + # are visible in every scope within the field's subtree, including + # creation-time fields such as the action's timeout — a wrap hook's + # timeout may forward "{{WrappedAction.Timeout}}". + for symbol in model._template_field_inject.get(field_name, set()): + symbol_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" + _add_symbol(validation_symbols, ResolutionScope.TEMPLATE, symbol_name) + + # Per-field scope override: fields that resolve at job creation (e.g. + # an Action's timeout/cancelation) validate at their declared scope + # rather than the model's ambient scope. + field_scope = model._template_field_scopes.get(field_name, current_scope) + errors.extend( _validate_model_template_variable_references( field_model, field_value, - current_scope, + field_scope, symbol_prefix, validation_symbols, (*loc, field_name), @@ -549,9 +563,15 @@ def _validate_let_bindings( enclosing: set = set(symbols[current_scope]) self_symbols: set = set(value_symbols.get("__self__", ScopedSymtabs())[current_scope]) + # A script's embedded files define Env.File.*/Task.File.* symbols that its + # `let` bindings may reference: the runtime allocates the file paths + # before evaluating the bindings (file contents are written after, so + # `data` can reference let-bound values) — matching openjd-rs. + file_symbols: set = set(value_symbols.get("embeddedFiles", ScopedSymtabs())[current_scope]) # Visible to the bindings: enclosing scope + this model's injected/defined - # symbols, minus the let names themselves (added progressively below). - accumulated: set = (enclosing | self_symbols) - let_names + # symbols + its embedded-file symbols, minus the let names themselves + # (added progressively below). + accumulated: set = (enclosing | self_symbols | file_symbols) - let_names for index, binding in enumerate(let_value): if not isinstance(binding, str): continue diff --git a/src/openjd/model/_symbol_table.py b/src/openjd/model/_symbol_table.py index c709c5f9..7e6d8763 100644 --- a/src/openjd/model/_symbol_table.py +++ b/src/openjd/model/_symbol_table.py @@ -20,6 +20,14 @@ class SymbolTable: # (rather than stashed dynamically) so it is preserved across copies and # unions; empty for non-EXPR symbol tables. expr_types: dict[str, str] + # Optional host-context path mapping rules (``openjd.expr.PathMappingRule`` + # values). When set (even to an empty list), EXPR expressions evaluated + # against this symbol table run with a host context: host-context + # functions such as ``apply_path_mapping`` are available and apply these + # rules — the v0 equivalent of openjd-rs's session-scope + # ``HostContext::WithRules``. ``None`` means no host context (template + # scope). + expr_host_rules: Optional[list[Any]] def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = None): """Initialize the SymbolTable @@ -30,10 +38,13 @@ def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = Non """ self._table = dict() self.expr_types = dict() + self.expr_host_rules = None if source is not None: if isinstance(source, SymbolTable): self._table.update(source._table) self.expr_types.update(source.expr_types) + if source.expr_host_rules is not None: + self.expr_host_rules = list(source.expr_host_rules) elif isinstance(source, dict): self._table.update(source) else: @@ -74,10 +85,14 @@ def union(self, *symtabs: Union[SymbolTable, dict[str, Any]]) -> SymbolTable: retval = SymbolTable() retval._table.update(self._table) retval.expr_types.update(self.expr_types) + if self.expr_host_rules is not None: + retval.expr_host_rules = list(self.expr_host_rules) for symtab in symtabs: if isinstance(symtab, SymbolTable): retval._table.update(symtab._table) retval.expr_types.update(symtab.expr_types) + if symtab.expr_host_rules is not None: + retval.expr_host_rules = list(symtab.expr_host_rules) elif isinstance(symtab, dict): retval._table.update(symtab) else: diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index ff65bb7b..ccbf3ffa 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -244,9 +244,11 @@ def __repr__(self) -> str: @dataclass(frozen=True, eq=False, **dataclass_kwargs) class JobCreateAsMetadata: # Only one of the following may be non-None - # model: Union[Type["OpenJDModel"], Callable[["OpenJDModel"], Type["OpenJDModel"]]] + # model: Union[Type["OpenJDModel"], Callable[["OpenJDModel", SymbolTable], Type["OpenJDModel"]]] model: Optional[Type["OpenJDModel"]] = field(default=None) - callable: Optional[Callable[["OpenJDModel"], Type["OpenJDModel"]]] = field(default=None) + callable: Optional[Callable[["OpenJDModel", SymbolTable], Type["OpenJDModel"]]] = field( + default=None + ) @dataclass(frozen=True, eq=False, **dataclass_kwargs) @@ -266,6 +268,17 @@ class JobCreationMetadata: 3. lists of a mix of FormatStrings and non-FormatStrings (e.g. ints,floats,regular-strings,etc) """ + typed_resolve_fields: set[str] = field(default_factory=set) + """The names of fields (a subset of ``resolve_fields``) that first attempt + RFC 0006 typed whole-field resolution: a field whose value is a single + whole-field ``{{ ... }}`` expression evaluating to a list keeps the native + list instead of a stringified rendering. Used by task-parameter ``range`` + fields so ``range: "{{Param.Values}}"`` with a ``LIST[*]`` job parameter + instantiates to the literal value list, matching openjd-rs. Fields whose + typed resolution does not apply (multi-segment format strings, non-list + results, or evaluation errors) fall back to normal string resolution. + """ + create_as: Optional[JobCreateAsMetadata] = field(default=None) """The class to create the model as during instantiation. Options: 1. None -- create as the model class itself. @@ -312,6 +325,20 @@ class JobCreationMetadata: Use-case: Resolving syntax sugar on StepTemplate before creating Step. """ + extends_symtab: Optional[Callable[["OpenJDModel", SymbolTable], SymbolTable]] = field( + default=None + ) + """A callable that returns the symbol table to use when instantiating this + model and its subtree, given the enclosing symbol table. The returned table + is used for all of the model's fields; the enclosing table is unaffected. + arg0 - The model being instantiated. + arg1 - The enclosing symbol table. + Use-case: A StepTemplate seeds Step.Name and evaluates its step-level + EXPR `let` bindings (RFC 0007 §3.6) so the step's parameter space, + host requirements, and script instantiate against them — mirroring + openjd-rs's per-step symbol table in instantiate_step. + """ + class OpenJDModel(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -343,6 +370,23 @@ class OpenJDModel(BaseModel): # "__self__" provides the variables exported by this model via `__template_variable_definitions`. _template_variable_sources: ClassVar[dict[str, set[str]]] = {} + # Per-field variable-scope overrides: fields listed here (and their + # submodels) validate their format-string references at the given scope + # instead of the model's ambient scope. Used for fields that resolve at + # job creation while their siblings resolve at run time — e.g. an + # Action's `timeout` and `cancelation` are creation-time fields + # (template scope: no Session.*, no Env.File.*/Task.File.*, no host + # functions) while its `command`/`args` resolve in the session. + _template_field_scopes: ClassVar[dict[str, ResolutionScope]] = {} + + # Per-field extra symbol injection: symbols listed here are visible in + # every scope, but only within the named field's subtree. Names use the + # DefinesTemplateVariables.inject spelling (a "|" prefix discards the + # parent scope prefix). Used for the RFC 0008 wrap hooks, whose + # WrappedAction.* / WrappedEnv.* / WrappedStep.* variables exist only + # within their hook's action. + _template_field_inject: ClassVar[dict[str, set[str]]] = {} + # ---- # Metadata used in the creation of a Job from a Job Template diff --git a/src/openjd/model/v2023_09/__init__.py b/src/openjd/model/v2023_09/__init__.py index 13f81813..68073d87 100644 --- a/src/openjd/model/v2023_09/__init__.py +++ b/src/openjd/model/v2023_09/__init__.py @@ -18,6 +18,7 @@ AttributeCapabilityValue, AttributeRequirement, AttributeRequirementTemplate, + CancelationMethodDeferred, CancelationMethodNotifyThenTerminate, CancelationMethodTerminate, CancelationMode, @@ -113,6 +114,7 @@ "AttributeCapabilityValue", "AttributeRequirement", "AttributeRequirementTemplate", + "CancelationMethodDeferred", "CancelationMethodNotifyThenTerminate", "CancelationMethodTerminate", "CancelationMode", diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index f7d21254..097b894c 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -15,12 +15,14 @@ field_validator, model_validator, ConfigDict, + Discriminator, StringConstraints, Field, PositiveInt, PositiveFloat, StrictBool, StrictInt, + Tag, ValidationError, ValidationInfo, ) @@ -46,6 +48,7 @@ prevalidate_model_template_variable_references, ) from .._range_expr import IntRangeExpr +from .._symbol_table import SymbolTable from .._types import ( DefinesTemplateVariables, JobCreateAsMetadata, @@ -283,6 +286,31 @@ class CancelationMode(str, Enum): NotifyPeriodType = Annotated[int, Field(ge=1, le=600)] +def _validate_notify_period_value( + v: Any, info: ValidationInfo +) -> Optional[Union[int, FormatString]]: + """Shared notifyPeriodInSeconds validation for + CancelationMethodNotifyThenTerminate and CancelationMethodDeferred.""" + if v is None: + return v + context = cast(Optional[ModelParsingContext], info.context) + if isinstance(v, str): + if context and "FEATURE_BUNDLE_1" not in context.extensions: + # Try to parse as int, fail if not + try: + return int(v) + except ValueError: + raise ValueError( + "notifyPeriodInSeconds as a format string requires the FEATURE_BUNDLE_1 extension." + ) + return validate_int_fmtstring_field(v, ge=1, context=context) + if isinstance(v, int): + if v < 1 or v > 600: + raise ValueError("notifyPeriodInSeconds must be between 1 and 600") + return v + return v + + class CancelationMethodNotifyThenTerminate(OpenJDModel_v2023_09): """Notify-then-terminate cancelation mode for an Action. @@ -323,24 +351,7 @@ class CancelationMethodNotifyThenTerminate(OpenJDModel_v2023_09): def _validate_notify_period( cls, v: Any, info: ValidationInfo ) -> Optional[Union[int, FormatString]]: - if v is None: - return v - context = cast(Optional[ModelParsingContext], info.context) - if isinstance(v, str): - if context and "FEATURE_BUNDLE_1" not in context.extensions: - # Try to parse as int, fail if not - try: - return int(v) - except ValueError: - raise ValueError( - "notifyPeriodInSeconds as a format string requires the FEATURE_BUNDLE_1 extension." - ) - return validate_int_fmtstring_field(v, ge=1, context=context) - if isinstance(v, int): - if v < 1 or v > 600: - raise ValueError("notifyPeriodInSeconds must be between 1 and 600") - return v - return v + return _validate_notify_period_value(v, info) class CancelationMethodTerminate(OpenJDModel_v2023_09): @@ -357,6 +368,130 @@ class CancelationMethodTerminate(OpenJDModel_v2023_09): mode: Literal[CancelationMode.TERMINATE] +class CancelationMethodDeferred(OpenJDModel_v2023_09): + """A cancelation whose ``mode`` is a format string, resolved at run + time (Template Schemas 5.3, FEATURE_BUNDLE_1 extension). + + What is the problem this solves? + + Format strings in general are *already* delay-processed: when a template + says ``args: ["{{WrappedAction.Command}}"]``, the parser just stores + "this is a format string" and the value gets resolved much later, inside + a running session, right before the action launches — that's when the + runtime seeds the ``WrappedAction.*`` variables from the action being + wrapped. "Resolve later" is the normal pipeline for every other field. + + ``mode`` is different because it isn't a normal value field — it's the + *schema selector*. The parser needs to know TERMINATE vs + NOTIFY_THEN_TERMINATE at parse time to decide what shape of object it's + even reading (only one of them allows ``notifyPeriodInSeconds``). So the + "which shape?" decision happens at parse time, but a forwarded value + like ``mode: "{{WrappedAction.Cancelation.Mode}}"`` only exists at run + time — that mismatch made round-trip cancelation forwarding in RFC 0008 + wrap hooks impossible (pydantic's discriminated union rejected the + template with "does not match any of the expected tags"). + + The fix is this class: the parser accepts a format string in ``mode`` + as a third, "decided later" state (gated on the FEATURE_BUNDLE_1 + extension), and the shape decision moves to resolution time, right + before the action runs: + + 1. The runtime seeds ``WrappedAction.Cancelation.Mode`` from the + wrapped action (``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, or + ``None``). + 2. It resolves the ``mode:`` expression against that. + 3. ``"TERMINATE"``/``"NOTIFY_THEN_TERMINATE"`` — the cancelation block + now acts as that method, and its sibling fields are validated + against that shape. ``None`` (null, whole-field expressions only) — + the whole ``cancelation:`` block is treated as never written. + Anything else — the action fails. + + Static validation is *not* deferred: at parse time the validator still + checks the expression is well-formed and that ``WrappedAction.*`` is + only referenced inside wrap hooks. Any format string is accepted — + normal interpolation like ``"{{Prefix}}_THEN_TERMINATE"`` is permitted; + only the resolved value is constrained. You just can't know *which* of + the two modes it'll be until the wrapped action is in front of you — + which is inherent to forwarding: the same wrap environment gets reused + across many steps whose cancelation settings differ. + + Mirrors ``CancelationMode::DeferredMode`` in openjd-rs. See + openjd-specifications Template Schemas 5.3 and RFC 0008 "Cancelation + behavior". + + Attributes: + mode (FormatString): A format string resolving to "TERMINATE" or + "NOTIFY_THEN_TERMINATE"; a whole-field interpolation expression + may also resolve to null. + notifyPeriodInSeconds (Optional[Union[int, FormatString]]): As on + CancelationMethodNotifyThenTerminate; only meaningful when the + mode resolves to NOTIFY_THEN_TERMINATE, and must resolve to + null when the mode resolves to TERMINATE. + """ + + mode: FormatString + notifyPeriodInSeconds: Optional[Union[NotifyPeriodType, FormatString]] = None # noqa: N815 + + _job_creation_metadata = JobCreationMetadata(resolve_fields={"notifyPeriodInSeconds"}) + + @field_validator("mode", mode="before") + @classmethod + def _validate_mode(cls, v: Any, info: ValidationInfo) -> Any: + if isinstance(v, str): + context = cast(Optional[ModelParsingContext], info.context) + if context and "FEATURE_BUNDLE_1" not in context.extensions: + raise ValueError( + "a format string in cancelation mode requires the FEATURE_BUNDLE_1 extension." + ) + # Any format string is permitted (normal format string + # behavior, Template Schemas 5.3); the resolved value is + # checked against the two mode names at run time. Only a + # whole-field expression additionally gets string? null + # semantics (a null result drops the cancelation object). + return v + + @field_validator("notifyPeriodInSeconds", mode="before") + @classmethod + def _validate_notify_period( + cls, v: Any, info: ValidationInfo + ) -> Optional[Union[int, FormatString]]: + return _validate_notify_period_value(v, info) + + +def _cancelation_discriminator(v: Any) -> Optional[str]: + """Callable discriminator for the cancelation union: routes the two + literal modes to their fixed-shape classes and a format-string mode to + :class:`CancelationMethodDeferred` (see that class's docstring for why + the mode decision can be deferred at all).""" + mode = v.get("mode") if isinstance(v, dict) else getattr(v, "mode", None) + if isinstance(mode, CancelationMode): + mode = mode.value + if isinstance(mode, str): + if mode == CancelationMode.NOTIFY_THEN_TERMINATE.value: + return "notify_then_terminate" + if mode == CancelationMode.TERMINATE.value: + return "terminate" + if "{{" in mode: + return "deferred" + if isinstance(v, CancelationMethodNotifyThenTerminate): + return "notify_then_terminate" + if isinstance(v, CancelationMethodTerminate): + return "terminate" + if isinstance(v, CancelationMethodDeferred): + return "deferred" + return None + + +CancelationMethod = Annotated[ + Union[ + Annotated[CancelationMethodNotifyThenTerminate, Tag("notify_then_terminate")], + Annotated[CancelationMethodTerminate, Tag("terminate")], + Annotated[CancelationMethodDeferred, Tag("deferred")], + ], + Discriminator(_cancelation_discriminator), +] + + ArgListType = Annotated[list[ArgString], Field(min_length=1)] # WRAP_ACTIONS (RFC 0008) wrap-hook field names on EnvironmentActions. @@ -436,20 +571,34 @@ class Action(OpenJDModel_v2023_09): timeout (Optional[int]): Maximum allowed runtime of the Action in seconds. Can be a format string with FEATURE_BUNDLE_1 extension. Default: No timeout - cancelation (Optional[Union[CancelationMethodNotifyThenTerminate, CancelationMethodTerminate]]): - If defined, provides details regarding how this action should be canceled. + cancelation (Optional[CancelationMethod]): If defined, provides details + regarding how this action should be canceled. One of + CancelationMethodNotifyThenTerminate, CancelationMethodTerminate, or + CancelationMethodDeferred (a format-string mode resolved + at run time; FEATURE_BUNDLE_1). Default: CancelationMethodTerminate """ command: CommandString args: Optional[ArgListType] = None timeout: Optional[Union[PositiveInt, FormatString]] = None - cancelation: Optional[ - Union[CancelationMethodNotifyThenTerminate, CancelationMethodTerminate] - ] = Field(None, discriminator="mode") + cancelation: Optional[CancelationMethod] = None _job_creation_metadata = JobCreationMetadata(resolve_fields={"timeout"}) + # `timeout` and `cancelation` (its notifyPeriodInSeconds and a deferred + # format-string mode) are plain @fmtstring fields resolved at job + # creation, before any session exists — unlike `command`/`args`, which + # resolve in the session. They therefore validate at template scope: no + # Session.*, no Env.File.*/Task.File.*, and no host-context functions. + # The RFC 0008 wrap hooks may still forward the wrapped action's values + # ("{{WrappedAction.Timeout}}"): the WrappedAction.* symbols are injected + # per-hook at every scope via EnvironmentActions._template_field_inject. + _template_field_scopes = { + "timeout": ResolutionScope.TEMPLATE, + "cancelation": ResolutionScope.TEMPLATE, + } + @field_validator("timeout", mode="before") @classmethod def _validate_timeout(cls, v: Any, info: ValidationInfo) -> Optional[Union[int, FormatString]]: @@ -511,6 +660,28 @@ class EnvironmentActions(OpenJDModel_v2023_09): onWrapTaskRun: Optional[Action] = Field(None) # noqa: N815 onWrapEnvExit: Optional[Action] = Field(None) # noqa: N815 + # RFC 0008: the wrapped-context variables exist only within their wrap + # hook's action, seeded by the runtime when the hook runs in place of the + # wrapped action. WrappedAction.* is available in all three hooks; + # WrappedEnv.Name only in the env-enter/exit hooks; WrappedStep.Name only + # in the task-run hook. Injected per-field at every scope so a hook's + # creation-scoped fields (timeout/cancelation) can round-trip forward + # the wrapped action's values ("{{WrappedAction.Timeout}}", + # "{{WrappedAction.Cancelation.Mode}}"). + _WRAPPED_ACTION_SYMBOLS: ClassVar[set[str]] = { + "|WrappedAction.Command", + "|WrappedAction.Args", + "|WrappedAction.Environment", + "|WrappedAction.Timeout", + "|WrappedAction.Cancelation.Mode", + "|WrappedAction.Cancelation.NotifyPeriodInSeconds", + } + _template_field_inject = { + "onWrapEnvEnter": _WRAPPED_ACTION_SYMBOLS | {"|WrappedEnv.Name"}, + "onWrapEnvExit": _WRAPPED_ACTION_SYMBOLS | {"|WrappedEnv.Name"}, + "onWrapTaskRun": _WRAPPED_ACTION_SYMBOLS | {"|WrappedStep.Name"}, + } + @model_validator(mode="before") @classmethod def _requires_oneof(cls, values: dict[str, Any], info: ValidationInfo) -> dict[str, Any]: @@ -557,6 +728,15 @@ def _requires_oneof(cls, values: dict[str, Any], info: ValidationInfo) -> dict[s on_enter = values.get("onEnter") on_exit = values.get("onExit") + # Base 2023-09 (§3.5) requires onEnter whenever a script is present; + # RFC 0008 relaxes this to "at least one action" when the + # WRAP_ACTIONS extension is declared. The strict base rule is only + # applied at template decode (context present) — job-instantiation + # re-validation has no parsing context, matching the other extension + # gates in this module. + if context is not None and "WRAP_ACTIONS" not in extensions: + if on_enter is None: + raise ValueError("onEnter is required.") if on_enter is None and on_exit is None: raise ValueError("Must define one of: onEnter or onExit") return values @@ -770,6 +950,42 @@ def validate_let_field(value: Any, info: ValidationInfo, *, simple_action: bool return value +# §3.4: the maximum number of values a task parameter's range may take on. +# Not raised by FEATURE_BUNDLE_1 in 2023-09 (matches openjd-rs's +# EffectiveLimits.max_task_param_range_len). +_MAX_TASK_PARAM_RANGE_LEN = 1024 + + +class NameIdentifierLengthMixin: + """Applies the §7.1 identifier length limit — 64 characters, or 512 with + FEATURE_BUNDLE_1 — to a model's ``name`` field. + + The static ``Identifier`` type constraint is the FEATURE_BUNDLE_1 maximum + (512); this tightens it to 64 when the extension is not declared. The + check is skipped when no parsing context is present (job-instantiation + re-validation), matching the other extension gates in this module. + """ + + @field_validator("name", check_fields=False) + @classmethod + def _validate_name_identifier_length(cls, v: str, info: ValidationInfo) -> str: + context = cast(Optional[ModelParsingContext], info.context) + if context is None: + return v + max_len = 512 if "FEATURE_BUNDLE_1" in context.extensions else 64 + if len(v) > max_len: + raise ValueError(f"name must be at most {max_len} characters long") + return v + + +def validate_task_param_range_list_len(value: Any) -> Any: + """§3.4: a task parameter's list-form range may define at most + ``_MAX_TASK_PARAM_RANGE_LEN`` values.""" + if isinstance(value, list) and len(value) > _MAX_TASK_PARAM_RANGE_LEN: + raise ValueError(f"range exceeds {_MAX_TASK_PARAM_RANGE_LEN} elements.") + return value + + class SimpleAction(OpenJDModel_v2023_09): """Syntax sugar for a script action with a specific interpreter. @@ -786,9 +1002,7 @@ class SimpleAction(OpenJDModel_v2023_09): script: DataString args: Optional[ArgListType] = None timeout: Optional[Union[PositiveInt, FormatString]] = None - cancelation: Optional[ - Union[CancelationMethodNotifyThenTerminate, CancelationMethodTerminate] - ] = Field(None, discriminator="mode") + cancelation: Optional[CancelationMethod] = None let: Optional[list[str]] = None # SimpleAction is syntax sugar that resolves to a StepScript (TASK scope), @@ -892,19 +1106,11 @@ class EnvironmentScript(OpenJDModel_v2023_09): f"|{ValueReferenceConstants.WORKING_DIRECTORY.value}", f"|{ValueReferenceConstants.HAS_PATH_MAPPING_RULES.value}", f"|{ValueReferenceConstants.PATH_MAPPING_RULES_FILE.value}", - # WRAP_ACTIONS (RFC 0008) variables, visible inside the wrap hooks. - # First-cut note: injected at the script scope, so they are visible - # to all of this environment's actions rather than only the wrap - # hooks. Precise per-hook scoping (and rejecting WrappedAction.* / - # WrappedEnv.* / WrappedStep.* outside their hook) is deferred and - # belongs with the sessions runtime that produces these variables; - # the conformance scope-restriction cases are runtime `jobs/` tests. - "|WrappedAction.Command", - "|WrappedAction.Args", - "|WrappedAction.Environment", - "|WrappedAction.Timeout", - "|WrappedEnv.Name", - "|WrappedStep.Name", + # The WRAP_ACTIONS (RFC 0008) WrappedAction.* / WrappedEnv.* / + # WrappedStep.* variables are injected per wrap hook via + # EnvironmentActions._template_field_inject, so they are visible + # only within their hook's action — including its creation-scoped + # timeout/cancelation fields for round-trip forwarding. }, ) _template_variable_sources = { @@ -1017,7 +1223,7 @@ class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): chunks: Optional[TaskChunksDefinition] = None -class IntTaskParameterDefinition(OpenJDModel_v2023_09): +class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of an integer-typed Task Parameter and its value range. Attributes: @@ -1041,14 +1247,23 @@ class IntTaskParameterDefinition(OpenJDModel_v2023_09): ) _template_variable_sources = {"__export__": {"__self__"}} - def _get_range_task_param_type(self: Any) -> Type[OpenJDModel]: + def _get_range_task_param_type(self: Any, symtab: SymbolTable) -> Type[OpenJDModel]: if isinstance(self.range, RangeString): + # RFC 0006 typed whole-field resolution: a range that is a single + # whole-field expression evaluating to a list instantiates as a + # literal value list (matching openjd-rs); otherwise the resolved + # string is parsed as a range expression. + from .._internal._create_job import resolve_whole_field_typed_list + + if resolve_whole_field_typed_list(self.range, symtab) is not None: + return RangeListTaskParameterDefinition return RangeExpressionTaskParameterDefinition return RangeListTaskParameterDefinition _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(callable=_get_range_task_param_type), resolve_fields={"range"}, + typed_resolve_fields={"range"}, exclude_fields={"name"}, ) @@ -1078,26 +1293,37 @@ def _validate_range_elements(cls, value: Any) -> Any: # they've all been evaluated if len(value.expressions) == 0: try: - IntRangeExpr.from_str(value) + parsed_range = IntRangeExpr.from_str(value) except Exception as e: raise ValueError(str(e)) + # §3.4: the range may take on at most 1024 values. + if len(parsed_range) > _MAX_TASK_PARAM_RANGE_LEN: + raise ValueError( + f"range expression expands to {len(parsed_range)} elements " + f"(max {_MAX_TASK_PARAM_RANGE_LEN})." + ) + else: + validate_task_param_range_list_len(value) return value def _validate_range_expr_requires_expr(value: Any, info: ValidationInfo) -> Any: """Shared ``range`` field validator for task-parameter definitions: a range expression (a ``RangeString`` format string, RFC 0007) requires the EXPR - extension to be declared. Used by the Float/String/Path task-parameter - definitions so the gate and its message are single-sourced. + extension to be declared, and a list-form range may take on at most 1024 + values (§3.4). Used by the Float/String/Path task-parameter definitions so + the gate and its message are single-sourced. """ if isinstance(value, RangeString): context = cast(Optional[ModelParsingContext], info.context) if context and "EXPR" not in context.extensions: raise ValueError("A range expression (format string) requires the EXPR extension.") + else: + validate_task_param_range_list_len(value) return value -class FloatTaskParameterDefinition(OpenJDModel_v2023_09): +class FloatTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of a float-typed Task Parameter and its value range. Attributes: @@ -1121,6 +1347,7 @@ class FloatTaskParameterDefinition(OpenJDModel_v2023_09): _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, + typed_resolve_fields={"range"}, exclude_fields={"name"}, ) @@ -1141,7 +1368,7 @@ def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: return _validate_range_expr_requires_expr(value, info) -class StringTaskParameterDefinition(OpenJDModel_v2023_09): +class StringTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of a string-typed Task Parameter and its value range. Attributes: @@ -1165,6 +1392,7 @@ class StringTaskParameterDefinition(OpenJDModel_v2023_09): _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, + typed_resolve_fields={"range"}, exclude_fields={"name"}, ) @@ -1174,7 +1402,7 @@ def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: return _validate_range_expr_requires_expr(value, info) -class PathTaskParameterDefinition(OpenJDModel_v2023_09): +class PathTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of a path-typed Task Parameter and its value range. Attributes: @@ -1198,6 +1426,7 @@ class PathTaskParameterDefinition(OpenJDModel_v2023_09): _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, + typed_resolve_fields={"range"}, exclude_fields={"name"}, ) @@ -1206,8 +1435,22 @@ class PathTaskParameterDefinition(OpenJDModel_v2023_09): def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: return _validate_range_expr_requires_expr(value, info) + @field_validator("range") + @classmethod + def _validate_no_empty_path_values(cls, value: Any) -> Any: + # §3.4.2: an empty string is not a valid path on any OS, so a PATH + # task parameter's range may not contain one. Format-string items + # with expressions resolve later; literal empties are rejected here. + if isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, FormatString) and len(item.expressions) > 0: + continue + if str(item) == "": + raise ValueError(f"range[{i}] must not be an empty string.") + return value + -class ChunkIntTaskParameterDefinition(OpenJDModel_v2023_09): +class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of an integer-typed Task Parameter, that is processed as chunks of tasks insteas of as individual tasks when running. @@ -1234,14 +1477,23 @@ class ChunkIntTaskParameterDefinition(OpenJDModel_v2023_09): ) _template_variable_sources = {"__export__": {"__self__"}} - def _get_range_task_param_type(self: Any) -> Type[OpenJDModel]: + def _get_range_task_param_type(self: Any, symtab: SymbolTable) -> Type[OpenJDModel]: if isinstance(self.range, RangeString): + # RFC 0006 typed whole-field resolution: a range that is a single + # whole-field expression evaluating to a list instantiates as a + # literal value list (matching openjd-rs); otherwise the resolved + # string is parsed as a range expression. + from .._internal._create_job import resolve_whole_field_typed_list + + if resolve_whole_field_typed_list(self.range, symtab) is not None: + return RangeListTaskParameterDefinition return RangeExpressionTaskParameterDefinition return RangeListTaskParameterDefinition _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(callable=_get_range_task_param_type), resolve_fields={"range"}, + typed_resolve_fields={"range"}, exclude_fields={"name"}, ) @@ -1284,9 +1536,17 @@ def _validate_range_elements(cls, value: Any) -> Any: # they've all been evaluated if len(value.expressions) == 0: try: - IntRangeExpr.from_str(value) + parsed_range = IntRangeExpr.from_str(value) except Exception as e: raise ValueError(str(e)) + # §3.4: the range may take on at most 1024 values. + if len(parsed_range) > _MAX_TASK_PARAM_RANGE_LEN: + raise ValueError( + f"range expression expands to {len(parsed_range)} elements " + f"(max {_MAX_TASK_PARAM_RANGE_LEN})." + ) + else: + validate_task_param_range_list_len(value) return value @@ -1511,6 +1771,20 @@ class Environment(OpenJDModel_v2023_09): description: Optional[Description] = None _template_variable_scope = ResolutionScope.SESSION + # §7.3.1: an environment's `variables` values are format strings resolved + # at session scope, so the Session.* value references are available to + # them — matching the script's actions (whose injection lives on the + # script model) and the openjd-rs validator. + _template_variable_definitions = DefinesTemplateVariables( + inject={ + f"|{ValueReferenceConstants.WORKING_DIRECTORY.value}", + f"|{ValueReferenceConstants.HAS_PATH_MAPPING_RULES.value}", + f"|{ValueReferenceConstants.PATH_MAPPING_RULES_FILE.value}", + }, + ) + _template_variable_sources = { + "variables": {"__self__"}, + } @field_validator("name") @classmethod @@ -1622,7 +1896,9 @@ class JobStringParameterDefinitionUserInterface(OpenJDModel_v2023_09): groupLabel: Optional[UserInterfaceLabelStringValue] = None -class JobStringParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): +class JobStringParameterDefinition( + NameIdentifierLengthMixin, OpenJDModel_v2023_09, JobParameterInterface +): """A Job Parameter of type string. Attributes: @@ -1861,7 +2137,9 @@ class JobPathParameterDefinitionUserInterface(OpenJDModel_v2023_09): fileFilterDefault: Optional[JobPathParameterDefinitionFileFilter] = None -class JobPathParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): +class JobPathParameterDefinition( + NameIdentifierLengthMixin, OpenJDModel_v2023_09, JobParameterInterface +): """A Job Parameter of type path. Attributes: @@ -2095,7 +2373,7 @@ class JobIntParameterDefinitionUserInterface(OpenJDModel_v2023_09): singleStepDelta: Optional[PositiveInt] = None -class JobIntParameterDefinition(OpenJDModel_v2023_09): +class JobIntParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """A Job Parameter of type integer. Attributes: @@ -2353,7 +2631,7 @@ class JobFloatParameterDefinitionUserInterface(OpenJDModel_v2023_09): singleStepDelta: Optional[PositiveFloat] = None -class JobFloatParameterDefinition(OpenJDModel_v2023_09): +class JobFloatParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """A Job Parameter of type float. Attributes: @@ -2381,6 +2659,25 @@ class JobFloatParameterDefinition(OpenJDModel_v2023_09): allowedValues: Optional[AllowedFloatParameterList] = None # noqa: N815 default: Optional[Decimal] = None + @model_validator(mode="after") + def _validate_decimals_requires_spin_box(self) -> Self: + # §2.4: `decimals` configures the places editable in a SPIN_BOX; it is + # meaningless for DROPDOWN_LIST and HIDDEN controls. The effective + # control defaults to DROPDOWN_LIST when allowedValues is provided and + # SPIN_BOX otherwise, matching openjd-rs's validate_ui rules. + ui = self.userInterface + if ui is not None and ui.decimals is not None: + control = ui.control + if control is None: + control = ( + FloatUserInterfaceControl.DROPDOWN_LIST + if self.allowedValues is not None + else FloatUserInterfaceControl.SPIN_BOX + ) + if control != FloatUserInterfaceControl.SPIN_BOX: + raise ValueError("decimals can only be provided when the control is SPIN_BOX.") + return self + _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), @@ -2833,7 +3130,9 @@ def _validate_attribute_list( # validate when those values are substituted. if isinstance(item, FormatString) and len(item.expressions) > 0: continue - if item not in standard_capability["values"]: + # §3.3.2: attribute values follow capability naming, which is + # case-insensitive, so "LINUX" matches "linux". + if item.lower() not in standard_capability["values"]: raise ValueError( f"Values must be from {' '.join(standard_capability['values'])}" ) @@ -2958,6 +3257,12 @@ class Step(OpenJDModel_v2023_09): parameterSpace: Optional[StepParameterSpace] = None # noqa: N815 hostRequirements: Optional[HostRequirements] = None dependencies: Optional[StepDependenciesList] = None + # RFC 0007 (EXPR): the step-level `let` bindings, preserved from the + # StepTemplate so the runtime can seed them when entering the step's + # environments — a step environment's variables and actions may reference + # them. The step's own script carries a merged copy (step bindings first) + # for the task-run path. + let: Optional[list[str]] = None class StepTemplate(OpenJDModel_v2023_09): @@ -3006,9 +3311,48 @@ class StepTemplate(OpenJDModel_v2023_09): def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: return validate_let_field(v, info) + def _extend_step_symtab(self: Any, symtab: SymbolTable) -> SymbolTable: + """Per-step symbol table for job instantiation, mirroring openjd-rs's + ``instantiate_step``: seeds ``Step.Name`` and evaluates the step-level + EXPR ``let`` bindings (RFC 0007 §3.6) in template scope, so the step's + parameter space, host requirements, and script instantiate against + them. Script-level ``let`` bindings are *not* evaluated here — they + resolve at session time. + + ``Step.Name`` and ``let`` references only pass template validation + with the EXPR extension enabled, so seeding them unconditionally does + not change the behavior of non-EXPR templates. + """ + step_symtab = SymbolTable(source=symtab) + step_symtab["Step.Name"] = str(self.name) + if self.let: + # Deferred import to keep the Rust expr surface off the non-EXPR + # path; `let` fields only exist when EXPR is declared. + from .._format_strings._nodes import ExprNode + + for binding in self.let: + name, sep, rhs = binding.partition("=") + name = name.strip() + rhs = rhs.strip() + if not sep or not name or not rhs: + # Malformed bindings are rejected by the `let` validator. + continue + try: + # evaluate_value keeps the engine's typed value (paths + # stay paths, float rendering fidelity is preserved) when + # the binding is later referenced. + step_symtab[name] = ExprNode(rhs).evaluate_value(symtab=step_symtab) + except ValueError as exc: + raise ValueError(f"let binding {name!r}: {exc}") + return step_symtab + _template_variable_sources = { "script": {"__self__", "parameterSpace"}, "stepEnvironments": {"__self__"}, + # RFC 0007 §3.6: step-level `let` names (defined on __self__) are in + # scope for the step's parameter space and host requirements. + "parameterSpace": {"__self__"}, + "hostRequirements": {"__self__"}, "python": {"__self__", "parameterSpace"}, "bash": {"__self__", "parameterSpace"}, "cmd": {"__self__", "parameterSpace"}, @@ -3017,8 +3361,9 @@ def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: } _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=Step), - exclude_fields={"python", "bash", "cmd", "powershell", "node", "let"}, + exclude_fields={"python", "bash", "cmd", "powershell", "node"}, transform=lambda t: cast("StepTemplate", t).resolve_syntax_sugar(), + extends_symtab=_extend_step_symtab, ) @field_validator("name") @@ -3121,9 +3466,11 @@ def resolve_syntax_sugar(self) -> "StepTemplate": # the Job and the runtime resolves it. The model has already # validated reference/shadowing rules across both scopes at decode. if self.let: + # The step's own `let` is preserved too (Step.let): the + # runtime seeds it when entering the step's environments. merged_let = [*self.let, *(self.script.let or [])] new_script = self.script.model_copy(update={"let": merged_let}) - return self.model_copy(update={"script": new_script, "let": None}) + return self.model_copy(update={"script": new_script}) return self for name, (command, ext, arg_prefix) in _INTERPRETER_MAP.items(): @@ -3178,6 +3525,9 @@ def resolve_syntax_sugar(self) -> "StepTemplate": parameterSpace=self.parameterSpace, hostRequirements=self.hostRequirements, dependencies=self.dependencies, + # Preserved for the runtime to seed when entering the step's + # environments (RFC 0007). + let=self.let, ) @@ -3232,7 +3582,7 @@ def _coerce_bool_value(value: Any) -> bool: raise ValueError("BOOL value must be a boolean, 0/1, 0.0/1.0, or a boolean string.") -class JobBoolParameterDefinition(OpenJDModel_v2023_09): +class JobBoolParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """A Job Parameter of type bool (EXPR extension, RFC 0007). Attributes: @@ -3474,7 +3824,9 @@ def _normalize_parameter_type_case(value: Any, info: ValidationInfo) -> Any: ) -class _JobListParameterDefinitionBase(OpenJDModel_v2023_09, JobParameterInterface): +class _JobListParameterDefinitionBase( + NameIdentifierLengthMixin, OpenJDModel_v2023_09, JobParameterInterface +): """Shared base for the EXPR (RFC 0007) ``LIST[*]`` job-parameter definitions. Collects the fields and machinery common to every list parameter type — the @@ -3607,7 +3959,9 @@ def _check_item(self, item: Any) -> None: _check_int_item(self.name, it, inner_item_c) -class JobRangeExprParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): +class JobRangeExprParameterDefinition( + NameIdentifierLengthMixin, OpenJDModel_v2023_09, JobParameterInterface +): """RANGE_EXPR job parameter (EXPR extension, RFC 0007). The value is an integer range expression string (e.g. ``"1-100:10"``) @@ -3734,10 +4088,13 @@ class JobTemplate(OpenJDModel_v2023_09): schemaStr: Optional[str] = Field(None, alias="$schema") # noqa: N815 _template_variable_scope = ResolutionScope.TEMPLATE + # Job.Name is available to the job's steps and environments, but only + # when the EXPR extension is enabled (RFC 0007 §7.3.1). + _template_variable_definitions = DefinesTemplateVariables(expr_inject={"|Job.Name"}) _template_variable_sources = { "name": {"parameterDefinitions"}, - "steps": {"parameterDefinitions"}, - "jobEnvironments": {"parameterDefinitions"}, + "steps": {"parameterDefinitions", "__self__"}, + "jobEnvironments": {"parameterDefinitions", "__self__"}, } _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=Job), @@ -4012,8 +4369,12 @@ class EnvironmentTemplate(OpenJDModel_v2023_09): environment: Environment _template_variable_scope = ResolutionScope.TEMPLATE + # Job.Name is available within an environment template's environment + # (RFC 0007 §7.3.1) when the EXPR extension is enabled: external + # environments run within a session that always belongs to a job. + _template_variable_definitions = DefinesTemplateVariables(expr_inject={"|Job.Name"}) _template_variable_sources = { - "environment": {"parameterDefinitions"}, + "environment": {"parameterDefinitions", "__self__"}, } @field_validator("extensions") diff --git a/test/openjd/expr/test_operation_limit.py b/test/openjd/expr/test_operation_limit.py index 8c1ae0b3..0ee72590 100644 --- a/test/openjd/expr/test_operation_limit.py +++ b/test/openjd/expr/test_operation_limit.py @@ -163,24 +163,28 @@ def test_list_concat_iterations_count(self) -> None: ) def test_list_multiply_iterations_count(self) -> None: - """List repetition counts the result elements.""" + """List repetition pre-charges the full projected result size (3000 + elements) before the limit check, so the reported count is the whole + charge rather than limit + 1.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("[1, 2, 3] * 1000", operation_limit=100) assert str(exc_info.value) == "".join( [ - op_limit_msg(100), + op_limit_msg(100, count=3001), " [1, 2, 3] * 1000\n", " ~~~~~~~~~~^~~~~~", ] ) def test_flatten_iterations_count(self) -> None: - """flatten() counts outer and inner list elements.""" + """flatten() pre-charges the repeated outer list (1000 elements plus + both operand charges) before iterating, so the reported count is the + whole charge rather than limit + 1.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("flatten([[1,2],[3,4]] * 500)", operation_limit=100) assert str(exc_info.value) == "".join( [ - op_limit_msg(100), + op_limit_msg(100, count=1002), " flatten([[1,2],[3,4]] * 500)\n", " ~~~~~~~~~~~~~~^~~~~", ] @@ -199,24 +203,28 @@ def test_repr_sh_list_iterations_count(self) -> None: ) def test_any_iterations_count(self) -> None: - """any() iterating a list counts the elements.""" + """any()'s argument list is pre-charged at construction (1000 + elements plus operand charges), so the reported count is the whole + charge rather than limit + 1.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("any([False] * 1000)", operation_limit=100) assert str(exc_info.value) == "".join( [ - op_limit_msg(100), + op_limit_msg(100, count=1002), " any([False] * 1000)\n", " ~~~~~~~~^~~~~~", ] ) def test_all_iterations_count(self) -> None: - """all() iterating a list counts the elements.""" + """all()'s argument list is pre-charged at construction (1000 + elements plus operand charges), so the reported count is the whole + charge rather than limit + 1.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("all([True] * 1000)", operation_limit=100) assert str(exc_info.value) == "".join( [ - op_limit_msg(100), + op_limit_msg(100, count=1002), " all([True] * 1000)\n", " ~~~~~~~^~~~~~", ] diff --git a/test/openjd/model_v0/_internal/test_create_job.py b/test/openjd/model_v0/_internal/test_create_job.py index b1e9f471..91f2ca0c 100644 --- a/test/openjd/model_v0/_internal/test_create_job.py +++ b/test/openjd/model_v0/_internal/test_create_job.py @@ -359,7 +359,7 @@ class TargetModel(BaseModelForTesting): class Model(BaseModelForTesting): f: str _job_creation_metadata = JobCreationMetadata( - create_as=JobCreateAsMetadata(callable=lambda m: TargetModel) + create_as=JobCreateAsMetadata(callable=lambda m, symtab: TargetModel) ) model = Model(f="some string") diff --git a/test/openjd/model_v0/v2023_09/test_action.py b/test/openjd/model_v0/v2023_09/test_action.py index e4c150bf..d415f4a0 100644 --- a/test/openjd/model_v0/v2023_09/test_action.py +++ b/test/openjd/model_v0/v2023_09/test_action.py @@ -6,7 +6,7 @@ from pydantic import ValidationError from openjd.model._parse import _parse_model -from openjd.model.v2023_09 import Action, EnvironmentActions, StepActions +from openjd.model.v2023_09 import Action, EnvironmentActions, ModelParsingContext, StepActions class TestAction: @@ -164,7 +164,6 @@ class TestEnvironmentActions: "data", ( pytest.param({"onEnter": {"command": "foo"}}, id="has onEnter"), - pytest.param({"onExit": {"command": "foo"}}, id="has onExit"), # For making sure our pre-validator logic is correct pytest.param( { @@ -188,10 +187,26 @@ def test_parse_success(self, data: dict[str, Any]) -> None: # THEN # no exception was raised. + def test_parse_onexit_only_with_wrap_actions_extension(self) -> None: + # RFC 0008: with the WRAP_ACTIONS extension declared, an environment + # may define any single action without a standalone onEnter. + + # GIVEN + context = ModelParsingContext(supported_extensions=["EXPR", "WRAP_ACTIONS"]) + + # WHEN + _parse_model(model=EnvironmentActions, obj={"onExit": {"command": "foo"}}, context=context) + + # THEN + # no exception was raised. + @pytest.mark.parametrize( "data", ( pytest.param({}, id="empty object"), + # §3.5: base 2023-09 requires onEnter whenever a script is + # present (the WRAP_ACTIONS extension relaxes this). + pytest.param({"onExit": {"command": "foo"}}, id="onExit only"), pytest.param({"onEnter": {"command": "foo"}, "onUnknown": "blah"}, id="unknown field"), ), ) From 1018407ba3e365f3c5dd4df480f73f22a5b3412d Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:33:42 -0700 Subject: [PATCH 02/13] =?UTF-8?q?fix:=20Address=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20CI=20build,=20=C2=A73.4=20parity,=20eval=20caching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes on top of the EXPR/WRAP_ACTIONS parity commit: - Re-comment the local-dev [patch.crates-io] block and regenerate Cargo.lock against crates.io so CI and published wheels resolve the openjd-* crates from the registry (fixes the all-red CI). - Enforce the §3.4 1024-value task-parameter range cap at job creation for expression-driven ranges (typed whole-field LIST resolution and RANGE_EXPR strings), matching openjd-rs's resolve-time checks. Conformance fixtures added to openjd-specifications. - Cache the EXPR engine symbol table and host-context profile on SymbolTable, keyed on a mutation version (versioned expr_types dict, property setters, __ior__; pickle via __reduce__). Removes the per-expression Rust-boundary rebuild: 8.6 -> 1.9 ms per 20-arg action. - Compute RFC 0006 typed whole-field resolution exactly once per field; the create_as callable now receives the typed-values mapping so the target-model decision and field value cannot disagree. - Single-source the whole-field check (FormatString.whole_field_expression), the let-binding evaluation loop (openjd.model.evaluate_let_bindings, parse-memoized, shared with openjd-sessions), and the Int/ChunkInt range helpers; extract _resolve_path_default_2023_09 and _compute_typed_resolutions to reduce branch counts; narrow the typed resolution's error swallow to ValueError. Note: this also makes three spec-conformant validation tightenings from the base commit explicit — onEnter required without WRAP_ACTIONS, the §3.4 range caps, and §7.1 name lengths — all matching openjd-rs; they newly reject previously-accepted templates, so the change set is not purely additive for template acceptance. Tests: 5342 passed (12 new: cache invalidation contract incl. |= / reassignment / pickle, single-evaluation invariant); mypy/black/ruff clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- Cargo.lock | 279 ++++++++++-------- Cargo.toml | 8 +- src/openjd/model/__init__.py | 2 + src/openjd/model/_create_job.py | 92 +++--- .../model/_format_strings/_expr_support.py | 67 ++++- .../model/_format_strings/_format_string.py | 50 +++- src/openjd/model/_format_strings/_nodes.py | 16 +- src/openjd/model/_internal/_create_job.py | 80 +++-- src/openjd/model/_let_bindings.py | 81 +++++ src/openjd/model/_symbol_table.py | 159 ++++++++-- src/openjd/model/_types.py | 15 +- src/openjd/model/v2023_09/_model.py | 162 +++++----- .../model_v0/_internal/test_create_job.py | 2 +- .../model_v0/test_symbol_table_cache.py | 218 ++++++++++++++ 14 files changed, 899 insertions(+), 332 deletions(-) create mode 100644 src/openjd/model/_let_bindings.py create mode 100644 test/openjd/model_v0/test_symbol_table_cache.py diff --git a/Cargo.lock b/Cargo.lock index 5ca22e43..e6e28520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,15 +52,15 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -82,7 +82,7 @@ dependencies = [ "manyhow", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -98,7 +98,7 @@ dependencies = [ "proc-macro2", "quote", "quote-use", - "syn", + "syn 2.0.119", ] [[package]] @@ -115,19 +115,19 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] @@ -138,9 +138,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "castaway" @@ -153,9 +153,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "shlex", @@ -169,9 +169,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chrono" @@ -232,7 +232,7 @@ checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -294,38 +294,38 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", "futures-macro", @@ -342,7 +342,7 @@ checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" dependencies = [ "attribute-derive", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -491,7 +491,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -520,9 +520,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -537,15 +537,15 @@ checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" dependencies = [ "value-bag", ] @@ -559,7 +559,7 @@ dependencies = [ "manyhow-macros", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -581,9 +581,9 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "matrixmultiply" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" dependencies = [ "autocfg", "rawpointer", @@ -591,15 +591,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -641,9 +641,9 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -695,7 +695,7 @@ dependencies = [ "num-traits", "pyo3", "pyo3-build-config", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", ] [[package]] @@ -706,7 +706,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.2.0" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5bbd809c2a32753829af3a133664a19d7bf93830e57267bc79014ce88675a24" dependencies = [ "regex", "regex-syntax", @@ -722,6 +724,8 @@ dependencies = [ [[package]] name = "openjd-model" version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e35e8e61b85a42fe39260f3e7b271c615e63fbb59323185f64bf1f76ac017e" dependencies = [ "indexmap", "openjd-expr", @@ -752,6 +756,8 @@ dependencies = [ [[package]] name = "openjd-sessions" version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b71eba47947c3eb8e74607983297cf431dd0724a2d7ece4d3b8f6be801ba17" dependencies = [ "bitflags", "futures-util", @@ -837,9 +843,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -878,9 +884,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -938,7 +944,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -950,7 +956,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -990,14 +996,14 @@ dependencies = [ "proc-macro2", "quote", "rustpython-parser", - "syn", + "syn 2.0.119", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1021,7 +1027,7 @@ dependencies = [ "proc-macro-utils", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1038,9 +1044,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha", @@ -1074,9 +1080,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1086,9 +1092,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1109,9 +1115,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustpython-ast" @@ -1182,7 +1188,7 @@ dependencies = [ "get-size2", "is-macro", "memchr", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustpython-ruff_python_trivia", "rustpython-ruff_source_file", "rustpython-ruff_text_size", @@ -1200,7 +1206,7 @@ dependencies = [ "compact_str", "get-size2", "memchr", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustpython-ruff_python_ast", "rustpython-ruff_python_trivia", "rustpython-ruff_text_size", @@ -1243,9 +1249,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -1255,9 +1261,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1284,22 +1290,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1313,9 +1319,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1453,9 +1459,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -1470,29 +1487,29 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "time" -version = "0.3.49" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -1518,9 +1535,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -1533,9 +1550,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -1548,20 +1565,20 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", @@ -1572,9 +1589,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ "indexmap", "serde_core", @@ -1605,9 +1622,9 @@ dependencies = [ [[package]] name = "toml_writer" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "typeid" @@ -1712,9 +1729,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.23.3" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -1723,9 +1740,9 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" +checksum = "ef73bfbaf3216cb59c205d7176bee1194e0d84348979da31f4a71fefe3c2054e" dependencies = [ "value-bag-serde1", "value-bag-sval2", @@ -1733,9 +1750,9 @@ dependencies = [ [[package]] name = "value-bag-serde1" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16530907bfe2999a1773ca5900a65101e092c70f642f25cc23ca0c43573262c5" +checksum = "5b92170db3db8a6354f12a5b7f13a5453928433e08fc46aee51eedfa8f7a28a1" dependencies = [ "erased-serde", "serde_core", @@ -1744,9 +1761,9 @@ dependencies = [ [[package]] name = "value-bag-sval2" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00ae130edd690eaa877e4f40605d534790d1cf1d651e7685bd6a144521b251f" +checksum = "0bf9832097ca044466ae3f1aa43a943d4e450b7c3b8cdf65363875df07da79a0" dependencies = [ "sval", "sval_buffer", @@ -1780,9 +1797,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -1793,9 +1810,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1803,31 +1820,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] name = "which" -version = "8.0.4" +version = "8.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d7cd18d4acb58fb3cdfe9ea54e6cd96a4e7d4cc45c56338b236e82dad47248" +checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" dependencies = [ "libc", ] @@ -1885,7 +1902,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1896,7 +1913,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1953,9 +1970,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "wit-bindgen" @@ -1965,26 +1982,26 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 85d76da4..19928848 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" # Cargo only honors [patch] at the workspace root, so it lives here rather # than in rust-bindings/Cargo.toml. Paths are relative to this file. # Re-comment before committing so CI/published wheels resolve from crates.io. -[patch.crates-io] -openjd-expr = { path = "../openjd-rs/crates/openjd-expr" } -openjd-model = { path = "../openjd-rs/crates/openjd-model" } -openjd-sessions = { path = "../openjd-rs/crates/openjd-sessions" } +# [patch.crates-io] +# openjd-expr = { path = "../openjd-rs/crates/openjd-expr" } +# openjd-model = { path = "../openjd-rs/crates/openjd-model" } +# openjd-sessions = { path = "../openjd-rs/crates/openjd-sessions" } diff --git a/src/openjd/model/__init__.py b/src/openjd/model/__init__.py index 6c4fcb79..5e76c1c5 100644 --- a/src/openjd/model/__init__.py +++ b/src/openjd/model/__init__.py @@ -28,6 +28,7 @@ ) from ._step_param_space_iter import StepParameterSpaceIterator from ._format_strings import FormatStringError +from ._let_bindings import evaluate_let_bindings from ._symbol_table import SymbolTable from ._types import ( EnvironmentTemplate, @@ -54,6 +55,7 @@ "decode_environment_template", "decode_job_template", "document_string_to_object", + "evaluate_let_bindings", "merge_job_parameter_definitions", "model_to_object", "parse_model", diff --git a/src/openjd/model/_create_job.py b/src/openjd/model/_create_job.py index acee41c2..d4e0f302 100644 --- a/src/openjd/model/_create_job.py +++ b/src/openjd/model/_create_job.py @@ -76,6 +76,53 @@ def _collect_missing_job_parameter_names( return available_parameters.difference(set(job_parameter_values.keys())) +def _resolve_path_default_2023_09( + param_name: str, + default: str, + *, + job_template_dir: Path, + allow_job_template_dir_walk_up: bool, + allow_uri_path_values: bool, +) -> str: + """Resolve a PATH parameter's default value string. + + - RFC 0006 (EXPR): URI-form defaults ("s3://...") are preserved verbatim — + they are not filesystem paths, so the template-dir join and walk-up + enforcement do not apply. Mirrors openjd-rs's preprocess_job_parameters + URI handling. + - Otherwise the default is made relative to ``job_template_dir``, with the + ``allow_job_template_dir_walk_up`` request enforced. + + Raises: + ValueError: If the default violates the walk-up restrictions. + """ + if default == "": + return default + if allow_uri_path_values and _is_uri(default): + return default + default_path = Path(default) + if default_path.is_absolute(): + # While we could permit absolute paths within the job template dir, + # we choose not to do so. A job template using absolute paths as path defaults + # within the template's directory isn't portable and it's easier to make + # them relative early in the creating a job. + if not allow_job_template_dir_walk_up: + raise ValueError( + f"The default value of PATH parameter {param_name} is an absolute path. Default paths must be relative, and are joined to the job template's directory." + ) + elif job_template_dir.is_absolute(): + # Note: Using os.path.normpath instead of Path.resolve, since + # Path.resolve makes changes to the path unexpected by users, + # like switching Windows drive letters to UNC paths. + default_path = Path(normpath(job_template_dir / default_path)) + if not allow_job_template_dir_walk_up and not default_path.is_relative_to(job_template_dir): + raise ValueError( + f"The default value of PATH parameter {param_name} references a path outside of the template directory. Walking up from the template directory is not permitted." + ) + default = str(default_path) + return default + + def _collect_defaults_2023_09( job_parameter_definitions: list[JobParameterDefinition], job_parameter_values: JobParameterInputValues, @@ -105,45 +152,14 @@ def _collect_defaults_2023_09( ) continue default = str(param.default) - # RFC 0006 (EXPR): URI-form PATH defaults ("s3://...") are - # preserved verbatim — they are not filesystem paths, so the - # template-dir join and walk-up enforcement do not apply. - # Mirrors openjd-rs's preprocess_job_parameters URI handling. - if ( - param.type.name == "PATH" - and allow_uri_path_values - and default != "" - and _is_uri(default) - ): - return_value[param.name] = ParameterValue( - type=ParameterValueType(param.type), value=default + if param.type.name == "PATH": + default = _resolve_path_default_2023_09( + param.name, + default, + job_template_dir=job_template_dir, + allow_job_template_dir_walk_up=allow_job_template_dir_walk_up, + allow_uri_path_values=allow_uri_path_values, ) - continue - # Make PATH defaults relative to job_template_dir, and - # enforce the `allow_job_template_dir_walk_up` parameter request. - if param.type.name == "PATH" and default != "": - default_path = Path(default) - if default_path.is_absolute(): - # While we could permit absolute paths within the job template dir, - # we choose not to do so. A job template using absolute paths as path defaults - # within the template's directory isn't portable and it's easier to make - # them relative early in the creating a job. - if not allow_job_template_dir_walk_up: - raise ValueError( - f"The default value of PATH parameter {param.name} is an absolute path. Default paths must be relative, and are joined to the job template's directory." - ) - elif job_template_dir.is_absolute(): - # Note: Using os.path.normpath instead of Path.resolve, since - # Path.resolve makes changes to the path unexpected by users, - # like switching Windows drive letters to UNC paths. - default_path = Path(normpath(job_template_dir / default_path)) - if not allow_job_template_dir_walk_up and not default_path.is_relative_to( - job_template_dir - ): - raise ValueError( - f"The default value of PATH parameter {param.name} references a path outside of the template directory. Walking up from the template directory is not permitted." - ) - default = str(default_path) return_value[param.name] = ParameterValue( type=ParameterValueType(param.type), value=default ) diff --git a/src/openjd/model/_format_strings/_expr_support.py b/src/openjd/model/_format_strings/_expr_support.py index 4b5838b1..f3abff18 100644 --- a/src/openjd/model/_format_strings/_expr_support.py +++ b/src/openjd/model/_format_strings/_expr_support.py @@ -74,14 +74,79 @@ def symtab_to_expr_values( expects (``"int"``, ``"list[int]"``) via :func:`expr_type_for_openjd_type`. Names whose type has no confident EXPR mapping (e.g. ``RANGE_EXPR``) are omitted so the engine infers them from the value. + + The built engine table is cached on the symbol table, keyed on its + mutation version and the path format: the construction (flattening every + symbol, one type-spec lookup per typed symbol, and the ``build_symbol_table`` + boundary call) is invariant between symbol-table mutations, but a session + action evaluates many expressions (command, each arg, timeout, each + embedded-file ``data``) against one unchanged table — without the cache + the whole table is rebuilt across the Rust boundary per expression. + Caching requires ``types`` to be the symbol table's own ``expr_types`` + (or ``None``), which is what the evaluation layer passes; any other + ``types`` object bypasses the cache. """ + cacheable = types is None or types is getattr(symtab, "expr_types", None) + version = getattr(symtab, "_version", None) + cache_key = ("engine_table", str(path_format)) + if cacheable and version is not None: + cache = symtab._expr_eval_cache + if cache is not None: + hit = cache.get(cache_key) + if hit is not None and hit[0] == version: + return hit[1] + flat = {name: symtab[name] for name in symtab.symbols} expr_types: dict[str, str] = {} for name, openjd_type in (types or {}).items(): spec = expr_type_for_openjd_type(openjd_type) if spec is not None: expr_types[name] = spec - return build_symbol_table(flat, expr_types or None, path_format=path_format) + built = build_symbol_table(flat, expr_types or None, path_format=path_format) + + if cacheable and version is not None: + if symtab._expr_eval_cache is None: + symtab._expr_eval_cache = {} + symtab._expr_eval_cache[cache_key] = (version, built) + return built + + +def profile_for_symtab(symtab: SymbolTable) -> ExprProfile: + """The EXPR evaluation profile for expressions evaluated against + ``symtab``. + + A symbol table carrying host-context path mapping rules (session scope, + ``expr_host_rules``) yields a profile with + ``HostContext.with_rules(...)`` so host-context functions such as + ``apply_path_mapping`` are available and apply those rules — mirroring + openjd-rs's session-scope ``HostContext::WithRules``. ``None`` rules mean + template scope (no host context). + + Cached on the symbol table per mutation version alongside the engine + table: the profile + host-context construction is a per-evaluation Rust + object build otherwise. The rules list itself must not be mutated in + place (assign a new list through ``expr_host_rules`` instead), which is + how the session runtime uses it. + """ + version = getattr(symtab, "_version", None) + cache_key = "host_profile" + if version is not None: + cache = symtab._expr_eval_cache + if cache is not None: + hit = cache.get(cache_key) + if hit is not None and hit[0] == version: + return hit[1] + + profile = ExprProfile.current() + host_rules = getattr(symtab, "expr_host_rules", None) + if host_rules is not None: + profile = profile.with_host_context(HostContext.with_rules(host_rules)) + + if version is not None: + if symtab._expr_eval_cache is None: + symtab._expr_eval_cache = {} + symtab._expr_eval_cache[cache_key] = (version, profile) + return profile def parse_expr_or_raise(expr: str) -> Any: diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index f43244d2..572c966c 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -128,6 +128,27 @@ def resolve(self, *, symtab: SymbolTable, path_format: Optional[object] = None) return "".join(resolved_list) + def whole_field_expression(self) -> Optional["ExpressionInfo"]: + """The format string's single whole-field expression, or ``None``. + + A format string is *whole-field* when it consists of exactly one + ``{{ ... }}`` expression with only whitespace outside the braces + (Template Schemas: fields with ``string?``/typed null semantics). + Single-sourced here so every whole-field check (typed value + resolution, RFC 0006 typed list instantiation) agrees on the rule. + """ + expressions = self.expressions + if len(expressions) != 1: + return None + info = expressions[0] + if info.expression is None: + return None + prefix = self.original_value[: info.start_pos] + suffix = self.original_value[info.end_pos :] + if prefix.strip() or suffix.strip(): + return None + return info + def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = None) -> Any: """Typed resolution of this format string (RFC 0005/0006). @@ -143,23 +164,20 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = Raises: FormatStringError: if the expression cannot be evaluated. """ - expressions = self.expressions - if len(expressions) == 1: - info = expressions[0] + info = self.whole_field_expression() + if info is not None: expression = info.expression - prefix = self.original_value[: info.start_pos] - suffix = self.original_value[info.end_pos :] - if expression is not None and not prefix.strip() and not suffix.strip(): - try: - return expression.evaluate_value(symtab=symtab, path_format=path_format) - except ExpressionError as exc: - raise FormatStringError( - string=self.original_value, - start=info.start_pos, - end=info.end_pos, - expr=expression.expr, - details=str(exc), - ) + assert expression is not None # guaranteed by whole_field_expression + try: + return expression.evaluate_value(symtab=symtab, path_format=path_format) + except ExpressionError as exc: + raise FormatStringError( + string=self.original_value, + start=info.start_pos, + end=info.end_pos, + expr=expression.expr, + details=str(exc), + ) return self.resolve(symtab=symtab, path_format=path_format) def _preprocess( diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 5e29afa9..8e74fdf6 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -228,23 +228,19 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: coercion). Re-raises engine errors as the model's ``ExpressionError``. """ from ._expr_support import ( - ExprProfile, - HostContext, map_eval_error, + profile_for_symtab, symtab_to_expr_values, ) + # Both the engine symbol table and the evaluation profile (which + # carries the session's host-context path mapping rules, if any) are + # cached on the symbol table per mutation version — see + # _expr_support for why (per-expression Rust-boundary rebuilds). values = symtab_to_expr_values( symtab, types=symtab.expr_types or None, path_format=path_format ) - profile = ExprProfile.current() - # A symbol table carrying host-context path mapping rules (session - # scope) enables host-context functions such as apply_path_mapping, - # applying those rules — mirroring openjd-rs's session-scope - # HostContext::WithRules. None means template scope (no host context). - host_rules = getattr(symtab, "expr_host_rules", None) - if host_rules is not None: - profile = profile.with_host_context(HostContext.with_rules(host_rules)) + profile = profile_for_symtab(symtab) try: return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index 59b36d76..f89b4595 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -14,6 +14,18 @@ __all__ = ("instantiate_model", "resolve_whole_field_typed_list") +def _expr_value_is_list(result: Any) -> bool: + """Whether an EXPR engine ``ExprValue`` result carries a list. + + The binding does not (yet) expose an ``is_list`` predicate, so the check + inspects the value's EXPR type string. Kept in exactly one place so the + interface-sniffing has a single seam to replace with a typed binding + predicate later. + """ + result_type = getattr(result, "type", None) + return result_type is not None and str(result_type).startswith("list[") + + def resolve_whole_field_typed_list(value: FormatString, symtab: SymbolTable) -> Any: """RFC 0006 typed whole-field resolution to a native list, or ``None``. @@ -22,31 +34,50 @@ def resolve_whole_field_typed_list(value: FormatString, symtab: SymbolTable) -> native Python list. Any other shape or result — multi-segment format strings, scalar results, or evaluation errors — returns ``None`` so the caller falls back to ordinary string resolution, mirroring the - typed-then-string fallback in openjd-rs's range instantiation. + typed-then-string fallback in openjd-rs's range instantiation. An + evaluation error is deliberately not raised here: the string-resolution + fallback re-evaluates and surfaces it with full format-string context. """ - expressions = value.expressions - if len(expressions) != 1: + info = value.whole_field_expression() + if info is None: return None - info = expressions[0] expression = info.expression - if expression is None: - return None - original = value.original_value - if original[: info.start_pos].strip() or original[info.end_pos :].strip(): + if expression is None: # pragma: no cover - excluded by whole_field_expression return None try: result = expression.evaluate_value(symtab=symtab) - except Exception: # noqa: BLE001 - fall back to string resolution + except ValueError: + # ExpressionError/FormatStringError (both ValueError subclasses): + # fall back to string resolution, which re-raises with context. return None # EXPR-backed evaluation returns the engine's ExprValue; unwrap lists. - result_type = getattr(result, "type", None) - if result_type is not None and str(result_type).startswith("list["): + if _expr_value_is_list(result): return result.item() if isinstance(result, list): return result return None +def _compute_typed_resolutions(model: "OpenJDModel", symtab: SymbolTable) -> dict[str, Any]: + """RFC 0006 typed whole-field resolutions for the model's + ``typed_resolve_fields``, computed exactly once per field. + + The result feeds both the ``create_as`` target-model decision and the + field instantiation in :func:`instantiate_model`, so the (Rust-boundary) + expression evaluation is not repeated and the two consumers cannot + disagree. Fields whose typed resolution does not apply are absent from + the mapping (the caller falls back to normal string resolution). + """ + typed_resolved: dict[str, Any] = {} + for field_name in model._job_creation_metadata.typed_resolve_fields: + field_value = getattr(model, field_name, None) + if isinstance(field_value, FormatString): + typed_value = resolve_whole_field_typed_list(field_value, symtab) + if typed_value is not None: + typed_resolved[field_name] = typed_value + return typed_resolved + + # Cache for TypeAdapter instances _type_adapter_cache: Dict[int, TypeAdapter] = {} @@ -139,6 +170,8 @@ def instantiate_model( # noqa: C901 if model._job_creation_metadata.transform is not None: model = model._job_creation_metadata.transform(model) + typed_resolved = _compute_typed_resolutions(model, symtab) + # Determine the target model to create as target_model = model.__class__ if model._job_creation_metadata.create_as is not None: @@ -146,7 +179,7 @@ def instantiate_model( # noqa: C901 if create_as_metadata.model is not None: target_model = create_as_metadata.model elif create_as_metadata.callable is not None: - target_model = create_as_metadata.callable(model, symtab) + target_model = create_as_metadata.callable(model, typed_resolved) for field_name in model.__class__.model_fields.keys(): if field_name in model._job_creation_metadata.exclude_fields: @@ -167,8 +200,10 @@ def instantiate_model( # noqa: C901 with capture_validation_errors(output_errors=errors, loc=(field_name,), input=field_value): # Instantiate and resolve format string expressions needs_resolve = field_name in model._job_creation_metadata.resolve_fields - typed_resolve = field_name in model._job_creation_metadata.typed_resolve_fields - if isinstance(field_value, list): + if field_name in typed_resolved: + # RFC 0006 typed whole-field resolution (computed above). + instantiated = typed_resolved[field_name] + elif isinstance(field_value, list): if field_name in model._job_creation_metadata.reshape_field_to_dict: key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] instantiated = _instantiate_list_field_as_dict( @@ -181,9 +216,7 @@ def instantiate_model( # noqa: C901 elif isinstance(field_value, dict): instantiated = _instantiate_dict_field(field_value, symtab, needs_resolve) else: - instantiated = _instantiate_noncollection_value( - field_value, symtab, needs_resolve, typed_resolve=typed_resolve - ) + instantiated = _instantiate_noncollection_value(field_value, symtab, needs_resolve) # Validate as the target field type using cached TypeAdapter type_adapter = get_type_adapter(target_field_type) @@ -210,8 +243,6 @@ def _instantiate_noncollection_value( value: Any, symtab: SymbolTable, needs_resolve: bool, - *, - typed_resolve: bool = False, ) -> Any: """Instantiate a single value that must not be a collection type (list, dict, etc). @@ -220,17 +251,14 @@ def _instantiate_noncollection_value( value (Any): Value to process. symtab (SymbolTable): Symbol table for format string value lookups. needs_resolve (bool): Whether to resolve the value as a format string. - typed_resolve (bool): Whether to first attempt RFC 0006 typed - whole-field resolution to a native list (task-parameter ``range`` - fields), falling back to string resolution. + + Note: fields named in ``typed_resolve_fields`` whose RFC 0006 typed + whole-field resolution succeeded never reach this function — + ``instantiate_model`` resolves them once up front. """ if isinstance(value, OpenJDModel): return instantiate_model(value, symtab) elif isinstance(value, FormatString) and needs_resolve: - if typed_resolve: - typed_value = resolve_whole_field_typed_list(value, symtab) - if typed_value is not None: - return typed_value value = value.resolve(symtab=symtab) return value diff --git a/src/openjd/model/_let_bindings.py b/src/openjd/model/_let_bindings.py new file mode 100644 index 00000000..651880b0 --- /dev/null +++ b/src/openjd/model/_let_bindings.py @@ -0,0 +1,81 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Shared evaluation of EXPR ``let`` bindings (RFC 0007 §3.6). + +One implementation of the "parse ``name = expression``, skip malformed, +evaluate, seed the symbol table" loop, used by: + +- the model's job instantiation (a StepTemplate's step-level bindings, + evaluated at create time so the step's parameter space and host + requirements resolve against them), and +- the ``openjd-sessions`` runtime (script-level bindings evaluated by the + runners, and step-level bindings re-applied when entering a step's + environments). + +Keeping it here single-sources the "malformed bindings are rejected by the +``let`` field validator at decode time; skip defensively at evaluation time" +policy, and the RHS parse memoization. +""" + +from functools import lru_cache +from typing import Any, Iterable + +from ._symbol_table import SymbolTable + +__all__ = ["evaluate_let_bindings"] + + +@lru_cache(maxsize=1024) +def _parse_rhs(rhs: str) -> Any: + """Parse a binding's RHS as a standalone EXPR expression. + + Memoized: a template's binding strings are invariant, but bindings are + re-applied on every environment enter/exit and every task run, so caching + the parse avoids re-parsing the same expression through the engine per + application. The returned node holds no symbol-table state — + ``evaluate_value`` takes the symbol table per call — so it is safe to + share across evaluations. Parse errors propagate and are not cached. + """ + # Deferred import to keep the Rust expr surface off the non-EXPR path; + # `let` fields only exist when the EXPR extension is declared. + from ._format_strings._nodes import ExprNode + + return ExprNode(rhs) + + +def evaluate_let_bindings(*, symtab: SymbolTable, let_bindings: Iterable[str]) -> None: + """Evaluate EXPR ``let`` bindings in order, seeding each into ``symtab``. + + ``let_bindings`` is an ordered list of ``"name = expression"`` strings. + Each RHS is evaluated against the symbol table built so far (so later + bindings can reference earlier ones), and the engine's typed result is + stored under the bound name — a let-bound path stays a path for property + access, and float rendering fidelity is preserved — matching the Rust + runtime's natively typed symbol table. + + Malformed bindings (missing ``=``, empty name or expression) are skipped: + the ``let`` field validator rejects them at decode time, so evaluation is + defensive here. + + Not to be confused with ``openjd.model._v1.evaluate_let_bindings`` (the + Rust-backed v1 surface), which has a different signature and returns a + new symbol table; this v0 helper mutates ``symtab`` in place. + + Raises: + ValueError: if a binding's expression fails to parse or evaluate; + the message names the binding. + """ + for binding in let_bindings: + name, sep, rhs = binding.partition("=") + name = name.strip() + rhs = rhs.strip() + if not sep or not name or not rhs: + # Malformed bindings are rejected by the `let` validator. + continue + try: + # evaluate_value keeps the engine's typed value (paths stay + # paths, float rendering fidelity is preserved) when the binding + # is later referenced. + symtab[name] = _parse_rhs(rhs).evaluate_value(symtab=symtab) + except ValueError as exc: + raise ValueError(f"let binding {name!r}: {exc}") diff --git a/src/openjd/model/_symbol_table.py b/src/openjd/model/_symbol_table.py index 7e6d8763..21fdd498 100644 --- a/src/openjd/model/_symbol_table.py +++ b/src/openjd/model/_symbol_table.py @@ -8,18 +8,84 @@ __all__ = ["SymbolTable"] +class _VersionedDict(dict): + """A ``dict`` that bumps its owning :class:`SymbolTable`'s version on + every mutation. + + The symbol table caches the EXPR engine's typed symbol table (an + expensive Rust-boundary construction) keyed on its version; ``expr_types`` + is mutated directly by callers (``symtab.expr_types[k] = v`` / + ``.update(...)``), so those mutations must be observable for the cache to + be sound. + + Mutations bump the version *after* the change lands: a concurrent reader + that snapshots mid-mutation then caches under the pre-bump version, and + the bump immediately supersedes that entry (self-healing), rather than + poisoning the cache under the post-bump version. + """ + + def __init__(self, owner: "SymbolTable", *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + # Backref to the owning SymbolTable; internal to this module. + self._owner = owner + + def __setitem__(self, key: Any, value: Any) -> None: + super().__setitem__(key, value) + self._owner._bump_version() + + def __delitem__(self, key: Any) -> None: + super().__delitem__(key) + self._owner._bump_version() + + def __ior__(self, other: Any) -> "_VersionedDict": # type: ignore[misc,override] + # mypy flags the __ior__/__or__ signature asymmetry inherent to + # augmenting only the in-place operator; `|` (non-mutating) needs no + # version bump and keeps dict's signature. + result = super().__ior__(other) + self._owner._bump_version() + return result + + def update(self, *args: Any, **kwargs: Any) -> None: + super().update(*args, **kwargs) + self._owner._bump_version() + + def setdefault(self, key: Any, default: Any = None) -> Any: + result = super().setdefault(key, default) + self._owner._bump_version() + return result + + def pop(self, *args: Any) -> Any: + result = super().pop(*args) + self._owner._bump_version() + return result + + def popitem(self) -> Any: + result = super().popitem() + self._owner._bump_version() + return result + + def clear(self) -> None: + super().clear() + self._owner._bump_version() + + +def _rebuild_symbol_table( + table: dict, expr_types: dict, expr_host_rules: Optional[list] +) -> "SymbolTable": + """Pickle reconstructor for :class:`SymbolTable` (see ``__reduce__``).""" + symtab = SymbolTable() + symtab._table.update(table) + symtab.expr_types.update(expr_types) + symtab._expr_host_rules = expr_host_rules + return symtab + + class SymbolTable: """ Class used to represent the available symbols that can be used for interpolation in the current context. """ _table: dict[str, Any] - # Optional mapping of symbol name -> OpenJD type name (e.g. "INT", - # "LIST[INT]") for EXPR-typed symbols. Used by the EXPR expression engine to - # coerce stored values to their declared type. Declared as a real field - # (rather than stashed dynamically) so it is preserved across copies and - # unions; empty for non-EXPR symbol tables. - expr_types: dict[str, str] # Optional host-context path mapping rules (``openjd.expr.PathMappingRule`` # values). When set (even to an empty list), EXPR expressions evaluated # against this symbol table run with a host context: host-context @@ -27,7 +93,16 @@ class SymbolTable: # rules — the v0 equivalent of openjd-rs's session-scope # ``HostContext::WithRules``. ``None`` means no host context (template # scope). - expr_host_rules: Optional[list[Any]] + _expr_host_rules: Optional[list[Any]] + # Monotonic mutation counter. Bumped by every mutation of ``_table``, + # ``expr_types``, or ``expr_host_rules`` so that the EXPR evaluation layer + # can cache the (expensive, Rust-boundary) typed engine symbol table and + # host-context profile per symbol-table state. See + # ``_format_strings._expr_support.symtab_to_expr_values``. + _version: int + # Opaque cache slot owned by the EXPR evaluation layer: maps cache keys + # to (version, value) pairs. Never copied to derived tables. + _expr_eval_cache: Optional[dict] def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = None): """Initialize the SymbolTable @@ -37,19 +112,66 @@ def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = Non gets initialized with the contents of the given source. Defaults to None. """ self._table = dict() - self.expr_types = dict() - self.expr_host_rules = None + self._expr_types: dict[str, str] = _VersionedDict(self) + self._expr_host_rules = None + self._version = 0 + self._expr_eval_cache = None if source is not None: if isinstance(source, SymbolTable): self._table.update(source._table) - self.expr_types.update(source.expr_types) - if source.expr_host_rules is not None: - self.expr_host_rules = list(source.expr_host_rules) + self._expr_types.update(source._expr_types) + if source._expr_host_rules is not None: + self._expr_host_rules = list(source._expr_host_rules) elif isinstance(source, dict): self._table.update(source) else: raise TypeError(f"Cannot initialize with type {type(source)}") + def _bump_version(self) -> None: + self._version += 1 + + @property + def expr_types(self) -> dict[str, str]: + """Optional mapping of symbol name -> OpenJD type name (e.g. "INT", + "LIST[INT]") for EXPR-typed symbols. Used by the EXPR expression + engine to coerce stored values to their declared type. Preserved + across copies and unions; empty for non-EXPR symbol tables. + + Mutations (including item assignment and ``update``) are tracked for + the EXPR evaluation cache; assigning a whole new mapping is also + supported. + """ + return self._expr_types + + @expr_types.setter + def expr_types(self, value: dict[str, str]) -> None: + # Rewrap into a versioned dict so subsequent in-place mutations keep + # invalidating the cache, and bump for the reassignment itself. + self._expr_types = _VersionedDict(self, value) + self._bump_version() + + @property + def expr_host_rules(self) -> Optional[list[Any]]: + return self._expr_host_rules + + @expr_host_rules.setter + def expr_host_rules(self, rules: Optional[list[Any]]) -> None: + self._expr_host_rules = rules + self._bump_version() + + def __reduce__(self) -> tuple: + # The versioned dict's owner backref cannot survive plain dict-subclass + # pickling (items are restored through __setitem__ before the instance + # state exists), so serialize plain data and rebuild. + return ( + _rebuild_symbol_table, + ( + dict(self._table), + dict(self._expr_types), + None if self._expr_host_rules is None else list(self._expr_host_rules), + ), + ) + def __repr__(self) -> str: return f"SymbolTable({self._table})" @@ -63,6 +185,7 @@ def __setitem__(self, symbol: str, value: Any) -> None: if not isinstance(symbol, str): raise TypeError("Symbol must be a string") self._table[symbol] = value + self._bump_version() @property def symbols(self) -> _AbstractSet[str]: @@ -84,15 +207,15 @@ def union(self, *symtabs: Union[SymbolTable, dict[str, Any]]) -> SymbolTable: """ retval = SymbolTable() retval._table.update(self._table) - retval.expr_types.update(self.expr_types) - if self.expr_host_rules is not None: - retval.expr_host_rules = list(self.expr_host_rules) + retval.expr_types.update(self._expr_types) + if self._expr_host_rules is not None: + retval._expr_host_rules = list(self._expr_host_rules) for symtab in symtabs: if isinstance(symtab, SymbolTable): retval._table.update(symtab._table) - retval.expr_types.update(symtab.expr_types) - if symtab.expr_host_rules is not None: - retval.expr_host_rules = list(symtab.expr_host_rules) + retval.expr_types.update(symtab._expr_types) + if symtab._expr_host_rules is not None: + retval._expr_host_rules = list(symtab._expr_host_rules) elif isinstance(symtab, dict): retval._table.update(symtab) else: diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index ccbf3ffa..47e6b2f0 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -243,12 +243,14 @@ def __repr__(self) -> str: @dataclass(frozen=True, eq=False, **dataclass_kwargs) class JobCreateAsMetadata: - # Only one of the following may be non-None - # model: Union[Type["OpenJDModel"], Callable[["OpenJDModel", SymbolTable], Type["OpenJDModel"]]] + # Only one of the following may be non-None. model: Optional[Type["OpenJDModel"]] = field(default=None) - callable: Optional[Callable[["OpenJDModel", SymbolTable], Type["OpenJDModel"]]] = field( - default=None - ) + # The callable receives the model and the mapping of typed whole-field + # resolutions (field name -> native value) that instantiate_model computed + # for the model's typed_resolve_fields — the same values the field + # instantiation uses — so target-model selection can depend on a typed + # resolution without re-evaluating the expression. + callable: Optional[Callable[["OpenJDModel", dict], Type["OpenJDModel"]]] = field(default=None) @dataclass(frozen=True, eq=False, **dataclass_kwargs) @@ -277,6 +279,9 @@ class JobCreationMetadata: instantiates to the literal value list, matching openjd-rs. Fields whose typed resolution does not apply (multi-segment format strings, non-list results, or evaluation errors) fall back to normal string resolution. + + ``instantiate_model`` evaluates each such field at most once; the result + is shared with the ``create_as`` callable (see JobCreateAsMetadata). """ create_as: Optional[JobCreateAsMetadata] = field(default=None) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 097b894c..80771079 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1214,6 +1214,16 @@ class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension chunks: Optional[TaskChunksDefinition] = None + @field_validator("range") + @classmethod + def _validate_range_len(cls, value: Any) -> Any: + # §3.4: enforce the range cap on the instantiation target as well, so + # ranges produced by RFC 0006 typed whole-field resolution (e.g. + # `range: "{{Param.Values}}"` with a LIST[*] parameter) are subject to + # the same limit as literal template ranges — matching openjd-rs's + # resolve-time checks in create_job (ranges.rs). + return validate_task_param_range_list_len(value) + class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): # element type of items in the range @@ -1222,6 +1232,69 @@ class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension chunks: Optional[TaskChunksDefinition] = None + @field_validator("range") + @classmethod + def _validate_range_len(cls, value: Any) -> Any: + # §3.4: a range expression that arrives via format-string resolution + # (e.g. `range: "{{RawParam.Frames}}"` with a RANGE_EXPR parameter) is + # only parsed at instantiation, so the expansion cap must be enforced + # here too — matching openjd-rs's resolve-time checks in create_job. + if isinstance(value, IntRangeExpr): + _check_range_expr_len(value) + return value + + +def _check_range_expr_len(parsed_range: IntRangeExpr) -> None: + """§3.4: a range expression may expand to at most 1024 values.""" + if len(parsed_range) > _MAX_TASK_PARAM_RANGE_LEN: + raise ValueError( + f"range expression expands to {len(parsed_range)} elements " + f"(max {_MAX_TASK_PARAM_RANGE_LEN})." + ) + + +def _range_task_param_target(model: Any, typed_values: dict) -> Type[OpenJDModel]: + """``create_as`` target-model selector shared by the INT and CHUNK[INT] + task-parameter definitions. + + RFC 0006 typed whole-field resolution: a ``range`` that is a single + whole-field expression evaluating to a list instantiates as a literal + value list (matching openjd-rs); otherwise the resolved string is parsed + as a range expression. ``typed_values`` holds the typed resolutions that + ``instantiate_model`` computed once for this model's + ``typed_resolve_fields`` — the same values the field instantiation will + use — so the target decision and the field value never disagree, and the + expression is not evaluated a second time here. + """ + if isinstance(model.range, RangeString): + if "range" in typed_values: + return RangeListTaskParameterDefinition + return RangeExpressionTaskParameterDefinition + return RangeListTaskParameterDefinition + + +def _validate_int_range_elements(value: Any) -> Any: + """Shared ``range`` post-validator for the INT and CHUNK[INT] + task-parameter definitions: a literal range-expression string must parse + and may expand to at most 1024 values (§3.4); a list-form range is + length-capped. Ranges containing format expressions defer to the + RangeExpressionTaskParameterDefinition model once they are resolved. + """ + if isinstance(value, FormatString): + # If there are no format expressions, we can validate the range expression. + # otherwise we defer to the RangeExressionTaskParameter model when + # they've all been evaluated + if len(value.expressions) == 0: + try: + parsed_range = IntRangeExpr.from_str(value) + except Exception as e: + raise ValueError(str(e)) + # §3.4: the range may take on at most 1024 values. + _check_range_expr_len(parsed_range) + else: + validate_task_param_range_list_len(value) + return value + class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09): """Definition of an integer-typed Task Parameter and its value range. @@ -1247,21 +1320,8 @@ class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 ) _template_variable_sources = {"__export__": {"__self__"}} - def _get_range_task_param_type(self: Any, symtab: SymbolTable) -> Type[OpenJDModel]: - if isinstance(self.range, RangeString): - # RFC 0006 typed whole-field resolution: a range that is a single - # whole-field expression evaluating to a list instantiates as a - # literal value list (matching openjd-rs); otherwise the resolved - # string is parsed as a range expression. - from .._internal._create_job import resolve_whole_field_typed_list - - if resolve_whole_field_typed_list(self.range, symtab) is not None: - return RangeListTaskParameterDefinition - return RangeExpressionTaskParameterDefinition - return RangeListTaskParameterDefinition - _job_creation_metadata = JobCreationMetadata( - create_as=JobCreateAsMetadata(callable=_get_range_task_param_type), + create_as=JobCreateAsMetadata(callable=_range_task_param_target), resolve_fields={"range"}, typed_resolve_fields={"range"}, exclude_fields={"name"}, @@ -1287,24 +1347,7 @@ def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: @field_validator("range") @classmethod def _validate_range_elements(cls, value: Any) -> Any: - if isinstance(value, FormatString): - # If there are no format expressions, we can validate the range expression. - # otherwise we defer to the RangeExressionTaskParameter model when - # they've all been evaluated - if len(value.expressions) == 0: - try: - parsed_range = IntRangeExpr.from_str(value) - except Exception as e: - raise ValueError(str(e)) - # §3.4: the range may take on at most 1024 values. - if len(parsed_range) > _MAX_TASK_PARAM_RANGE_LEN: - raise ValueError( - f"range expression expands to {len(parsed_range)} elements " - f"(max {_MAX_TASK_PARAM_RANGE_LEN})." - ) - else: - validate_task_param_range_list_len(value) - return value + return _validate_int_range_elements(value) def _validate_range_expr_requires_expr(value: Any, info: ValidationInfo) -> Any: @@ -1477,21 +1520,8 @@ class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v20 ) _template_variable_sources = {"__export__": {"__self__"}} - def _get_range_task_param_type(self: Any, symtab: SymbolTable) -> Type[OpenJDModel]: - if isinstance(self.range, RangeString): - # RFC 0006 typed whole-field resolution: a range that is a single - # whole-field expression evaluating to a list instantiates as a - # literal value list (matching openjd-rs); otherwise the resolved - # string is parsed as a range expression. - from .._internal._create_job import resolve_whole_field_typed_list - - if resolve_whole_field_typed_list(self.range, symtab) is not None: - return RangeListTaskParameterDefinition - return RangeExpressionTaskParameterDefinition - return RangeListTaskParameterDefinition - _job_creation_metadata = JobCreationMetadata( - create_as=JobCreateAsMetadata(callable=_get_range_task_param_type), + create_as=JobCreateAsMetadata(callable=_range_task_param_target), resolve_fields={"range"}, typed_resolve_fields={"range"}, exclude_fields={"name"}, @@ -1530,24 +1560,7 @@ def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: @field_validator("range") @classmethod def _validate_range_elements(cls, value: Any) -> Any: - if isinstance(value, FormatString): - # If there are no format expressions, we can validate the range expression. - # otherwise we defer to the RangeExressionTaskParameter model when - # they've all been evaluated - if len(value.expressions) == 0: - try: - parsed_range = IntRangeExpr.from_str(value) - except Exception as e: - raise ValueError(str(e)) - # §3.4: the range may take on at most 1024 values. - if len(parsed_range) > _MAX_TASK_PARAM_RANGE_LEN: - raise ValueError( - f"range expression expands to {len(parsed_range)} elements " - f"(max {_MAX_TASK_PARAM_RANGE_LEN})." - ) - else: - validate_task_param_range_list_len(value) - return value + return _validate_int_range_elements(value) TaskParameterDefinition = Union[ @@ -3326,24 +3339,9 @@ def _extend_step_symtab(self: Any, symtab: SymbolTable) -> SymbolTable: step_symtab = SymbolTable(source=symtab) step_symtab["Step.Name"] = str(self.name) if self.let: - # Deferred import to keep the Rust expr surface off the non-EXPR - # path; `let` fields only exist when EXPR is declared. - from .._format_strings._nodes import ExprNode - - for binding in self.let: - name, sep, rhs = binding.partition("=") - name = name.strip() - rhs = rhs.strip() - if not sep or not name or not rhs: - # Malformed bindings are rejected by the `let` validator. - continue - try: - # evaluate_value keeps the engine's typed value (paths - # stay paths, float rendering fidelity is preserved) when - # the binding is later referenced. - step_symtab[name] = ExprNode(rhs).evaluate_value(symtab=step_symtab) - except ValueError as exc: - raise ValueError(f"let binding {name!r}: {exc}") + from .._let_bindings import evaluate_let_bindings + + evaluate_let_bindings(symtab=step_symtab, let_bindings=self.let) return step_symtab _template_variable_sources = { diff --git a/test/openjd/model_v0/_internal/test_create_job.py b/test/openjd/model_v0/_internal/test_create_job.py index 91f2ca0c..5b5571e7 100644 --- a/test/openjd/model_v0/_internal/test_create_job.py +++ b/test/openjd/model_v0/_internal/test_create_job.py @@ -359,7 +359,7 @@ class TargetModel(BaseModelForTesting): class Model(BaseModelForTesting): f: str _job_creation_metadata = JobCreationMetadata( - create_as=JobCreateAsMetadata(callable=lambda m, symtab: TargetModel) + create_as=JobCreateAsMetadata(callable=lambda m, typed_values: TargetModel) ) model = Model(f="some string") diff --git a/test/openjd/model_v0/test_symbol_table_cache.py b/test/openjd/model_v0/test_symbol_table_cache.py new file mode 100644 index 00000000..d4d63ad8 --- /dev/null +++ b/test/openjd/model_v0/test_symbol_table_cache.py @@ -0,0 +1,218 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for the EXPR evaluation cache on SymbolTable. + +The engine symbol table (a Rust-boundary construction) and the evaluation +profile are cached per symbol-table mutation version so a session action +that resolves many expressions (command, args, timeout, embedded files) +against one unchanged table does not rebuild the engine table per +expression — and so any mutation (``__setitem__``, ``expr_types``, +``expr_host_rules``) invalidates the cache. +""" + +import pytest + +from openjd.model import SymbolTable +from openjd.model._format_strings._expr_support import ( + profile_for_symtab, + symtab_to_expr_values, +) + + +class TestSymtabEngineTableCache: + def test_repeated_builds_are_cached(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["Param.X"] = "10" + symtab.expr_types["Param.X"] = "INT" + + # WHEN + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + + # THEN: the exact same engine table object is returned. + assert first is second + + def test_setitem_invalidates(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=None) + + # WHEN + symtab["Param.Y"] = "20" + second = symtab_to_expr_values(symtab, types=None) + + # THEN + assert first is not second + + def test_expr_types_mutation_invalidates(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + + # WHEN: a direct expr_types mutation (how create_job and the session + # runtime record types) must be observed by the cache. + symtab.expr_types["Param.X"] = "INT" + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + + # THEN + assert first is not second + + def test_expr_types_update_invalidates(self) -> None: + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + symtab.expr_types.update({"Param.X": "INT"}) + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert first is not second + + def test_foreign_types_mapping_bypasses_cache(self) -> None: + # A caller-supplied types mapping that is not the table's own + # expr_types must not poison or consult the cache. + symtab = SymbolTable() + symtab["Param.X"] = "10" + cached = symtab_to_expr_values(symtab, types=None) + foreign = symtab_to_expr_values(symtab, types={"Param.X": "INT"}) + assert cached is not foreign + # And the cached entry is still served for the standard call form. + assert symtab_to_expr_values(symtab, types=None) is cached + + def test_path_format_is_part_of_the_key(self) -> None: + from openjd.expr import PathFormat + + symtab = SymbolTable() + symtab["Param.P"] = "/tmp/x" + posix = symtab_to_expr_values(symtab, types=None, path_format=PathFormat.POSIX) + windows = symtab_to_expr_values(symtab, types=None, path_format=PathFormat.WINDOWS) + assert posix is not windows + assert symtab_to_expr_values(symtab, types=None, path_format=PathFormat.POSIX) is posix + + def test_expr_types_ior_invalidates(self) -> None: + # `symtab.expr_types |= {...}` goes through dict.__ior__, which the + # versioned dict must intercept (a C-level merge would silently skip + # the version bump and serve a stale engine table). + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + symtab.expr_types |= {"Param.X": "INT"} + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert first is not second + + def test_expr_types_reassignment_invalidates_and_stays_tracked(self) -> None: + # Reassigning a whole new mapping must invalidate, and subsequent + # in-place mutations of the new mapping must keep invalidating. + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + symtab.expr_types = {"Param.X": "INT"} + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert first is not second + symtab.expr_types["Param.Y"] = "FLOAT" + third = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert second is not third + + def test_pickle_round_trip(self) -> None: + # The versioned dict's owner backref cannot survive plain + # dict-subclass pickling; SymbolTable.__reduce__ rebuilds instead. + import pickle + + symtab = SymbolTable() + symtab["Param.X"] = "10" + symtab.expr_types["Param.X"] = "INT" + restored = pickle.loads(pickle.dumps(symtab)) + assert restored["Param.X"] == "10" + assert restored.expr_types == {"Param.X": "INT"} + # The restored table's cache tracking works. + first = symtab_to_expr_values(restored, types=restored.expr_types or None) + assert symtab_to_expr_values(restored, types=restored.expr_types or None) is first + restored["Param.Y"] = "20" + assert symtab_to_expr_values(restored, types=restored.expr_types or None) is not first + + def test_copies_do_not_share_cache(self) -> None: + symtab = SymbolTable() + symtab["Param.X"] = "10" + original = symtab_to_expr_values(symtab, types=None) + derived = SymbolTable(source=symtab) + # The derived table builds its own engine table... + assert symtab_to_expr_values(derived, types=None) is not original + # ...and the original's cache entry is untouched. + assert symtab_to_expr_values(symtab, types=None) is original + + +class TestProfileCache: + def test_profile_cached_and_invalidated_by_host_rules(self) -> None: + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = profile_for_symtab(symtab) + assert profile_for_symtab(symtab) is first + + # Assigning host rules (how the session runtime enables the host + # context) must invalidate the cached profile. + symtab.expr_host_rules = [] + second = profile_for_symtab(symtab) + assert second is not first + assert profile_for_symtab(symtab) is second + + +class TestTypedResolutionSingleEvaluation: + def test_range_expression_evaluated_once_per_definition( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """RFC 0006 typed whole-field range resolution: the expression is + evaluated exactly once per task-parameter definition — the create_as + target-model decision and the field value share the result.""" + from pathlib import Path + + from openjd.model import create_job, decode_job_template, preprocess_job_parameters + from openjd.model._internal import _create_job as internal_create_job + + calls: list[str] = [] + real = internal_create_job.resolve_whole_field_typed_list + + def counting(value, symtab): # type: ignore[no-untyped-def] + calls.append(value.original_value) + return real(value, symtab) + + monkeypatch.setattr(internal_create_job, "resolve_whole_field_typed_list", counting) + + template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": [ + {"name": "Values", "type": "LIST[INT]", "default": [1, 2, 3]} + ], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "V", "type": "INT", "range": "{{Param.Values}}"} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + }, + supported_extensions=["EXPR"], + ) + params = preprocess_job_parameters( + job_template=template, + job_parameter_values={}, + job_template_dir=Path("/tmp"), + current_working_dir=Path("/tmp"), + ) + job = create_job(job_template=template, job_parameter_values=params) + + # THEN: exactly one evaluation of the range expression... + assert calls == ["{{Param.Values}}"] + # ...and it instantiated as the native list. + steps = job.steps + step = steps["S"] if isinstance(steps, dict) else steps[0] + assert step.parameterSpace is not None + tpd = step.parameterSpace.taskParameterDefinitions + tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] + assert list(tp.range) == [1, 2, 3] From cd1b3f12ab64a291e2cef7773ec8225c7791b1ce Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:18:19 -0700 Subject: [PATCH 03/13] fix: Pin openjd-* crates to the published releases with DeferredMode CI resolves openjd-expr/model/sessions from crates.io (the local [patch.crates-io] override stays commented out), but the previous pins (expr 0.2.0, model 0.3.1, sessions 0.3.2) predate the CancelationMode::DeferredMode surface and the FormatString-typed embedded-file filename the bindings use, failing every build with E0599/E0308. Bump to the published releases that carry that surface: - openjd-expr 0.2.1 - openjd-model 0.4.0 - openjd-sessions 0.4.0 Cargo.lock is regenerated against the registry (source + checksum entries verified), and THIRD-PARTY-LICENSES.txt is regenerated via scripts/check_third_party_licenses.sh --update so the lock-driven license check passes; it was also stale for earlier transitive lock updates (tinyvec, futures, wasm-bindgen, granit-parser, ...). No binding code changes were needed: the crate compiles cleanly against the published API (fmt, clippy -D warnings, test, doc all green), the full Python suite passes (5342 passed, 24 skipped, 3 xfailed), and the sessions-repo let-bindings/wrap smoke tests pass (36 passed). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- Cargo.lock | 8 ++-- THIRD-PARTY-LICENSES.txt | 93 ++++++++++++++++++++-------------------- rust-bindings/Cargo.toml | 6 +-- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6e28520..77c263bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -723,9 +723,9 @@ dependencies = [ [[package]] name = "openjd-model" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e35e8e61b85a42fe39260f3e7b271c615e63fbb59323185f64bf1f76ac017e" +checksum = "708aae249e4894aad4effb1ff72ed4d0aefe4b2b4481971901c31c388b155967" dependencies = [ "indexmap", "openjd-expr", @@ -755,9 +755,9 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b71eba47947c3eb8e74607983297cf431dd0724a2d7ece4d3b8f6be801ba17" +checksum = "0f3575a272f79ca47691fb4b7b6fa4d704b452617483e542f0a90cf4efbdb281" dependencies = [ "bitflags", "futures-util", diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index fdc85ee8..6413aae8 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -611,7 +611,7 @@ PERFORMANCE OF THIS SOFTWARE. ** encoding_rs; version 0.8.35 -- https://crates.io/crates/encoding_rs ** nohash-hasher; version 0.2.0 -- https://crates.io/crates/nohash-hasher ** static_assertions; version 1.1.0 -- https://crates.io/crates/static_assertions -** tinyvec; version 1.11.0 -- https://crates.io/crates/tinyvec +** tinyvec; version 1.12.0 -- https://crates.io/crates/tinyvec Apache License Version 2.0, January 2004 @@ -1447,7 +1447,7 @@ PERFORMANCE OF THIS SOFTWARE. ------ -** zerocopy; version 0.8.52 -- https://crates.io/crates/zerocopy +** zerocopy; version 0.8.55 -- https://crates.io/crates/zerocopy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -1860,11 +1860,11 @@ PERFORMANCE OF THIS SOFTWARE. ------ -** futures-core; version 0.3.32 -- https://crates.io/crates/futures-core -** futures-macro; version 0.3.32 -- https://crates.io/crates/futures-macro -** futures-sink; version 0.3.32 -- https://crates.io/crates/futures-sink -** futures-task; version 0.3.32 -- https://crates.io/crates/futures-task -** futures-util; version 0.3.32 -- https://crates.io/crates/futures-util +** futures-core; version 0.3.33 -- https://crates.io/crates/futures-core +** futures-macro; version 0.3.33 -- https://crates.io/crates/futures-macro +** futures-sink; version 0.3.33 -- https://crates.io/crates/futures-sink +** futures-task; version 0.3.33 -- https://crates.io/crates/futures-task +** futures-util; version 0.3.33 -- https://crates.io/crates/futures-util Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -2071,10 +2071,10 @@ limitations under the License. ------ ** ahash; version 0.8.12 -- https://crates.io/crates/ahash -** arc-swap; version 1.9.1 -- https://crates.io/crates/arc-swap +** arc-swap; version 1.9.2 -- https://crates.io/crates/arc-swap ** base64; version 0.22.1 -- https://crates.io/crates/base64 -** bitflags; version 2.13.0 -- https://crates.io/crates/bitflags -** bstr; version 1.12.1 -- https://crates.io/crates/bstr +** bitflags; version 2.13.1 -- https://crates.io/crates/bitflags +** bstr; version 1.13.0 -- https://crates.io/crates/bstr ** bumpalo; version 3.20.3 -- https://crates.io/crates/bumpalo ** cfg-if; version 1.0.4 -- https://crates.io/crates/cfg-if ** either; version 1.16.0 -- https://crates.io/crates/either @@ -2086,28 +2086,28 @@ limitations under the License. ** heck; version 0.5.0 -- https://crates.io/crates/heck ** indexmap; version 2.14.0 -- https://crates.io/crates/indexmap ** itertools; version 0.14.0 -- https://crates.io/crates/itertools -** js-sys; version 0.3.102 -- https://crates.io/crates/js-sys -** log; version 0.4.32 -- https://crates.io/crates/log +** js-sys; version 0.3.103 -- https://crates.io/crates/js-sys +** log; version 0.4.33 -- https://crates.io/crates/log ** num-traits; version 0.2.19 -- https://crates.io/crates/num-traits ** once_cell; version 1.21.4 -- https://crates.io/crates/once_cell ** ordermap; version 1.2.0 -- https://crates.io/crates/ordermap ** pyo3-log; version 0.13.4 -- https://crates.io/crates/pyo3-log -** regex-automata; version 0.4.14 -- https://crates.io/crates/regex-automata +** regex-automata; version 0.4.16 -- https://crates.io/crates/regex-automata ** regex-syntax; version 0.8.11 -- https://crates.io/crates/regex-syntax -** regex; version 1.12.4 -- https://crates.io/crates/regex +** regex; version 1.13.1 -- https://crates.io/crates/regex ** signal-hook-registry; version 1.4.8 -- https://crates.io/crates/signal-hook-registry ** smallvec; version 1.15.2 -- https://crates.io/crates/smallvec ** unicode-normalization; version 0.1.25 -- https://crates.io/crates/unicode-normalization ** unicode-width; version 0.2.2 -- https://crates.io/crates/unicode-width ** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 -** uuid; version 1.23.3 -- https://crates.io/crates/uuid -** value-bag; version 1.12.0 -- https://crates.io/crates/value-bag +** uuid; version 1.24.0 -- https://crates.io/crates/uuid +** value-bag; version 1.13.1 -- https://crates.io/crates/value-bag ** wasi; version 0.11.1+wasi-snapshot-preview1 -- https://crates.io/crates/wasi ** wasip2; version 1.0.4+wasi-0.2.12 -- https://crates.io/crates/wasip2 -** wasm-bindgen-macro-support; version 0.2.125 -- https://crates.io/crates/wasm-bindgen-macro-support -** wasm-bindgen-macro; version 0.2.125 -- https://crates.io/crates/wasm-bindgen-macro -** wasm-bindgen-shared; version 0.2.125 -- https://crates.io/crates/wasm-bindgen-shared -** wasm-bindgen; version 0.2.125 -- https://crates.io/crates/wasm-bindgen +** wasm-bindgen-macro-support; version 0.2.126 -- https://crates.io/crates/wasm-bindgen-macro-support +** wasm-bindgen-macro; version 0.2.126 -- https://crates.io/crates/wasm-bindgen-macro +** wasm-bindgen-shared; version 0.2.126 -- https://crates.io/crates/wasm-bindgen-shared +** wasm-bindgen; version 0.2.126 -- https://crates.io/crates/wasm-bindgen ** wit-bindgen; version 0.57.1 -- https://crates.io/crates/wit-bindgen Apache License Version 2.0, January 2004 @@ -2522,36 +2522,37 @@ limitations under the License. ** arraydeque; version 0.5.1 -- https://crates.io/crates/arraydeque ** get-size-derive2; version 0.7.4 -- https://crates.io/crates/get-size-derive2 ** get-size2; version 0.7.4 -- https://crates.io/crates/get-size2 -** granit-parser; version 0.0.3 -- https://crates.io/crates/granit-parser +** granit-parser; version 0.0.7 -- https://crates.io/crates/granit-parser ** itoa; version 1.0.18 -- https://crates.io/crates/itoa -** libc; version 0.2.186 -- https://crates.io/crates/libc +** libc; version 0.2.189 -- https://crates.io/crates/libc ** manyhow-macros; version 0.11.4 -- https://crates.io/crates/manyhow-macros -** openjd-expr; version 0.1.2 -- https://crates.io/crates/openjd-expr -** openjd-model; version 0.3.0 -- https://crates.io/crates/openjd-model -** openjd-sessions; version 0.3.1 -- https://crates.io/crates/openjd-sessions +** openjd-expr; version 0.2.1 -- https://crates.io/crates/openjd-expr +** openjd-model; version 0.4.0 -- https://crates.io/crates/openjd-model +** openjd-sessions; version 0.4.0 -- https://crates.io/crates/openjd-sessions ** pin-project-lite; version 0.2.17 -- https://crates.io/crates/pin-project-lite -** portable-atomic; version 1.13.1 -- https://crates.io/crates/portable-atomic -** proc-macro2; version 1.0.106 -- https://crates.io/crates/proc-macro2 +** portable-atomic; version 1.14.0 -- https://crates.io/crates/portable-atomic +** proc-macro2; version 1.0.107 -- https://crates.io/crates/proc-macro2 ** pyo3-ffi; version 0.29.0 -- https://crates.io/crates/pyo3-ffi ** pyo3-macros-backend; version 0.29.0 -- https://crates.io/crates/pyo3-macros-backend ** pyo3-macros; version 0.29.0 -- https://crates.io/crates/pyo3-macros ** pyo3; version 0.29.0 -- https://crates.io/crates/pyo3 -** quote; version 1.0.45 -- https://crates.io/crates/quote +** quote; version 1.0.47 -- https://crates.io/crates/quote ** r-efi; version 5.3.0 -- https://crates.io/crates/r-efi ** r-efi; version 6.0.0 -- https://crates.io/crates/r-efi -** rustc-hash; version 2.1.2 -- https://crates.io/crates/rustc-hash -** rustversion; version 1.0.22 -- https://crates.io/crates/rustversion +** rustc-hash; version 2.1.3 -- https://crates.io/crates/rustc-hash +** rustversion; version 1.0.23 -- https://crates.io/crates/rustversion ** ryu; version 1.0.23 -- https://crates.io/crates/ryu -** serde-saphyr; version 0.0.27 -- https://crates.io/crates/serde-saphyr -** serde; version 1.0.228 -- https://crates.io/crates/serde -** serde_core; version 1.0.228 -- https://crates.io/crates/serde_core -** serde_derive; version 1.0.228 -- https://crates.io/crates/serde_derive -** serde_json; version 1.0.150 -- https://crates.io/crates/serde_json +** serde-saphyr; version 0.0.29 -- https://crates.io/crates/serde-saphyr +** serde; version 1.0.229 -- https://crates.io/crates/serde +** serde_core; version 1.0.229 -- https://crates.io/crates/serde_core +** serde_derive; version 1.0.229 -- https://crates.io/crates/serde_derive +** serde_json; version 1.0.151 -- https://crates.io/crates/serde_json ** shlex; version 2.0.1 -- https://crates.io/crates/shlex ** siphasher; version 1.0.3 -- https://crates.io/crates/siphasher -** syn; version 2.0.118 -- https://crates.io/crates/syn -** thiserror-impl; version 2.0.18 -- https://crates.io/crates/thiserror-impl -** thiserror; version 2.0.18 -- https://crates.io/crates/thiserror +** syn; version 2.0.119 -- https://crates.io/crates/syn +** syn; version 3.0.3 -- https://crates.io/crates/syn +** thiserror-impl; version 2.0.19 -- https://crates.io/crates/thiserror-impl +** thiserror; version 2.0.19 -- https://crates.io/crates/thiserror ** unicode-ident; version 1.0.24 -- https://crates.io/crates/unicode-ident Apache License Version 2.0, January 2004 @@ -2659,7 +2660,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------ -** mio; version 1.2.1 -- https://crates.io/crates/mio +** mio; version 1.2.2 -- https://crates.io/crates/mio Copyright (c) 2014 Carl Lerche and other MIO contributors Permission is hereby granted, free of charge, to any person obtaining a copy @@ -2682,7 +2683,7 @@ THE SOFTWARE. ------ -** which; version 8.0.4 -- https://crates.io/crates/which +** which; version 8.0.5 -- https://crates.io/crates/which Copyright (c) 2015 fangyuanziti Permission is hereby granted, free of charge, to any person obtaining a copy @@ -2705,7 +2706,7 @@ THE SOFTWARE. ------ -** bytes; version 1.11.1 -- https://crates.io/crates/bytes +** bytes; version 1.12.1 -- https://crates.io/crates/bytes Copyright (c) 2018 Carl Lerche Permission is hereby granted, free of charge, to any @@ -2792,7 +2793,7 @@ DEALINGS IN THE SOFTWARE. ------ -** tokio-macros; version 2.7.0 -- https://crates.io/crates/tokio-macros +** tokio-macros; version 2.7.1 -- https://crates.io/crates/tokio-macros MIT License Copyright (c) 2019 Yoshua Wuyts @@ -2921,8 +2922,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. ------ -** tokio-util; version 0.7.18 -- https://crates.io/crates/tokio-util -** tokio; version 1.52.3 -- https://crates.io/crates/tokio +** tokio-util; version 0.7.19 -- https://crates.io/crates/tokio-util +** tokio; version 1.53.1 -- https://crates.io/crates/tokio MIT License Copyright (c) Tokio Contributors @@ -2947,7 +2948,7 @@ SOFTWARE. ------ -** zmij; version 1.0.21 -- https://crates.io/crates/zmij +** zmij; version 1.0.23 -- https://crates.io/crates/zmij Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -3000,7 +3001,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------ ** aho-corasick; version 1.1.4 -- https://crates.io/crates/aho-corasick -** memchr; version 2.8.2 -- https://crates.io/crates/memchr +** memchr; version 2.8.3 -- https://crates.io/crates/memchr The MIT License (MIT) Copyright (c) 2015 Andrew Gallant diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 1c514322..08c61f17 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.2.0" -openjd-model = "0.3.1" -openjd-sessions = "0.3.2" +openjd-expr = "0.2.1" +openjd-model = "0.4.0" +openjd-sessions = "0.4.0" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" From 6edc75acbffa347843a0ce68baeaa061c6855dfd Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:44:27 -0700 Subject: [PATCH 04/13] test: Port deferred-cancelation and None-rendering tests from #313 Port the test-only additions of PR #313: None-value rendering in format strings and nodes (RFC 0005 null rendering) and the TestCancelationRoundTripForwarding cases for wrap-hook cancelation forwarding (RFC 0008). These pin behavior whose production code #318 already carries, so #313 can close as superseded. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../format_strings/test_format_string.py | 21 ++++ .../model_v0/format_strings/test_node.py | 30 +++++ .../model_v0/v2023_09/test_wrap_actions.py | 105 ++++++++++++++++++ 3 files changed, 156 insertions(+) diff --git a/test/openjd/model_v0/format_strings/test_format_string.py b/test/openjd/model_v0/format_strings/test_format_string.py index a65b6994..0e3c45d3 100644 --- a/test/openjd/model_v0/format_strings/test_format_string.py +++ b/test/openjd/model_v0/format_strings/test_format_string.py @@ -108,6 +108,27 @@ def test_multiple_expressions( # THEN assert format_string.resolve(symtab=symtab) == expected + @pytest.mark.parametrize( + "input, expected", + [ + pytest.param("MODE=<{{Test.val}}>", "MODE=<>", id="none-only"), + pytest.param("{{Test.val}}", "", id="whole-string-none"), + ], + ) + def test_none_value_renders_as_empty(self, input: str, expected: str) -> None: + # A None value interpolates as the empty string, matching the EXPR + # engine's null rendering (RFC 0005). Relevant for the nullable + # WrappedAction.Cancelation.* injected symbols (RFC 0008 follow-up). + # GIVEN + symtab = SymbolTable() + + # WHEN + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) + symtab["Test.val"] = None + + # THEN + assert format_string.resolve(symtab=symtab) == expected + def test_without_entry_in_table(self): # GIVEN input = " {{ Test.val }}-{{ Test.end}} " diff --git a/test/openjd/model_v0/format_strings/test_node.py b/test/openjd/model_v0/format_strings/test_node.py index adc34045..da0c8d51 100644 --- a/test/openjd/model_v0/format_strings/test_node.py +++ b/test/openjd/model_v0/format_strings/test_node.py @@ -39,3 +39,33 @@ def test_repr(self): # THEN assert str(node) == "FullName(Test.Name)" + + def test_evaluate_to_str_renders_none_as_empty(self): + # A None value interpolates as the empty string, matching the EXPR + # engine's null rendering (RFC 0005). Relevant for nullable injected + # symbols such as WrappedAction.Cancelation.Mode (string?) and + # WrappedAction.Cancelation.NotifyPeriodInSeconds (int?), which are + # None when the wrapped action defines no + # (RFC 0008 follow-up). + # GIVEN + symtab = SymbolTable() + symtab["Test.Name"] = None + node = FullNameNode("Test.Name") + + # WHEN + result = node.evaluate_to_str(symtab=symtab) + + # THEN + assert result == "" + + def test_evaluate_to_str_coerces_value_with_str(self): + # GIVEN + symtab = SymbolTable() + symtab["Test.Name"] = 45 + node = FullNameNode("Test.Name") + + # WHEN + result = node.evaluate_to_str(symtab=symtab) + + # THEN + assert result == "45" diff --git a/test/openjd/model_v0/v2023_09/test_wrap_actions.py b/test/openjd/model_v0/v2023_09/test_wrap_actions.py index f45ea4e4..90cc97b4 100644 --- a/test/openjd/model_v0/v2023_09/test_wrap_actions.py +++ b/test/openjd/model_v0/v2023_09/test_wrap_actions.py @@ -294,3 +294,108 @@ def test_create_job_with_step_env_wrap_succeeds(self): job = create_job(job_template=jt, job_parameter_values={}) step_env_actions = job.steps[0].stepEnvironments[0].script.actions assert step_env_actions.onWrapTaskRun is not None + + +class TestCancelationRoundTripForwarding: + """Cancelation round-trip forwarding (openjd-rs PR #261 review, + discussion r3597453516). A wrap hook must be able to forward the + wrapped action's cancelation verbatim via whole-field expressions:: + + cancelation: + mode: "{{WrappedAction.Cancelation.Mode}}" + notifyPeriodInSeconds: "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}" + + Under RFC 0005 type-forwarding, a single outer ``{{...}}`` forwards + the expression's type into the field: a string mode behaves like the + literal, and a null result means the field is omitted (a null mode + drops the whole cancelation object; a null period falls back to the + schema default). + + Mirrors ``cancelation_round_trip_*`` in openjd-rs + ``crates/openjd-model/tests/integration/test_wrap_actions.rs``. + + Note: the review example also forwards ``timeout:``; that is + deliberately out of scope pending the WrappedAction.Timeout + int-vs-int? question (the 0-when-unset sentinel is not a valid + ````). + """ + + _ROUND_TRIP_EXTS = ["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"] + + def _wrap_actions(self, cancelation: dict) -> dict: + actions = { + "onWrapEnvEnter": _cmd("{{WrappedAction.Command}}"), + "onWrapTaskRun": { + "command": "echo", + "args": ["{{WrappedAction.Command}}"], + "cancelation": cancelation, + }, + "onWrapEnvExit": _cmd("{{WrappedAction.Command}}"), + } + return actions + + def test_full_cancelation_forwarding_accepted(self): + # Mark's example from the PR #261 review thread (minus timeout): + # the whole-field expressions in the wrap hook's cancelation + # block must parse and validate. + tmpl = _env_template( + self._wrap_actions( + { + "mode": "{{WrappedAction.Cancelation.Mode}}", + "notifyPeriodInSeconds": ( + "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}" + ), + } + ), + extensions=self._ROUND_TRIP_EXTS, + ) + _decode(tmpl, extensions=self._ROUND_TRIP_EXTS) + + def test_notify_period_only_forwarding_accepted(self): + # The narrower forward: a literal mode with only the notify + # period forwarded. notifyPeriodInSeconds is already @fmtstring + # under FEATURE_BUNDLE_1; the whole-field expression is int? and + # a null result must drop the field (schema default applies). + tmpl = _env_template( + self._wrap_actions( + { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": ( + "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}" + ), + } + ), + extensions=self._ROUND_TRIP_EXTS, + ) + _decode(tmpl, extensions=self._ROUND_TRIP_EXTS) + + def test_fmtstring_mode_requires_feature_bundle_1(self): + # The format-string mode form is gated on FEATURE_BUNDLE_1 + # (Template Schemas 5.3); with only WRAP_ACTIONS + EXPR it must + # be rejected. + tmpl = _env_template( + self._wrap_actions({"mode": "{{WrappedAction.Cancelation.Mode}}"}), + extensions=["WRAP_ACTIONS", "EXPR"], + ) + with pytest.raises(DecodeValidationError, match="FEATURE_BUNDLE_1"): + _decode(tmpl, extensions=["WRAP_ACTIONS", "EXPR"]) + + @pytest.mark.parametrize( + "mode", + [ + pytest.param("TERMIN{{WrappedAction.Cancelation.Mode}}", id="leading-text"), + pytest.param("{{ 'NOTIFY' }}_THEN_TERMINATE", id="trailing-text"), + ], + ) + def test_partial_fmtstring_mode_accepted(self, mode: str): + # A format-string mode gets normal format string behavior + # (Template Schemas 5.3): partial interpolation is statically + # valid, and the resolved value is checked against the two mode + # names at run time. Only a whole-field expression additionally + # gets string? null semantics (a null result drops the + # cancelation object). + tmpl = _env_template( + self._wrap_actions({"mode": mode}), + extensions=self._ROUND_TRIP_EXTS, + ) + _decode(tmpl, extensions=self._ROUND_TRIP_EXTS) From 86c6796423031bd362583be20dcb5531a766fcbf Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:15:48 -0700 Subject: [PATCH 05/13] test: Cover v0 let bindings, symbol-table mutators, and typed resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise coverage over the 94% gate (93.42% -> 94.39%): - test_let_bindings.py (new): the shared v0 evaluate_let_bindings loop — chained bindings, typed value preservation (float rendering fidelity, let-bound path property access), malformed-binding skipping, ValueError naming the failing binding, RHS parse memoization (cache_info), and parse errors propagating uncached. - test_symbol_table_cache.py: every remaining _VersionedDict mutator (__delitem__, setdefault, pop, popitem, clear) bumps the version and invalidates the engine-table cache. - test_symbol_table.py: expr_host_rules copied by the copy constructor and union; __repr__. - test_expr_node.py: symbol-free static validation errors at parse time, ExprNode.__repr__, InterpolationExpression.evaluate_value error wrapping and typed result, FormatString.resolve_value whole-field typed resolution, fallback, and error paths. - test_validator_functions.py (new): missing-context internal errors, float ge constraint, and validate_list_field error aggregation. - test_step_dependency_graph.py: max_indegree / max_outdegree. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../_internal/test_validator_functions.py | 69 ++++++++ .../model_v0/format_strings/test_expr_node.py | 78 +++++++++ test/openjd/model_v0/test_let_bindings.py | 153 ++++++++++++++++++ .../model_v0/test_step_dependency_graph.py | 36 +++++ test/openjd/model_v0/test_symbol_table.py | 32 ++++ .../model_v0/test_symbol_table_cache.py | 59 +++++++ 6 files changed, 427 insertions(+) create mode 100644 test/openjd/model_v0/_internal/test_validator_functions.py create mode 100644 test/openjd/model_v0/test_let_bindings.py diff --git a/test/openjd/model_v0/_internal/test_validator_functions.py b/test/openjd/model_v0/_internal/test_validator_functions.py new file mode 100644 index 00000000..e7d9fd4f --- /dev/null +++ b/test/openjd/model_v0/_internal/test_validator_functions.py @@ -0,0 +1,69 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Direct unit tests for ``openjd.model._internal._validator_functions``. + +These validators back the int/float format-string model fields; the happy +paths are exercised end-to-end via template parsing, so this file pins the +error paths that templates cannot cheaply reach: the missing-context +internal errors, constraint violations, and per-item error aggregation in +``validate_list_field``. +""" + +from decimal import Decimal +from functools import partial + +import pytest +from pydantic_core import PydanticKnownError, ValidationError + +from openjd.model._internal import ( + validate_float_fmtstring_field, + validate_int_fmtstring_field, + validate_list_field, +) + + +class TestMissingContextInternalError: + def test_int_validator_str_without_context_raises(self) -> None: + # A raw str with no parsing context is an internal error: the value + # should already have been converted to a FormatString upstream. + with pytest.raises(ValueError, match="Internal parsing error"): + validate_int_fmtstring_field("10", context=None) + + def test_float_validator_str_without_context_raises(self) -> None: + with pytest.raises(ValueError, match="Internal parsing error"): + validate_float_fmtstring_field("1.5", context=None) + + +class TestFloatConstraint: + def test_ge_violation_raises_known_error(self) -> None: + with pytest.raises(PydanticKnownError) as excinfo: + validate_float_fmtstring_field(Decimal("-1.5"), ge=Decimal("0"), context=None) + assert excinfo.value.type == "greater_than_equal" + + def test_ge_satisfied_returns_value(self) -> None: + assert validate_float_fmtstring_field( + Decimal("1.5"), ge=Decimal("0"), context=None + ) == Decimal("1.5") + + +class TestValidateListField: + def test_known_error_items_collected_with_location(self) -> None: + # An item validator raising PydanticKnownError (e.g. a ge constraint) + # is copied verbatim into the aggregate error with the item's index. + validator = partial(validate_int_fmtstring_field, ge=0) + with pytest.raises(ValidationError) as excinfo: + validate_list_field([1, -2, 3, -4], validator, context=None) + errors = excinfo.value.errors() + assert [e["loc"] for e in errors] == [(1,), (3,)] + assert all(e["type"] == "greater_than_equal" for e in errors) + + def test_value_error_items_collected_with_location(self) -> None: + with pytest.raises(ValidationError) as excinfo: + validate_list_field([1, "not-an-int"], validate_int_fmtstring_field, context=None) + errors = excinfo.value.errors() + assert [e["loc"] for e in errors] == [(1,)] + assert errors[0]["type"] == "value_error" + + def test_all_valid_returns_value_unchanged(self) -> None: + value = [1, 2, 3] + assert validate_list_field(value, validate_int_fmtstring_field, context=None) is value diff --git a/test/openjd/model_v0/format_strings/test_expr_node.py b/test/openjd/model_v0/format_strings/test_expr_node.py index 908122b6..e54fa7fe 100644 --- a/test/openjd/model_v0/format_strings/test_expr_node.py +++ b/test/openjd/model_v0/format_strings/test_expr_node.py @@ -308,3 +308,81 @@ def test_union_coercion_still_works_on_derived_table(self): def test_default_expr_types_is_empty(self): assert SymbolTable().expr_types == {} + + +class TestExprNodeStaticValidation: + """Symbol-free expressions are statically validated at parse (`check`) + time so literal/semantic errors surface before instantiation.""" + + @pytest.mark.parametrize( + "expr,fragment", + [ + pytest.param("1 / 0", "Division by zero", id="division-by-zero"), + pytest.param("nosuchfunc(1)", "Unknown function", id="unknown-function"), + pytest.param("1 + 'a'", "'+' operator", id="type-mismatch"), + ], + ) + def test_symbol_free_semantic_error_raises_at_parse(self, expr, fragment): + with pytest.raises(ExpressionError) as excinfo: + ExprNode(expr) + assert fragment in str(excinfo.value) + + def test_repr(self): + assert repr(ExprNode("1 + 2")) == "Expr(1 + 2)" + + +class TestInterpolationExpressionEvaluateValue: + """`InterpolationExpression.evaluate_value` (the typed-value form used by + let bindings) wraps engine ValueErrors as the model's ExpressionError, + mirroring `evaluate`.""" + + def test_evaluate_value_error_is_wrapped(self): + from openjd.model._format_strings._expression import InterpolationExpression + + expression = InterpolationExpression("Undefined.X + 1", context=_ctx(["EXPR"])) + with pytest.raises(ExpressionError) as excinfo: + expression.evaluate_value(symtab=SymbolTable()) + msg = str(excinfo.value) + assert msg.startswith("Expression failed validation:") + assert "Undefined variable: 'Undefined.X'" in msg + + def test_evaluate_value_returns_typed_engine_value(self): + from openjd.model._format_strings._expression import InterpolationExpression + + expression = InterpolationExpression("1.50", context=_ctx(["EXPR"])) + value = expression.evaluate_value(symtab=SymbolTable()) + # Rendering fidelity is preserved on the engine value. + assert str(value) == "1.50" + + +class TestFormatStringResolveValue: + """`FormatString.resolve_value` (RFC 0005/0006 typed resolution): a + whole-field `{{ ... }}` EXPR expression resolves to the engine's typed + value; anything else falls back to ordinary string resolution.""" + + def test_whole_field_returns_typed_value(self): + fs = FormatString("{{ [1, 2, 3] }}", context=_ctx(["EXPR"])) + value = fs.resolve_value(symtab=SymbolTable()) + assert value.item() == [1, 2, 3] + + def test_whole_field_with_surrounding_whitespace(self): + # Only-whitespace outside the braces still counts as whole-field. + fs = FormatString(" {{ 1 + 2 }} ", context=_ctx(["EXPR"])) + assert fs.resolve_value(symtab=SymbolTable()).item() == 3 + + def test_multi_segment_falls_back_to_string_resolution(self): + fs = FormatString("a={{ 1 + 2 }}", context=_ctx(["EXPR"])) + assert fs.resolve_value(symtab=SymbolTable()) == "a=3" + + def test_no_expression_falls_back_to_string_resolution(self): + fs = FormatString("plain text", context=_ctx(["EXPR"])) + assert fs.resolve_value(symtab=SymbolTable()) == "plain text" + + def test_evaluation_error_raises_format_string_error(self): + from openjd.model._format_strings._format_string import FormatStringError + + fs = FormatString("{{ Undefined.X + 1 }}", context=_ctx(["EXPR"])) + with pytest.raises(FormatStringError) as excinfo: + fs.resolve_value(symtab=SymbolTable()) + msg = str(excinfo.value) + assert "Undefined.X" in msg diff --git a/test/openjd/model_v0/test_let_bindings.py b/test/openjd/model_v0/test_let_bindings.py new file mode 100644 index 00000000..fe21d020 --- /dev/null +++ b/test/openjd/model_v0/test_let_bindings.py @@ -0,0 +1,153 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the v0 ``openjd.model.evaluate_let_bindings`` helper. + +This is the shared EXPR ``let``-binding evaluation loop (RFC 0007 §3.6) used +by the model's job instantiation and by the ``openjd-sessions`` runtime. It +mutates a :class:`SymbolTable` in place, seeding each binding's typed engine +value under the bound name. + +Not to be confused with ``openjd.model._v1.evaluate_let_bindings`` (the +Rust-backed v1 surface, tested in ``test/openjd/model_v1``), which returns a +new symbol table. +""" + +import pytest + +from openjd.model import SymbolTable, evaluate_let_bindings +from openjd.model._let_bindings import _parse_rhs + + +class TestEvaluateLetBindings: + def test_single_binding(self) -> None: + # GIVEN + symtab = SymbolTable() + + # WHEN + evaluate_let_bindings(symtab=symtab, let_bindings=["End = 1 + 9"]) + + # THEN + assert "End" in symtab + assert symtab["End"].item() == 10 + + def test_chained_bindings_reference_earlier_names(self) -> None: + # Each binding is evaluated against the symbol table built so far, + # so later bindings can reference earlier ones. + symtab = SymbolTable() + + evaluate_let_bindings( + symtab=symtab, + let_bindings=[ + "A = 10 + 1", + "B = A * 2", + "C = A + B", + ], + ) + + assert symtab["A"].item() == 11 + assert symtab["B"].item() == 22 + assert symtab["C"].item() == 33 + + def test_existing_symbols_visible_to_bindings(self) -> None: + # Bindings evaluate against the caller's symbol table, so pre-seeded + # symbols (e.g. job parameters) are referencable. + symtab = SymbolTable() + symtab["Param.X"] = 10 + symtab.expr_types["Param.X"] = "INT" + + evaluate_let_bindings(symtab=symtab, let_bindings=["Doubled = Param.X * 2"]) + + assert symtab["Doubled"].item() == 20 + + def test_float_rendering_fidelity_preserved(self) -> None: + # The engine's typed value is stored (not a Python float), so the + # declared trailing zeros survive str() coercion (RFC 0005). + symtab = SymbolTable() + + evaluate_let_bindings(symtab=symtab, let_bindings=["F = 1.50"]) + + assert str(symtab["F"]) == "1.50" + + def test_let_bound_path_keeps_property_access(self) -> None: + # A let-bound path stays a path: a later binding can use path + # property access on it, matching the Rust runtime's typed table. + symtab = SymbolTable() + + evaluate_let_bindings( + symtab=symtab, + let_bindings=[ + "P = path('/a/b/render.exr')", + "N = P.name", + ], + ) + + assert str(symtab["N"]) == "render.exr" + + @pytest.mark.parametrize( + "binding", + [ + pytest.param("no equals sign here", id="missing-equals"), + pytest.param(" = 1 + 2", id="empty-name"), + pytest.param("Name = ", id="empty-rhs"), + pytest.param("=", id="only-equals"), + ], + ) + def test_malformed_bindings_are_skipped(self, binding: str) -> None: + # Malformed bindings are rejected by the `let` field validator at + # decode time; the evaluation loop skips them defensively without + # raising or seeding anything. + symtab = SymbolTable() + + evaluate_let_bindings(symtab=symtab, let_bindings=[binding]) + + assert len(symtab.symbols) == 0 + + def test_evaluation_error_raises_valueerror_naming_binding(self) -> None: + # An RHS that fails to evaluate raises ValueError with the binding + # name in the message. + symtab = SymbolTable() + + with pytest.raises(ValueError) as excinfo: + evaluate_let_bindings(symtab=symtab, let_bindings=["X = Undefined.Y + 1"]) + + assert str(excinfo.value).startswith("let binding 'X':") + + def test_parse_error_raises_valueerror_naming_binding(self) -> None: + # A parse failure (ExpressionError is a ValueError) is re-raised + # naming the binding, same as an evaluation failure. + symtab = SymbolTable() + + with pytest.raises(ValueError) as excinfo: + evaluate_let_bindings(symtab=symtab, let_bindings=["Bad = 1 +"]) + + assert str(excinfo.value).startswith("let binding 'Bad':") + + def test_rhs_parse_is_memoized(self) -> None: + # The RHS parse is lru_cached: re-applying the same binding string + # (as the sessions runtime does per env enter/exit and task run) + # parses once and hits the cache thereafter. + _parse_rhs.cache_clear() + + evaluate_let_bindings(symtab=SymbolTable(), let_bindings=["A = 2 + 3"]) + info = _parse_rhs.cache_info() + assert info.misses == 1 + assert info.hits == 0 + + evaluate_let_bindings(symtab=SymbolTable(), let_bindings=["A = 2 + 3"]) + info = _parse_rhs.cache_info() + assert info.misses == 1 + assert info.hits == 1 + + def test_parse_errors_are_not_cached(self) -> None: + # lru_cache does not cache raised exceptions: the same bad RHS + # raises on every application rather than serving a stale success. + _parse_rhs.cache_clear() + + for _ in range(2): + with pytest.raises(ValueError) as excinfo: + evaluate_let_bindings(symtab=SymbolTable(), let_bindings=["Bad = 1 +"]) + assert str(excinfo.value).startswith("let binding 'Bad':") + + info = _parse_rhs.cache_info() + assert info.hits == 0 + assert info.misses == 2 diff --git a/test/openjd/model_v0/test_step_dependency_graph.py b/test/openjd/model_v0/test_step_dependency_graph.py index e72fc43e..849c1c27 100644 --- a/test/openjd/model_v0/test_step_dependency_graph.py +++ b/test/openjd/model_v0/test_step_dependency_graph.py @@ -233,3 +233,39 @@ def test_topo_sort_cycle_error(self) -> None: # THEN assert "circular dependency was found" in str(e) assert "S1 -> S2 -> S3 -> S4 -> S5 -> S6 -> S7 -> S1" in str(e) + + +class TestStepDependencyGraphDegrees_2023_09: + def _graph(self) -> StepDependencyGraph: + # Foo depends on Bar and Buz; Bar depends on Buz. + template_data = { + "specificationVersion": "jobtemplate-2023-09", + "name": "Job", + "steps": [ + { + "name": "Foo", + "script": {"actions": {"onRun": {"command": "foo"}}}, + "dependencies": [{"dependsOn": "Bar"}, {"dependsOn": "Buz"}], + }, + { + "name": "Bar", + "script": {"actions": {"onRun": {"command": "foo"}}}, + "dependencies": [{"dependsOn": "Buz"}], + }, + { + "name": "Buz", + "script": {"actions": {"onRun": {"command": "foo"}}}, + }, + ], + } + job_template = parse_model(model=JobTemplate_2023_09, obj=template_data) + job = create_job(job_template=job_template, job_parameter_values=dict()) + return StepDependencyGraph(job=job) + + def test_max_indegree(self) -> None: + # Foo has the most dependencies (2). + assert self._graph().max_indegree == 2 + + def test_max_outdegree(self) -> None: + # Buz has the most dependents (2). + assert self._graph().max_outdegree == 2 diff --git a/test/openjd/model_v0/test_symbol_table.py b/test/openjd/model_v0/test_symbol_table.py index 580e0f2c..4c005b76 100644 --- a/test/openjd/model_v0/test_symbol_table.py +++ b/test/openjd/model_v0/test_symbol_table.py @@ -137,3 +137,35 @@ def test_union_multiple(self): assert result["Test.Symtab3"] == "Three" assert result["Overlap1"] == 2, "Later arguments win duplicates" assert result["Overlap2"] == 3, "Later arguments win duplicates" + + +class TestSymbolTableExprHostRules: + """``expr_host_rules`` (the v0 host-context marker) is copied by + both the copy constructor and ``union`` so a derived table keeps the + session's host context.""" + + def test_copy_constructor_copies_host_rules(self) -> None: + source = SymbolTable() + source["Param.X"] = "10" + source.expr_host_rules = [] + copied = SymbolTable(source=source) + assert copied.expr_host_rules == [] + # It's a copy, not a shared list. + assert copied.expr_host_rules is not source.expr_host_rules + + def test_union_copies_host_rules_from_self(self) -> None: + left = SymbolTable() + left.expr_host_rules = [] + merged = left.union(SymbolTable()) + assert merged.expr_host_rules == [] + + def test_union_copies_host_rules_from_operand(self) -> None: + right = SymbolTable() + right.expr_host_rules = [] + merged = SymbolTable().union(right) + assert merged.expr_host_rules == [] + + def test_repr(self) -> None: + symtab = SymbolTable() + symtab["Param.X"] = "10" + assert repr(symtab) == "SymbolTable({'Param.X': '10'})" diff --git a/test/openjd/model_v0/test_symbol_table_cache.py b/test/openjd/model_v0/test_symbol_table_cache.py index d4d63ad8..09e457ad 100644 --- a/test/openjd/model_v0/test_symbol_table_cache.py +++ b/test/openjd/model_v0/test_symbol_table_cache.py @@ -216,3 +216,62 @@ def counting(value, symtab): # type: ignore[no-untyped-def] tpd = step.parameterSpace.taskParameterDefinitions tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] assert list(tp.range) == [1, 2, 3] + + +class TestVersionedDictMutationsBumpVersion: + """Every ``_VersionedDict`` mutation method must bump the owning + SymbolTable's version, or the EXPR evaluation cache would serve a stale + engine table after that mutation. ``__setitem__``/``update``/``__ior__`` + are covered above via cache invalidation; this pins the remaining + mutators directly against the version counter.""" + + def test_delitem_bumps_version(self) -> None: + symtab = SymbolTable() + symtab.expr_types["Param.X"] = "INT" + before = symtab._version + del symtab.expr_types["Param.X"] + assert symtab._version > before + assert "Param.X" not in symtab.expr_types + + def test_setdefault_bumps_version(self) -> None: + symtab = SymbolTable() + before = symtab._version + assert symtab.expr_types.setdefault("Param.X", "INT") == "INT" + assert symtab._version > before + # setdefault on an existing key still bumps (conservative: a no-op + # bump only invalidates a cache entry, never poisons one). + before = symtab._version + assert symtab.expr_types.setdefault("Param.X", "FLOAT") == "INT" + assert symtab._version > before + + def test_pop_bumps_version(self) -> None: + symtab = SymbolTable() + symtab.expr_types["Param.X"] = "INT" + before = symtab._version + assert symtab.expr_types.pop("Param.X") == "INT" + assert symtab._version > before + + def test_popitem_bumps_version(self) -> None: + symtab = SymbolTable() + symtab.expr_types["Param.X"] = "INT" + before = symtab._version + assert symtab.expr_types.popitem() == ("Param.X", "INT") + assert symtab._version > before + + def test_clear_bumps_version(self) -> None: + symtab = SymbolTable() + symtab.expr_types["Param.X"] = "INT" + before = symtab._version + symtab.expr_types.clear() + assert symtab._version > before + assert symtab.expr_types == {} + + def test_delitem_invalidates_engine_table_cache(self) -> None: + # End-to-end: a deletion mutation must be observed by the cache. + symtab = SymbolTable() + symtab["Param.X"] = "10" + symtab.expr_types["Param.X"] = "INT" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + del symtab.expr_types["Param.X"] + second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert first is not second From 53055ff9f2725e207fdbe025347db0bec37f765e Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:15:57 -0700 Subject: [PATCH 06/13] chore: Regenerate THIRD-PARTY-LICENSES (annotated-types 0.7.0 -> 0.8.0) Dependency resolution now pulls annotated-types 0.8.0; regenerated via scripts/check_third_party_licenses.sh --update and verified with the script's check mode. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- THIRD-PARTY-LICENSES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 6413aae8..fa0fb1ed 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -1,6 +1,6 @@ -** annotated-types; version 0.7.0 -- https://pypi.org/project/annotated-types/ +** annotated-types; version 0.8.0 -- https://pypi.org/project/annotated-types/ The MIT License (MIT) Copyright (c) 2022 the contributors From 1deda17ec435b105f6fd08e924244bb2182a115f Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:28:02 -0700 Subject: [PATCH 07/13] fix: Windows-safe test paths and CodeQL alerts - Use pytest's tmp_path instead of Path("/tmp") in the single-evaluation regression test: WindowsPath("/tmp") has no drive letter, so is_absolute() is False and preprocess_job_parameters rejects it on the Windows CI matrix. - Bind mutating expressions to variables before asserting in the _VersionedDict version-bump tests (CodeQL py/side-effect-in-assert). - Define _VersionedDict.__eq__ (plain dict equality; the owner backref is bookkeeping, not value state) and explicit __hash__ = None (CodeQL py/missing-equals). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/_symbol_table.py | 9 +++++++++ .../openjd/model_v0/test_symbol_table_cache.py | 18 +++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/openjd/model/_symbol_table.py b/src/openjd/model/_symbol_table.py index 21fdd498..0edb378e 100644 --- a/src/openjd/model/_symbol_table.py +++ b/src/openjd/model/_symbol_table.py @@ -29,6 +29,15 @@ def __init__(self, owner: "SymbolTable", *args: Any, **kwargs: Any) -> None: # Backref to the owning SymbolTable; internal to this module. self._owner = owner + def __eq__(self, other: object) -> bool: + # The owner backref is bookkeeping, not value state: equality is + # plain dict equality (two tables' expr_types with the same contents + # compare equal regardless of which SymbolTable owns them). + return super().__eq__(other) + + # dict subclasses are unhashable; keep that explicit alongside __eq__. + __hash__ = None # type: ignore[assignment] + def __setitem__(self, key: Any, value: Any) -> None: super().__setitem__(key, value) self._owner._bump_version() diff --git a/test/openjd/model_v0/test_symbol_table_cache.py b/test/openjd/model_v0/test_symbol_table_cache.py index 09e457ad..0271c2e5 100644 --- a/test/openjd/model_v0/test_symbol_table_cache.py +++ b/test/openjd/model_v0/test_symbol_table_cache.py @@ -158,7 +158,7 @@ def test_profile_cached_and_invalidated_by_host_rules(self) -> None: class TestTypedResolutionSingleEvaluation: def test_range_expression_evaluated_once_per_definition( - self, monkeypatch: pytest.MonkeyPatch + self, monkeypatch: pytest.MonkeyPatch, tmp_path: "Path" ) -> None: """RFC 0006 typed whole-field range resolution: the expression is evaluated exactly once per task-parameter definition — the create_as @@ -202,8 +202,8 @@ def counting(value, symtab): # type: ignore[no-untyped-def] params = preprocess_job_parameters( job_template=template, job_parameter_values={}, - job_template_dir=Path("/tmp"), - current_working_dir=Path("/tmp"), + job_template_dir=tmp_path, + current_working_dir=tmp_path, ) job = create_job(job_template=template, job_parameter_values=params) @@ -236,26 +236,30 @@ def test_delitem_bumps_version(self) -> None: def test_setdefault_bumps_version(self) -> None: symtab = SymbolTable() before = symtab._version - assert symtab.expr_types.setdefault("Param.X", "INT") == "INT" + inserted = symtab.expr_types.setdefault("Param.X", "INT") + assert inserted == "INT" assert symtab._version > before # setdefault on an existing key still bumps (conservative: a no-op # bump only invalidates a cache entry, never poisons one). before = symtab._version - assert symtab.expr_types.setdefault("Param.X", "FLOAT") == "INT" + existing = symtab.expr_types.setdefault("Param.X", "FLOAT") + assert existing == "INT" assert symtab._version > before def test_pop_bumps_version(self) -> None: symtab = SymbolTable() symtab.expr_types["Param.X"] = "INT" before = symtab._version - assert symtab.expr_types.pop("Param.X") == "INT" + popped = symtab.expr_types.pop("Param.X") + assert popped == "INT" assert symtab._version > before def test_popitem_bumps_version(self) -> None: symtab = SymbolTable() symtab.expr_types["Param.X"] = "INT" before = symtab._version - assert symtab.expr_types.popitem() == ("Param.X", "INT") + popped_item = symtab.expr_types.popitem() + assert popped_item == ("Param.X", "INT") assert symtab._version > before def test_clear_bumps_version(self) -> None: From 914ea051592fdab0d5db50991a700baa83ece2d3 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:38:52 -0700 Subject: [PATCH 08/13] fix: Defer forwardable action fields to run time; cache key and lint fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes: 1. Defer run-time-resolvable action fields to run time, matching openjd-rs (job/create_job/instantiate.rs clones timeout and cancelation unresolved into the Job; the session resolves them right before the action runs). Previously Action.timeout and the cancelation classes' notifyPeriodInSeconds were in resolve_fields, so create_job resolved them at job creation. A wrap hook forwarding "{{WrappedAction.Timeout}}" or "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}" (RFC 0008 round-trip forwarding) decoded cleanly but crashed create_job with "Undefined variable" because those symbols only exist at run time. The fields now carry their raw FormatStrings through job instantiation; validation still happens at TEMPLATE scope at decode time (unchanged, matching openjd-rs format_strings.rs). Deliberate behavior change: a plain FEATURE_BUNDLE_1 template with timeout "{{Param.X}}" now carries the FormatString into the Job instead of resolving it at creation — resolution moves to the session, as in openjd-rs. The sessions runtime already resolves FormatString timeouts/notify periods at run time (_resolve_action_timeout, resolve_effective_cancelation). Fixes the wrap-cancelation-roundtrip-job-environment conformance fixture, which failed at create_job for exactly this bug. 2. Fix a typed/untyped cache-key collision in symtab_to_expr_values: priming the engine-table cache with types=None on a symbol table whose expr_types is non-empty cached an untyped table that a later typed call at the same mutation version was served, losing the type coercions. The cache condition is now strict: cacheable iff types is the symtab's own expr_types, or types is None and the symtab has no expr_types. 3. Module-level Path import in symbol-table cache tests (ruff F821/F401), retained from the previous revision of this commit. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../model/_format_strings/_expr_support.py | 11 ++- src/openjd/model/v2023_09/_model.py | 39 +++++--- .../model_v0/test_symbol_table_cache.py | 41 ++++++++- .../v2023_09/test_feature_bundle_1.py | 88 +++++++++++++++++++ .../model_v0/v2023_09/test_wrap_actions.py | 60 ++++++++++++- 5 files changed, 220 insertions(+), 19 deletions(-) diff --git a/src/openjd/model/_format_strings/_expr_support.py b/src/openjd/model/_format_strings/_expr_support.py index f3abff18..a0222c70 100644 --- a/src/openjd/model/_format_strings/_expr_support.py +++ b/src/openjd/model/_format_strings/_expr_support.py @@ -83,10 +83,15 @@ def symtab_to_expr_values( embedded-file ``data``) against one unchanged table — without the cache the whole table is rebuilt across the Rust boundary per expression. Caching requires ``types`` to be the symbol table's own ``expr_types`` - (or ``None``), which is what the evaluation layer passes; any other - ``types`` object bypasses the cache. + (which is what the evaluation layer passes), or ``None`` when the symbol + table has no ``expr_types``; any other combination bypasses the cache. + In particular an untyped (``types=None``) build on a symbol table that + *has* ``expr_types`` is never cached — otherwise a later typed call at + the same mutation version would be served the stale untyped table and + lose the type coercions (typed/untyped cache-key collision). """ - cacheable = types is None or types is getattr(symtab, "expr_types", None) + symtab_expr_types = getattr(symtab, "expr_types", None) + cacheable = types is symtab_expr_types or (types is None and not symtab_expr_types) version = getattr(symtab, "_version", None) cache_key = ("engine_table", str(path_format)) if cacheable and version is not None: diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 80771079..19389dee 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -344,7 +344,13 @@ class CancelationMethodNotifyThenTerminate(OpenJDModel_v2023_09): mode: Literal[CancelationMode.NOTIFY_THEN_TERMINATE] notifyPeriodInSeconds: Optional[Union[NotifyPeriodType, FormatString]] = None # noqa: N815 - _job_creation_metadata = JobCreationMetadata(resolve_fields={"notifyPeriodInSeconds"}) + # A FormatString notifyPeriodInSeconds is NOT resolved at job creation: + # it is carried through unresolved and resolved at run time by the + # session, matching openjd-rs (job/create_job/instantiate.rs clones the + # cancelation object unresolved into the Job). This is what lets RFC 0008 + # wrap hooks forward "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}", + # whose symbols only exist at run time. + _job_creation_metadata = JobCreationMetadata() @field_validator("notifyPeriodInSeconds", mode="before") @classmethod @@ -432,7 +438,13 @@ class CancelationMethodDeferred(OpenJDModel_v2023_09): mode: FormatString notifyPeriodInSeconds: Optional[Union[NotifyPeriodType, FormatString]] = None # noqa: N815 - _job_creation_metadata = JobCreationMetadata(resolve_fields={"notifyPeriodInSeconds"}) + # Neither `mode` nor a FormatString `notifyPeriodInSeconds` is resolved + # at job creation: the whole deferred cancelation object is carried + # through unresolved and resolved at run time by the session, matching + # openjd-rs (job/create_job/instantiate.rs). Run-time-only symbols such + # as WrappedAction.Cancelation.* are how RFC 0008 wrap hooks forward the + # wrapped action's cancelation. + _job_creation_metadata = JobCreationMetadata() @field_validator("mode", mode="before") @classmethod @@ -584,16 +596,23 @@ class Action(OpenJDModel_v2023_09): timeout: Optional[Union[PositiveInt, FormatString]] = None cancelation: Optional[CancelationMethod] = None - _job_creation_metadata = JobCreationMetadata(resolve_fields={"timeout"}) + # A FormatString `timeout` is NOT resolved at job creation: it is carried + # through job instantiation unresolved and resolved at run time by the + # session, matching openjd-rs (job/create_job/instantiate.rs clones + # timeout and cancelation unresolved into the Job, and the session + # resolves them right before the action runs). This is what lets RFC 0008 + # wrap hooks forward "{{WrappedAction.Timeout}}" — those symbols only + # exist at run time. + _job_creation_metadata = JobCreationMetadata() # `timeout` and `cancelation` (its notifyPeriodInSeconds and a deferred - # format-string mode) are plain @fmtstring fields resolved at job - # creation, before any session exists — unlike `command`/`args`, which - # resolve in the session. They therefore validate at template scope: no - # Session.*, no Env.File.*/Task.File.*, and no host-context functions. - # The RFC 0008 wrap hooks may still forward the wrapped action's values - # ("{{WrappedAction.Timeout}}"): the WrappedAction.* symbols are injected - # per-hook at every scope via EnvironmentActions._template_field_inject. + # format-string mode) still VALIDATE at template scope: no Session.*, no + # Env.File.*/Task.File.*, and no host-context functions (matching + # openjd-rs format_strings.rs). The RFC 0008 wrap hooks may forward the + # wrapped action's values ("{{WrappedAction.Timeout}}"): the + # WrappedAction.* symbols are injected per-hook at every scope via + # EnvironmentActions._template_field_inject, and openjd-rs's + # symtab-filter preserves them for run time. _template_field_scopes = { "timeout": ResolutionScope.TEMPLATE, "cancelation": ResolutionScope.TEMPLATE, diff --git a/test/openjd/model_v0/test_symbol_table_cache.py b/test/openjd/model_v0/test_symbol_table_cache.py index 0271c2e5..7388abf6 100644 --- a/test/openjd/model_v0/test_symbol_table_cache.py +++ b/test/openjd/model_v0/test_symbol_table_cache.py @@ -10,6 +10,8 @@ ``expr_host_rules``) invalidates the cache. """ +from pathlib import Path + import pytest from openjd.model import SymbolTable @@ -68,6 +70,42 @@ def test_expr_types_update_invalidates(self) -> None: second = symtab_to_expr_values(symtab, types=symtab.expr_types or None) assert first is not second + def test_untyped_prime_does_not_poison_typed_call(self) -> None: + # Regression: priming the cache with types=None on a symbol table + # whose expr_types is non-empty must NOT cache an untyped engine + # table — a later typed call at the same mutation version would be + # served the stale untyped table and lose the type coercions. + from openjd.expr import ExprProfile + + from openjd.model._format_strings._expr_support import parse_expr_or_raise + + symtab = SymbolTable() + symtab["Param.X"] = "10" + symtab.expr_types["Param.X"] = "INT" + + # WHEN: prime with an untyped build on the typed symtab. + untyped = symtab_to_expr_values(symtab, types=None) + + # THEN: the subsequent typed call is not served the stale table... + typed = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert typed is not untyped + + # ...and coerces correctly: arithmetic sees the int 10, not "10". + parsed = parse_expr_or_raise("Param.X + 1") + result = parsed.evaluate(values=typed, profile=ExprProfile.current()) + assert result.item() == 11 + + # The typed build is the one cached for the standard call form. + assert symtab_to_expr_values(symtab, types=symtab.expr_types or None) is typed + + def test_untyped_call_still_cached_when_symtab_has_no_types(self) -> None: + # types=None on a symtab without expr_types remains cacheable (this + # is the evaluation layer's call form for untyped tables). + symtab = SymbolTable() + symtab["Param.X"] = "10" + first = symtab_to_expr_values(symtab, types=symtab.expr_types or None) + assert symtab_to_expr_values(symtab, types=symtab.expr_types or None) is first + def test_foreign_types_mapping_bypasses_cache(self) -> None: # A caller-supplied types mapping that is not the table's own # expr_types must not poison or consult the cache. @@ -158,12 +196,11 @@ def test_profile_cached_and_invalidated_by_host_rules(self) -> None: class TestTypedResolutionSingleEvaluation: def test_range_expression_evaluated_once_per_definition( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: "Path" + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """RFC 0006 typed whole-field range resolution: the expression is evaluated exactly once per task-parameter definition — the create_as target-model decision and the field value share the result.""" - from pathlib import Path from openjd.model import create_job, decode_job_template, preprocess_job_parameters from openjd.model._internal import _create_job as internal_create_job diff --git a/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py b/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py index 8e07a918..15ae2c65 100644 --- a/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py +++ b/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py @@ -620,6 +620,94 @@ def test_amount_requirement_min_greater_than_max_resolved_fails(self) -> None: ) assert "max" in str(excinfo.value).lower() + def test_timeout_and_notify_period_carried_through_unresolved(self) -> None: + """timeout and notifyPeriodInSeconds format strings are NOT resolved + at job creation: they are carried through the instantiated Job as raw + FormatStrings and resolved at run time by the session, matching + openjd-rs (job/create_job/instantiate.rs clones timeout+cancelation + unresolved into the Job). This is what allows RFC 0008 wrap hooks to + forward run-time-only symbols such as WrappedAction.Timeout.""" + from openjd.model import create_job, decode_job_template + from openjd.model._format_strings import FormatString + from openjd.model._types import ParameterValue, ParameterValueType + + template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["FEATURE_BUNDLE_1"], + "name": "Test", + "parameterDefinitions": [{"name": "TO", "type": "INT", "default": 30}], + "steps": [ + { + "name": "Step1", + "script": { + "actions": { + "onRun": { + "command": "echo", + "timeout": "{{Param.TO}}", + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": "{{Param.TO}}", + }, + } + } + }, + } + ], + }, + supported_extensions=[ExtensionName.FEATURE_BUNDLE_1], + ) + + # WHEN + job = create_job( + job_template=template, + job_parameter_values={"TO": ParameterValue(type=ParameterValueType.INT, value="30")}, + ) + + # THEN: creation succeeds and both fields carry the raw FormatString. + on_run = job.steps[0].script.actions.onRun + assert isinstance(on_run.timeout, FormatString) + assert str(on_run.timeout) == "{{Param.TO}}" + assert on_run.cancelation is not None + assert isinstance(on_run.cancelation.notifyPeriodInSeconds, FormatString) + assert str(on_run.cancelation.notifyPeriodInSeconds) == "{{Param.TO}}" + + def test_literal_timeout_and_notify_period_stay_ints_through_create_job(self) -> None: + """Literal integer timeout/notifyPeriodInSeconds are unaffected by + the run-time deferral of format-string values.""" + from openjd.model import create_job, decode_job_template + + template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["FEATURE_BUNDLE_1"], + "name": "Test", + "steps": [ + { + "name": "Step1", + "script": { + "actions": { + "onRun": { + "command": "echo", + "timeout": 30, + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": 15, + }, + } + } + }, + } + ], + }, + supported_extensions=[ExtensionName.FEATURE_BUNDLE_1], + ) + job = create_job(job_template=template, job_parameter_values={}) + on_run = job.steps[0].script.actions.onRun + assert on_run.timeout == 30 + assert on_run.cancelation is not None + assert on_run.cancelation.notifyPeriodInSeconds == 15 + class TestAmountRequirementEdgeCases: """Edge case tests for AmountRequirementTemplate.""" diff --git a/test/openjd/model_v0/v2023_09/test_wrap_actions.py b/test/openjd/model_v0/v2023_09/test_wrap_actions.py index 90cc97b4..e0400f37 100644 --- a/test/openjd/model_v0/v2023_09/test_wrap_actions.py +++ b/test/openjd/model_v0/v2023_09/test_wrap_actions.py @@ -295,6 +295,57 @@ def test_create_job_with_step_env_wrap_succeeds(self): step_env_actions = job.steps[0].stepEnvironments[0].script.actions assert step_env_actions.onWrapTaskRun is not None + def test_create_job_with_full_forwarding_defers_runtime_fields(self): + # Regression: a wrap hook forwarding the wrapped action's timeout and + # cancelation ("{{WrappedAction.Timeout}}", + # "{{WrappedAction.Cancelation.*}}") decoded cleanly but crashed + # create_job with "Undefined variable" — those symbols only exist at + # run time. Like openjd-rs (job/create_job/instantiate.rs clones + # timeout+cancelation unresolved into the Job), the fields must be + # carried through job instantiation as raw FormatStrings and left for + # the session to resolve. + from openjd.model._format_strings import FormatString + + hook = { + "command": "{{WrappedAction.Command}}", + "args": ["{{WrappedAction.Args}}"], + "timeout": "{{WrappedAction.Timeout}}", + "cancelation": { + "mode": "{{WrappedAction.Cancelation.Mode}}", + "notifyPeriodInSeconds": ("{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}"), + }, + } + wrap_env = { + "name": "W", + "script": { + "actions": { + "onWrapEnvEnter": dict(hook), + "onWrapTaskRun": dict(hook), + "onWrapEnvExit": dict(hook), + } + }, + } + exts = ("WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1") + jt = _decode_job(_job_template(job_environments=[wrap_env], extensions=exts)) + + # WHEN + job = create_job(job_template=jt, job_parameter_values={}) + + # THEN: creation succeeds and every run-time-resolvable field on the + # instantiated hook action is the raw, unresolved FormatString. + action = job.jobEnvironments[0].script.actions.onWrapTaskRun + assert isinstance(action.timeout, FormatString) + assert str(action.timeout) == "{{WrappedAction.Timeout}}" + assert isinstance(action.cancelation.mode, FormatString) + assert str(action.cancelation.mode) == "{{WrappedAction.Cancelation.Mode}}" + assert isinstance(action.cancelation.notifyPeriodInSeconds, FormatString) + assert ( + str(action.cancelation.notifyPeriodInSeconds) + == "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}" + ) + assert isinstance(action.command, FormatString) + assert str(action.command) == "{{WrappedAction.Command}}" + class TestCancelationRoundTripForwarding: """Cancelation round-trip forwarding (openjd-rs PR #261 review, @@ -314,10 +365,11 @@ class TestCancelationRoundTripForwarding: Mirrors ``cancelation_round_trip_*`` in openjd-rs ``crates/openjd-model/tests/integration/test_wrap_actions.rs``. - Note: the review example also forwards ``timeout:``; that is - deliberately out of scope pending the WrappedAction.Timeout - int-vs-int? question (the 0-when-unset sentinel is not a valid - ````). + Note: the review example also forwards ``timeout:``; that works too — + see TestWrapActionsCreateJob + .test_create_job_with_full_forwarding_defers_runtime_fields, which + covers full forwarding (command/args/timeout/cancelation) through + create_job. """ _ROUND_TRIP_EXTS = ["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"] From 53b0fd162205e77ec8df927285921c2e425f8d20 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 08:32:04 -0700 Subject: [PATCH 09/13] fix: Pickle-safe deferred FormatStrings; scope parity for SimpleAction and wrap hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix three review findings on the run-time-deferred timeout/cancelation work: 1. Jobs carrying deferred FormatStrings were unpicklable and un-deepcopyable. FormatString.__new__ requires a keyword-only parsing context, but str.__getnewargs__ supplies only the value, so pickle.loads, copy.copy/deepcopy, and model_copy(deep=True) raised TypeError - and subclasses that default the context (ArgString, CommandString) either failed pickling on the engine-backed parse tree or re-parsed EXPR-grammar strings with the legacy parser. FormatString now snapshots the parse-relevant context (spec revision + extensions) at construction and implements __getnewargs_ex__ (reconstructing through __new__ with a context that reproduces the original grammar) plus __getstate__ returning None (the parse tree is rebuilt, never serialized). EXPR strings round-trip with identical typed evaluation. 2. SimpleAction validated timeout/cancelation at the ambient TASK scope while the identical desugared Action form validates them at TEMPLATE scope. Add the same _template_field_scopes to SimpleAction, so e.g. a bash-sugar timeout referencing Task.Param.* is now rejected exactly like the script-form equivalent. 3. Wrap-hook timeout/cancelation could reference WrappedEnv.Name / WrappedStep.Name because all per-hook symbols were injected at TEMPLATE scope. openjd-rs validates hook timeout/cancelation against template symbols + WrappedAction.* only. Split the injection: _template_field_inject keeps WrappedAction.* (all scopes) and the new _template_field_inject_session carries WrappedEnv.Name / WrappedStep.Name at SESSION scope, so hook command/args still accept them but timeout/cancelation reject them. 4. §3.4.2 empty-PATH propagation (flagged by the automated PR review): PathTaskParameterDefinition rejects literal empty-string range items at parse time, but RangeListTaskParameterDefinition - the model that RFC 0006 typed whole-field resolution instantiates ranges into - had no such check, so `range: "{{Param.Paths}}"` with a LIST[PATH] job parameter containing "" flowed into the instantiated Job. Add the matching validator on the create-time target model (PATH only; STRING ranges legitimately allow ""), mirroring openjd-rs's resolve-time check in create_job (ranges.rs). Also reword three stale comments claiming create-time resolution (the fields validate at template scope but resolve at run time), and hoist redundant function-local imports in the FB1 and wrap-actions tests. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../model/_format_strings/_format_string.py | 46 ++++- .../_variable_reference_validation.py | 21 ++- src/openjd/model/_types.py | 26 ++- src/openjd/model/v2023_09/_model.py | 52 +++++- test/openjd/model_v0/test_pickle.py | 165 ++++++++++++++++++ .../v2023_09/test_create_job_expr_params.py | 61 +++++++ .../v2023_09/test_feature_bundle_1.py | 88 ++++++++-- .../model_v0/v2023_09/test_wrap_actions.py | 61 ++++++- 8 files changed, 486 insertions(+), 34 deletions(-) create mode 100644 test/openjd/model_v0/test_pickle.py diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index 572c966c..b47245e2 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -7,7 +7,18 @@ from .._symbol_table import SymbolTable from ._dyn_constrained_str import DynamicConstrainedStr from ._expression import InterpolationExpression -from .._types import ModelParsingContextInterface +from .._types import ModelParsingContextInterface, SpecificationRevision + + +class _ReconstructionContext(ModelParsingContextInterface): + """Minimal parsing context used when a FormatString is reconstructed by + pickle or the copy module (see ``FormatString.__getnewargs_ex__``). + + It carries only the parse-relevant snapshot (specification revision and + extension set) that the FormatString recorded at construction time, so + the reconstructed instance is re-parsed under the same grammar (EXPR vs + legacy) as the original. + """ @dataclass @@ -31,6 +42,13 @@ def __init__(self, *, string: str, start: int, end: int, expr: str = "", details class FormatString(DynamicConstrainedStr): _processed_list: list[Union[str, ExpressionInfo]] + # Parse-relevant snapshot of the construction context, recorded so that + # pickle/copy reconstruction re-parses the string under the same grammar + # (EXPR vs legacy). The context object itself is not retained: it is + # shared and mutable (its extension set is narrowed while the template's + # `extensions` field is validated), so we snapshot at construction time. + _parse_spec_rev: SpecificationRevision + _parse_extensions: frozenset[str] def __new__(cls, value: str, *, context: ModelParsingContextInterface): """ @@ -53,9 +71,35 @@ def __new__(cls, value: str, *, context: ModelParsingContextInterface): FormatStringError: if the original string is nonvalid. """ self = super().__new__(cls, value, context=context) + self._parse_spec_rev = context.spec_rev + self._parse_extensions = frozenset(context.extensions) self._processed_list = self._preprocess(context=context) return self + def __getnewargs_ex__(self) -> tuple[tuple[str], dict[str, Any]]: + """Support for pickling and copying (``pickle``, ``copy.copy``, + ``copy.deepcopy``, and pydantic's ``model_copy(deep=True)``). + + ``str.__getnewargs__`` supplies only the string value, which cannot + satisfy the keyword-only ``context`` argument of ``__new__`` — and for + subclasses that default the argument, it would re-parse an + EXPR-grammar string with the legacy parser. Reconstruct through + ``__new__`` with a context carrying the recorded parse snapshot so + the copy is parsed exactly as the original was. + """ + context = _ReconstructionContext( + spec_rev=self._parse_spec_rev, + supported_extensions=self._parse_extensions, + ) + return ((str(self),), {"context": context}) + + def __getstate__(self) -> None: + """No instance state is serialized: ``_processed_list`` holds + engine-backed parse trees that cannot be pickled, and ``__new__`` + fully rebuilds it (and the parse snapshot) from the string value and + the reconstruction context (see ``__getnewargs_ex__``).""" + return None + @property def original_value(self) -> str: """ diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index 10ad7863..b98df6c5 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -415,15 +415,26 @@ def _validate_model_template_variable_references( # Per-field extra symbols (e.g. the RFC 0008 WrappedAction.* variables # within their wrap hook's action). Added at TEMPLATE scope so they # are visible in every scope within the field's subtree, including - # creation-time fields such as the action's timeout — a wrap hook's - # timeout may forward "{{WrappedAction.Timeout}}". + # the template-scoped timeout/cancelation fields — a wrap hook's + # timeout may forward "{{WrappedAction.Timeout}}" for run-time + # resolution. for symbol in model._template_field_inject.get(field_name, set()): symbol_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" _add_symbol(validation_symbols, ResolutionScope.TEMPLATE, symbol_name) - # Per-field scope override: fields that resolve at job creation (e.g. - # an Action's timeout/cancelation) validate at their declared scope - # rather than the model's ambient scope. + # Per-field SESSION-scoped symbols (e.g. the RFC 0008 + # WrappedEnv.Name / WrappedStep.Name variables): visible to the + # field subtree's session/task-scoped fields (a hook's command/args) + # but not to its template-scoped fields (timeout/cancelation), + # matching openjd-rs's wrap-hook validation. + for symbol in model._template_field_inject_session.get(field_name, set()): + symbol_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" + _add_symbol(validation_symbols, ResolutionScope.SESSION, symbol_name) + + # Per-field scope override: fields such as an Action's + # timeout/cancelation validate at their declared (template) scope + # rather than the model's ambient scope, even though their values are + # carried through job creation unresolved and resolve at run time. field_scope = model._template_field_scopes.get(field_name, current_scope) errors.extend( diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index 47e6b2f0..f731ccac 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -377,21 +377,31 @@ class OpenJDModel(BaseModel): # Per-field variable-scope overrides: fields listed here (and their # submodels) validate their format-string references at the given scope - # instead of the model's ambient scope. Used for fields that resolve at - # job creation while their siblings resolve at run time — e.g. an - # Action's `timeout` and `cancelation` are creation-time fields - # (template scope: no Session.*, no Env.File.*/Task.File.*, no host - # functions) while its `command`/`args` resolve in the session. + # instead of the model's ambient scope. Used for fields that VALIDATE at + # template scope (no Session.*, no Env.File.*/Task.File.*, no host + # functions) while their siblings validate at the ambient session/task + # scope — e.g. an Action's `timeout` and `cancelation`, which are carried + # through job creation unresolved and resolve at run time, but may only + # reference template-scope symbols (plus per-field injections below). _template_field_scopes: ClassVar[dict[str, ResolutionScope]] = {} # Per-field extra symbol injection: symbols listed here are visible in # every scope, but only within the named field's subtree. Names use the # DefinesTemplateVariables.inject spelling (a "|" prefix discards the - # parent scope prefix). Used for the RFC 0008 wrap hooks, whose - # WrappedAction.* / WrappedEnv.* / WrappedStep.* variables exist only - # within their hook's action. + # parent scope prefix). Used for the RFC 0008 wrap hooks' WrappedAction.* + # variables, which exist only within their hook's action but must be + # visible even to the hook's template-scoped fields (timeout/cancelation) + # for round-trip forwarding. _template_field_inject: ClassVar[dict[str, set[str]]] = {} + # As _template_field_inject, but the symbols are injected at SESSION + # scope: visible to the field subtree's session/task-scoped fields + # (e.g. a wrap hook's command/args) but NOT to its template-scoped + # fields (timeout/cancelation). Used for the RFC 0008 WrappedEnv.Name / + # WrappedStep.Name variables, which openjd-rs excludes from hook + # timeout/cancelation validation. + _template_field_inject_session: ClassVar[dict[str, set[str]]] = {} + # ---- # Metadata used in the creation of a Job from a Job Template diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 19389dee..e9ff6386 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -683,10 +683,16 @@ class EnvironmentActions(OpenJDModel_v2023_09): # hook's action, seeded by the runtime when the hook runs in place of the # wrapped action. WrappedAction.* is available in all three hooks; # WrappedEnv.Name only in the env-enter/exit hooks; WrappedStep.Name only - # in the task-run hook. Injected per-field at every scope so a hook's - # creation-scoped fields (timeout/cancelation) can round-trip forward - # the wrapped action's values ("{{WrappedAction.Timeout}}", - # "{{WrappedAction.Cancelation.Mode}}"). + # in the task-run hook. + # + # WrappedAction.* is injected at every scope so a hook's template-scoped + # fields (timeout/cancelation) can round-trip forward the wrapped + # action's values ("{{WrappedAction.Timeout}}", + # "{{WrappedAction.Cancelation.Mode}}"). WrappedEnv.Name / + # WrappedStep.Name are injected at SESSION scope only: a hook's + # command/args may reference them, but its timeout/cancelation may not — + # matching openjd-rs (format_strings.rs validates hook + # timeout/cancelation against template symbols + WrappedAction.* only). _WRAPPED_ACTION_SYMBOLS: ClassVar[set[str]] = { "|WrappedAction.Command", "|WrappedAction.Args", @@ -696,9 +702,14 @@ class EnvironmentActions(OpenJDModel_v2023_09): "|WrappedAction.Cancelation.NotifyPeriodInSeconds", } _template_field_inject = { - "onWrapEnvEnter": _WRAPPED_ACTION_SYMBOLS | {"|WrappedEnv.Name"}, - "onWrapEnvExit": _WRAPPED_ACTION_SYMBOLS | {"|WrappedEnv.Name"}, - "onWrapTaskRun": _WRAPPED_ACTION_SYMBOLS | {"|WrappedStep.Name"}, + "onWrapEnvEnter": _WRAPPED_ACTION_SYMBOLS, + "onWrapEnvExit": _WRAPPED_ACTION_SYMBOLS, + "onWrapTaskRun": _WRAPPED_ACTION_SYMBOLS, + } + _template_field_inject_session = { + "onWrapEnvEnter": {"|WrappedEnv.Name"}, + "onWrapEnvExit": {"|WrappedEnv.Name"}, + "onWrapTaskRun": {"|WrappedStep.Name"}, } @model_validator(mode="before") @@ -1040,6 +1051,17 @@ class SimpleAction(OpenJDModel_v2023_09): "args": {"__self__"}, } + # Parity with Action: `timeout` and `cancelation` VALIDATE at template + # scope (no Session.*, no Env.File.*/Task.File.*, no host-context + # functions) even though the desugared SimpleAction is otherwise + # TASK-scoped. The desugared form is an ordinary Action, whose identical + # fields validate at template scope — matching openjd-rs + # (format_strings.rs validates these fields against the template symtab). + _template_field_scopes = { + "timeout": ResolutionScope.TEMPLATE, + "cancelation": ResolutionScope.TEMPLATE, + } + @field_validator("let") @classmethod def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: @@ -1243,6 +1265,22 @@ def _validate_range_len(cls, value: Any) -> Any: # resolve-time checks in create_job (ranges.rs). return validate_task_param_range_list_len(value) + @field_validator("range") + @classmethod + def _validate_no_empty_path_values(cls, value: Any, info: ValidationInfo) -> Any: + # §3.4.2: an empty string is not a valid path on any OS, so a PATH + # task parameter's range may not contain one. The template model's + # parse-time validator cannot see values that arrive via RFC 0006 + # typed whole-field resolution (e.g. `range: "{{Param.Paths}}"` with + # a LIST[PATH] job parameter), so the check is enforced on the + # instantiation target as well — matching openjd-rs's resolve-time + # check in create_job (ranges.rs). + if info.data.get("type") == TaskParameterType.PATH and isinstance(value, list): + for i, item in enumerate(value): + if str(item) == "": + raise ValueError(f"range[{i}] must not be an empty string.") + return value + class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): # element type of items in the range diff --git a/test/openjd/model_v0/test_pickle.py b/test/openjd/model_v0/test_pickle.py new file mode 100644 index 00000000..127fb99d --- /dev/null +++ b/test/openjd/model_v0/test_pickle.py @@ -0,0 +1,165 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Regression tests: Jobs carrying deferred (run-time-resolved) FormatStrings +must be picklable and copyable. + +FormatString.__new__ requires a keyword-only parsing context, but +str.__getnewargs__ supplies only the string value, so pickle.loads, +copy.copy/deepcopy, and model_copy(deep=True) raised TypeError — and +subclasses that default the context (e.g. ArgString) silently re-parsed +EXPR-grammar strings with the legacy parser (or failed pickling on the +engine-backed parse tree). FormatString now records a parse-context snapshot +and reconstructs through __getnewargs_ex__, re-parsing under the same grammar +(see FormatString.__getstate__/__getnewargs_ex__). +""" + +import copy +import pickle + +from openjd.model import SymbolTable, create_job, decode_job_template +from openjd.model._format_strings import FormatString +from openjd.model._types import ParameterValue, ParameterValueType +from openjd.model.v2023_09 import ArgString, ModelParsingContext + + +def _job_with_deferred_fields(extensions: list[str], *, timeout: str): + """An instantiated Job whose onRun action carries a deferred (unresolved) + FormatString timeout and a deferred cancelation (mode + notifyPeriod).""" + template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": list(extensions), + "name": "PickleTest", + "parameterDefinitions": [ + {"name": "TO", "type": "INT", "default": 30}, + {"name": "Mode", "type": "STRING", "default": "TERMINATE"}, + ], + "steps": [ + { + "name": "Step1", + "script": { + "actions": { + "onRun": { + "command": "echo", + "timeout": timeout, + "cancelation": { + "mode": "{{Param.Mode}}", + "notifyPeriodInSeconds": "{{Param.TO}}", + }, + } + } + }, + } + ], + }, + supported_extensions=extensions, + ) + return create_job( + job_template=template, + job_parameter_values={ + "TO": ParameterValue(type=ParameterValueType.INT, value="30"), + "Mode": ParameterValue(type=ParameterValueType.STRING, value="TERMINATE"), + }, + ) + + +def _deferred_symtab() -> SymbolTable: + symtab = SymbolTable() + symtab["Param.TO"] = 30 + symtab["Param.Mode"] = "TERMINATE" + return symtab + + +def _assert_deferred_fields_equivalent(original, reconstructed) -> None: + """The reconstructed action's deferred FormatStrings must be FormatStrings + that evaluate identically to the originals against the same symtab.""" + symtab = _deferred_symtab() + for attr_path in ("timeout", "cancelation.mode", "cancelation.notifyPeriodInSeconds"): + orig_value = original + recon_value = reconstructed + for part in attr_path.split("."): + orig_value = getattr(orig_value, part) + recon_value = getattr(recon_value, part) + assert isinstance(recon_value, FormatString), attr_path + assert type(recon_value) is type(orig_value), attr_path + assert str(recon_value) == str(orig_value), attr_path + assert recon_value.resolve(symtab=symtab) == orig_value.resolve(symtab=symtab), attr_path + + +class TestJobPickleRoundTrip: + def test_expr_job_pickle_round_trip(self) -> None: + # GIVEN an EXPR + FEATURE_BUNDLE_1 job with deferred timeout and + # cancelation, where the timeout uses EXPR-only arithmetic (the + # legacy grammar cannot parse it — a reconstruction under a default + # context would fail or change semantics). + job = _job_with_deferred_fields(["EXPR", "FEATURE_BUNDLE_1"], timeout="{{ Param.TO * 2 }}") + + # WHEN + reconstructed = pickle.loads(pickle.dumps(job)) + + # THEN + original_action = job.steps[0].script.actions.onRun + recon_action = reconstructed.steps[0].script.actions.onRun + _assert_deferred_fields_equivalent(original_action, recon_action) + symtab = _deferred_symtab() + assert recon_action.timeout.resolve(symtab=symtab) == "60" + + def test_legacy_fb1_job_pickle_round_trip(self) -> None: + # GIVEN a legacy (non-EXPR) FEATURE_BUNDLE_1 job with a deferred + # FormatString timeout. + job = _job_with_deferred_fields(["FEATURE_BUNDLE_1"], timeout="{{Param.TO}}") + + # WHEN + reconstructed = pickle.loads(pickle.dumps(job)) + + # THEN + original_action = job.steps[0].script.actions.onRun + recon_action = reconstructed.steps[0].script.actions.onRun + _assert_deferred_fields_equivalent(original_action, recon_action) + symtab = _deferred_symtab() + assert recon_action.timeout.resolve(symtab=symtab) == "30" + + def test_expr_job_deepcopy_and_model_copy(self) -> None: + # GIVEN + job = _job_with_deferred_fields(["EXPR", "FEATURE_BUNDLE_1"], timeout="{{ Param.TO * 2 }}") + original_action = job.steps[0].script.actions.onRun + + # WHEN / THEN: both deep-copy paths reconstruct working FormatStrings. + for job_copy in (copy.deepcopy(job), job.model_copy(deep=True)): + copied_action = job_copy.steps[0].script.actions.onRun + _assert_deferred_fields_equivalent(original_action, copied_action) + + +class TestFormatStringPickleFaithfulness: + """The reconstruction context must reproduce the original parse mode: + re-parsing an EXPR-grammar string under a default (legacy) context would + reject it outright or silently change its evaluation semantics.""" + + def test_expr_arg_string_typed_round_trip(self) -> None: + # GIVEN an ArgString whose whole-field EXPR list expression only + # parses under the EXPR grammar. + context = ModelParsingContext(supported_extensions=["EXPR", "FEATURE_BUNDLE_1"]) + original = ArgString('{{ ["a","b c"] }}', context=context) + + # WHEN + reconstructed = pickle.loads(pickle.dumps(original)) + + # THEN: still an EXPR whole-field expression with identical typed + # (RFC 0005) evaluation, not a legacy re-parse. + assert type(reconstructed) is ArgString + assert reconstructed.whole_field_expression() is not None + symtab = SymbolTable() + original_value = original.resolve_value(symtab=symtab) + reconstructed_value = reconstructed.resolve_value(symtab=symtab) + assert type(reconstructed_value) is type(original_value) + assert str(reconstructed_value) == str(original_value) + assert reconstructed.resolve(symtab=symtab) == original.resolve(symtab=symtab) + + def test_legacy_format_string_copy(self) -> None: + # A plain legacy-grammar FormatString survives copy.copy and deepcopy. + context = ModelParsingContext() + original = FormatString("prefix {{ Task.Param.Frame }} suffix", context=context) + symtab = SymbolTable() + symtab["Task.Param.Frame"] = 7 + for reconstructed in (copy.copy(original), copy.deepcopy(original)): + assert str(reconstructed) == str(original) + assert reconstructed.resolve(symtab=symtab) == "prefix 7 suffix" diff --git a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py index b9bf5aea..a4c7b6ed 100644 --- a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py +++ b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py @@ -95,3 +95,64 @@ def test_type_mismatch_rejected_at_decode(self): # catches it (it previously slipped through name-only validation). with pytest.raises(DecodeValidationError, match=r"range_expr"): self._decode_with_expr("Param.Frames + 1") + + +class TestTypedPathRangeEmptyValues: + """§3.4.2: an empty string is not a valid path on any OS. The template + model rejects literal empty items in a PATH task parameter's range at + parse time, but a range that arrives via RFC 0006 typed whole-field + resolution (``range: "{{Param.Paths}}"`` with a LIST[PATH] job parameter) + is only seen at instantiation — the target model must reject it there, + matching openjd-rs's resolve-time check in create_job (ranges.rs). + """ + + def _create_with_typed_range(self, list_type, task_type, default): + jt = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "extensions": ["EXPR"], + "parameterDefinitions": [{"name": "Items", "type": list_type, "default": default}], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "V", "type": task_type, "range": "{{Param.Items}}"} + ] + }, + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.V}}"]}} + }, + } + ], + }, + supported_extensions=["EXPR"], + ) + return create_job(job_template=jt, job_parameter_values={}) + + def test_empty_path_in_typed_range_rejected(self): + # A LIST[PATH] parameter containing "" must not flow into the + # instantiated Job's PATH task-parameter range. + with pytest.raises(DecodeValidationError, match=r"must not be an empty string"): + self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "", "/b"]) + + def test_valid_paths_in_typed_range_accepted(self): + # Positive polarity: valid paths still instantiate. + job = self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "/b"]) + steps = job.steps + step = steps["S"] if isinstance(steps, dict) else steps[0] + assert step.parameterSpace is not None + tpd = step.parameterSpace.taskParameterDefinitions + tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] + assert [str(v) for v in tp.range] == ["/a", "/b"] + + def test_empty_string_in_typed_string_range_accepted(self): + # No over-rejection: STRING task parameters legitimately allow "". + job = self._create_with_typed_range("LIST[STRING]", "STRING", ["a", "", "b"]) + steps = job.steps + step = steps["S"] if isinstance(steps, dict) else steps[0] + assert step.parameterSpace is not None + tpd = step.parameterSpace.taskParameterDefinitions + tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] + assert [str(v) for v in tp.range] == ["a", "", "b"] diff --git a/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py b/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py index 15ae2c65..1f10aaf9 100644 --- a/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py +++ b/test/openjd/model_v0/v2023_09/test_feature_bundle_1.py @@ -5,7 +5,10 @@ from pydantic import ValidationError from openjd.model import create_job, decode_job_template +from openjd.model._errors import DecodeValidationError +from openjd.model._format_strings import FormatString from openjd.model._parse import _parse_model +from openjd.model._types import ParameterValue, ParameterValueType from openjd.model.v2023_09 import ( Action, ArgString, @@ -414,6 +417,80 @@ def test_simple_action_with_cancelation(self) -> None: assert result.cancelation.notifyPeriodInSeconds == 30 +class TestSimpleActionFieldScopes: + """Parity with Action: a SimpleAction's timeout/cancelation validate at + TEMPLATE scope (no Task.*/Session.* symbols), exactly like the script-form + Action the sugar desugars to. Regression: they used to validate at the + ambient TASK scope, accepting references the desugared form rejects.""" + + @staticmethod + def _template_with_bash_step(bash: dict) -> dict: + return { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["FEATURE_BUNDLE_1"], + "name": "Test", + "parameterDefinitions": [{"name": "TO", "type": "INT", "default": 30}], + "steps": [ + { + "name": "Step1", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Frame", "type": "INT", "range": "1-3"} + ] + }, + "bash": bash, + } + ], + } + + def test_timeout_task_param_rejected(self) -> None: + """A task-parameter reference in a SimpleAction timeout is rejected, + matching the script-form equivalent.""" + template = self._template_with_bash_step( + {"script": "echo hi", "timeout": "{{Task.Param.Frame}}"} + ) + with pytest.raises(DecodeValidationError, match=r"Task\.Param\.Frame does not exist"): + decode_job_template(template=template, supported_extensions=["FEATURE_BUNDLE_1"]) + + def test_timeout_job_param_accepted(self) -> None: + """A job-parameter reference (template scope) is still accepted.""" + template = self._template_with_bash_step({"script": "echo hi", "timeout": "{{Param.TO}}"}) + decode_job_template(template=template, supported_extensions=["FEATURE_BUNDLE_1"]) + + def test_cancelation_notify_period_task_param_rejected(self) -> None: + """cancelation validates at template scope too.""" + template = self._template_with_bash_step( + { + "script": "echo hi", + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": "{{Task.Param.Frame}}", + }, + } + ) + with pytest.raises(DecodeValidationError, match=r"Task\.Param\.Frame does not exist"): + decode_job_template(template=template, supported_extensions=["FEATURE_BUNDLE_1"]) + + def test_cancelation_notify_period_job_param_accepted(self) -> None: + template = self._template_with_bash_step( + { + "script": "echo hi", + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": "{{Param.TO}}", + }, + } + ) + decode_job_template(template=template, supported_extensions=["FEATURE_BUNDLE_1"]) + + def test_script_task_param_still_accepted(self) -> None: + """The SimpleAction's script/args remain TASK-scoped.""" + template = self._template_with_bash_step( + {"script": "echo {{Task.Param.Frame}}", "args": ["{{Task.Param.Frame}}"]} + ) + decode_job_template(template=template, supported_extensions=["FEATURE_BUNDLE_1"]) + + class TestScriptInterpreterSyntaxSugar: """Tests for script interpreter syntax sugar in StepTemplate.""" @@ -535,8 +612,6 @@ class TestCreateJobWithFormatStrings: def test_amount_requirement_min_max_resolved_valid(self) -> None: """Test that resolved min/max values are validated correctly.""" - from openjd.model import create_job, decode_job_template - from openjd.model._types import ParameterValue, ParameterValueType template = decode_job_template( template={ @@ -579,9 +654,6 @@ def test_amount_requirement_min_max_resolved_valid(self) -> None: def test_amount_requirement_min_greater_than_max_resolved_fails(self) -> None: """Test that resolved min > max fails validation.""" - from openjd.model import create_job, decode_job_template - from openjd.model._errors import DecodeValidationError - from openjd.model._types import ParameterValue, ParameterValueType template = decode_job_template( template={ @@ -627,9 +699,6 @@ def test_timeout_and_notify_period_carried_through_unresolved(self) -> None: openjd-rs (job/create_job/instantiate.rs clones timeout+cancelation unresolved into the Job). This is what allows RFC 0008 wrap hooks to forward run-time-only symbols such as WrappedAction.Timeout.""" - from openjd.model import create_job, decode_job_template - from openjd.model._format_strings import FormatString - from openjd.model._types import ParameterValue, ParameterValueType template = decode_job_template( template={ @@ -675,7 +744,6 @@ def test_timeout_and_notify_period_carried_through_unresolved(self) -> None: def test_literal_timeout_and_notify_period_stay_ints_through_create_job(self) -> None: """Literal integer timeout/notifyPeriodInSeconds are unaffected by the run-time deferral of format-string values.""" - from openjd.model import create_job, decode_job_template template = decode_job_template( template={ @@ -846,8 +914,6 @@ def test_job_name_513_chars_with_extension_fails(self) -> None: def test_job_name_format_string_longer_than_limit_succeeds(self) -> None: """Test that format string longer than limit passes if resolved value is shorter.""" - from openjd.model import create_job, decode_job_template - from openjd.model._types import ParameterValue, ParameterValueType # Template name with format string is > 128 chars, but resolved is short long_prefix = "J" * 120 diff --git a/test/openjd/model_v0/v2023_09/test_wrap_actions.py b/test/openjd/model_v0/v2023_09/test_wrap_actions.py index e0400f37..83cfb835 100644 --- a/test/openjd/model_v0/v2023_09/test_wrap_actions.py +++ b/test/openjd/model_v0/v2023_09/test_wrap_actions.py @@ -16,6 +16,7 @@ decode_environment_template, decode_job_template, ) +from openjd.model._format_strings import FormatString _ALL_EXTS = ["TASK_CHUNKING", "REDACTED_ENV_VARS", "FEATURE_BUNDLE_1", "EXPR", "WRAP_ACTIONS"] @@ -265,6 +266,64 @@ def test_wrapped_action_timeout_in_wrap_hook_ok(self): _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"])) +class TestWrapHookTimeoutCancelationScope: + """A wrap hook's timeout/cancelation validate against template symbols + + WrappedAction.* ONLY — WrappedEnv.Name / WrappedStep.Name are session + symbols visible to the hook's command/args but not to its + template-scoped fields, matching openjd-rs + (validate_v2023_09/format_strings.rs). Regression: all per-hook symbols + used to be injected at TEMPLATE scope, so timeout could reference + WrappedEnv.Name.""" + + _FB1_EXTS = ["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"] + + def test_wrapped_env_name_in_hook_timeout_rejected(self): + actions = dict(_ALL_THREE) + actions["onWrapEnvEnter"] = { + "command": "echo", + "timeout": "{{ WrappedEnv.Name }}", + } + with pytest.raises( + DecodeValidationError, match=r"WrappedEnv\.Name does not exist at this location" + ): + _decode(_env_template(actions, extensions=self._FB1_EXTS)) + + def test_wrapped_step_name_in_hook_cancelation_mode_rejected(self): + actions = dict(_ALL_THREE) + actions["onWrapTaskRun"] = { + "command": "echo", + "cancelation": {"mode": "{{ WrappedStep.Name }}"}, + } + with pytest.raises( + DecodeValidationError, match=r"WrappedStep\.Name does not exist at this location" + ): + _decode(_env_template(actions, extensions=self._FB1_EXTS)) + + def test_wrapped_env_name_in_hook_args_still_accepted(self): + # command/args validate at the ambient session scope, where + # WrappedEnv.Name / WrappedStep.Name remain visible. + actions = dict(_ALL_THREE) + actions["onWrapEnvExit"] = { + "command": "echo", + "args": ["{{ WrappedEnv.Name }}"], + } + _decode(_env_template(actions, extensions=self._FB1_EXTS)) + + def test_wrapped_action_round_trip_forwarding_still_accepted(self): + # The round-trip forwarding case must not regress: WrappedAction.* + # stays visible to the hook's timeout/cancelation. + actions = dict(_ALL_THREE) + actions["onWrapTaskRun"] = { + "command": "{{ WrappedAction.Command }}", + "timeout": "{{ WrappedAction.Timeout }}", + "cancelation": { + "mode": "{{ WrappedAction.Cancelation.Mode }}", + "notifyPeriodInSeconds": "{{ WrappedAction.Cancelation.NotifyPeriodInSeconds }}", + }, + } + _decode(_env_template(actions, extensions=self._FB1_EXTS)) + + # ── RFC 0008 create_job instantiation ────────────────────────────────── # # create_job re-validates the instantiated Job submodels without a parsing @@ -304,8 +363,6 @@ def test_create_job_with_full_forwarding_defers_runtime_fields(self): # timeout+cancelation unresolved into the Job), the fields must be # carried through job instantiation as raw FormatStrings and left for # the session to resolve. - from openjd.model._format_strings import FormatString - hook = { "command": "{{WrappedAction.Command}}", "args": ["{{WrappedAction.Args}}"], From e02b013edeee1cef8f50bf76b1db38e2641ab47a Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:24:22 -0700 Subject: [PATCH 10/13] fix: Expect native path separators in Windows typed PATH range test The typed-range positive-polarity test asserted the literal POSIX strings '/a' and '/b', but PATH values resolved through the EXPR surface are separator-normalized to the host's native form (the documented Path invariant in openjd-rs value.rs), yielding '\\a' and '\\b' on Windows. Assert the platform-appropriate expected values, following the existing convention in test_lists.py. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../model_v0/v2023_09/test_create_job_expr_params.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py index a4c7b6ed..7cbb69b8 100644 --- a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py +++ b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py @@ -9,6 +9,8 @@ symbol table can coerce them. """ +import sys + import pytest from openjd.model import ( @@ -145,7 +147,10 @@ def test_valid_paths_in_typed_range_accepted(self): assert step.parameterSpace is not None tpd = step.parameterSpace.taskParameterDefinitions tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] - assert [str(v) for v in tp.range] == ["/a", "/b"] + # PATH range values are normalized to the host's native separator + # (same convention as test_lists.py's path coercion tests). + expected = ["\\a", "\\b"] if sys.platform == "win32" else ["/a", "/b"] + assert [str(v) for v in tp.range] == expected def test_empty_string_in_typed_string_range_accepted(self): # No over-rejection: STRING task parameters legitimately allow "". From ca44376cae3ee55e71e0f9e356d84022e4453c7a Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:29:36 -0700 Subject: [PATCH 11/13] fix: Type-faithful ranges, PATH symbol contracts, per-step typed validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three parity fixes against the openjd-rs reference and the spec, from an independent external review (all three confirmed with two-sided repros): - Typed whole-field range resolution now preserves engine element type variants and enforces element/target agreement via a per-model coercion hook (JobCreationMetadata.typed_resolve_coerce), mirroring openjd-rs's per-variant checks in create_job (ranges.rs): INT ranges accept only int elements ("Expected int in range, got bool"), FLOAT accepts int|float, and STRING/PATH ranges take each element's spec display form (bool -> true/false, nested list -> "[1, 2]"). Previously the engine list was unwrapped with .item(), so a LIST[BOOL] parameter silently produced 1/0 task values that the Rust implementation rejects. - PATH and LIST[PATH] symbol contracts now follow RFC 0005 "Job Parameter Types" and Template Schemas 2.12/7.3.1: LIST[PATH] gets the scalar-PATH scoping (processed Param.* at SESSION scope only; RawParam.* at TEMPLATE scope), create_job no longer seeds a template-scope Param.* for LIST[PATH], and RawParam.* for path types is typed string/list[string] (Rust: build_template_scope_symtab, parameters.rs build_symbol_table). Fixes both the false accept ("{{ Param.Paths[0].name }}" in a job name) and the false reject ("{{ RawParam.File.upper() }}"). - Type-aware expression validation is now threaded per defining model through the traversal (ScopedSymtabs.types, populated from each definition's sibling `type` field with raw-variable string semantics via TemplateVariableDef.raw), replacing the single root-level job-parameter map. Task.Param.*/Task.RawParam.* references are now type-checked, with per-step separation for same-named task parameters of different types, matching openjd-rs's build_task_scope_symtab; conflicting merged types drop to name-only validation rather than last-writer-wins. CHUNK[INT] stays untyped (deliberate: Python's session runtime skips it; Rust types it RANGE_EXPR — parity question tracked in a code comment). Tests: 37 new/updated across test_typed_symbol_contracts.py and test_create_job_expr_params.py (element matrix, scope contracts, per-step localization, raw-string method access, name-only fallback preservation). Suite: 5437 passed; coverage 94%; mypy/black/ruff clean; sessions suite and conformance spot-runs (2.12/2.13/2.16/3.4 fixtures) unchanged. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/_create_job.py | 19 +- src/openjd/model/_internal/_create_job.py | 29 ++- .../_variable_reference_validation.py | 162 ++++++++++----- src/openjd/model/_types.py | 18 ++ src/openjd/model/v2023_09/_model.py | 121 ++++++++++- .../v2023_09/test_create_job_expr_params.py | 126 +++++++++++- .../v2023_09/test_typed_symbol_contracts.py | 191 ++++++++++++++++++ 7 files changed, 596 insertions(+), 70 deletions(-) create mode 100644 test/openjd/model_v0/v2023_09/test_typed_symbol_contracts.py diff --git a/src/openjd/model/_create_job.py b/src/openjd/model/_create_job.py index d4e0f302..1ae7c585 100644 --- a/src/openjd/model/_create_job.py +++ b/src/openjd/model/_create_job.py @@ -405,20 +405,27 @@ def create_job( # during EXPR expression evaluation — matching openjd-rs, whose # create_job symbol table carries typed ExprValues for every # parameter (an INT param is an int, so `{{ Param.X + 3 }}` works). - # STRING needs no coercion, and PATH stays string-typed in template - # scope: openjd-rs excludes Param.* for PATH here (host-context only) - # and seeds RawParam.* for PATH as a plain string. + # STRING needs no coercion. Path-typed parameters (PATH and + # LIST[PATH]) have no template-scope Param.* symbol at all: the + # processed value (with path mapping) only exists at host/session + # scope, so openjd-rs excludes Param.* for both here and seeds only + # RawParam.* — as a plain string for PATH and a list[string] for + # LIST[PATH] (RFC 0005 "Job Parameter Types"; parameters.rs + # build_symbol_table). # The typing only affects the EXPR evaluation path; non-EXPR # templates keep their existing string-based interpolation. expr_types: dict[str, str] = {} for name, param in all_job_parameter_values.items(): prefix = ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value raw_prefix = ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value - if param.type != "PATH": + if param.type.name not in ("PATH", "LIST_PATH"): symtab[f"{prefix}.{name}"] = all_job_parameter_values[name].value + if param.type.name != "STRING": + expr_types[f"{prefix}.{name}"] = param.type.value symtab[f"{raw_prefix}.{name}"] = all_job_parameter_values[name].value - if param.type.name not in ("STRING", "PATH"): - expr_types[f"{prefix}.{name}"] = param.type.value + if param.type.name == "LIST_PATH": + expr_types[f"{raw_prefix}.{name}"] = ParameterValueType.LIST_STRING.value + elif param.type.name not in ("STRING", "PATH"): expr_types[f"{raw_prefix}.{name}"] = param.type.value symtab.expr_types.update(expr_types) # RFC 0007 §7.3.1 (EXPR): Job.Name is the job's resolved name, diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index f89b4595..23c823b9 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -50,14 +50,27 @@ def resolve_whole_field_typed_list(value: FormatString, symtab: SymbolTable) -> # ExpressionError/FormatStringError (both ValueError subclasses): # fall back to string resolution, which re-raises with context. return None - # EXPR-backed evaluation returns the engine's ExprValue; unwrap lists. + # EXPR-backed evaluation returns the engine's ExprValue. List results are + # returned RAW (not unwrapped) so the element type variants survive to the + # coercion hook (JobCreationMetadata.typed_resolve_coerce) — unwrapping + # via .item() erases them (a bool becomes int 1/0), which is how typed + # ranges diverged from openjd-rs's per-variant checks (ranges.rs). if _expr_value_is_list(result): - return result.item() + return result if isinstance(result, list): return result return None +def unwrap_typed_resolution(raw: Any) -> Any: + """Unwrap a typed whole-field resolution result to native Python: the + engine's list ``ExprValue`` becomes a plain list; native results pass + through. Used when a model defines no ``typed_resolve_coerce`` hook.""" + if _expr_value_is_list(raw): + return raw.item() + return raw + + def _compute_typed_resolutions(model: "OpenJDModel", symtab: SymbolTable) -> dict[str, Any]: """RFC 0006 typed whole-field resolutions for the model's ``typed_resolve_fields``, computed exactly once per field. @@ -201,8 +214,16 @@ def instantiate_model( # noqa: C901 # Instantiate and resolve format string expressions needs_resolve = field_name in model._job_creation_metadata.resolve_fields if field_name in typed_resolved: - # RFC 0006 typed whole-field resolution (computed above). - instantiated = typed_resolved[field_name] + # RFC 0006 typed whole-field resolution (computed above). The + # model's coercion hook (when set) enforces element/target + # type agreement on the engine's typed elements — errors + # raised here aggregate into the model's validation errors. + raw_typed = typed_resolved[field_name] + coerce = model._job_creation_metadata.typed_resolve_coerce + if coerce is not None: + instantiated = coerce(model, field_name, raw_typed) + else: + instantiated = unwrap_typed_resolution(raw_typed) elif isinstance(field_value, list): if field_name in model._job_creation_metadata.reshape_field_to_dict: key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index b98df6c5..78c2545a 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -126,13 +126,46 @@ class ScopedSymtabs(defaultdict): def __init__(self, **kwargs: Any) -> None: super().__init__(set, **kwargs) + # Symbol name -> EXPR type string (e.g. "Param.X" -> "int") for + # type-aware expression validation (EXPR extension). Populated from + # each defining model's sibling `type` field as the definitions are + # collected, and threaded through the traversal alongside the symbol + # sets — so each step's Task.Param.* types stay local to that step, + # mirroring openjd-rs's per-step symtabs (format_strings.rs + # build_task_scope_symtab). Names without a confident type mapping + # are absent; expressions touching them fall back to name-only checks. + self.types: dict[str, str] = {} + # Names whose merged type definitions conflicted (same symbol defined + # with different types). Dropped from `types` so the validator falls + # back to name-only checking rather than validating against an + # arbitrary winner. + self._type_conflicts: set[str] = set() def update_self(self, other: "ScopedSymtabs") -> "ScopedSymtabs": """Union the other's contents into self.""" for k, v in other.items(): self[k] |= v + for name in other._type_conflicts: + self._mark_type_conflict(name) + for name, expr_type in other.types.items(): + self.set_type(name, expr_type) return self + def set_type(self, name: str, expr_type: str) -> None: + """Record a symbol's EXPR type; conflicting definitions drop the + entry (name-only fallback) rather than last-writer-wins.""" + if name in self._type_conflicts: + return + existing = self.types.get(name) + if existing is not None and existing != expr_type: + self._mark_type_conflict(name) + return + self.types[name] = expr_type + + def _mark_type_conflict(self, name: str) -> None: + self._type_conflicts.add(name) + self.types.pop(name, None) + def prevalidate_model_template_variable_references( cls: Type[OpenJDModel], @@ -167,10 +200,6 @@ def prevalidate_model_template_variable_references( ) if context is not None: context._prevalidation_expr_enabled = expr_enabled # type: ignore[attr-defined] - # Build a map of job-parameter symbol -> EXPR type (for type-aware - # expression validation). Only job parameters are mapped; expressions - # referencing other symbols fall back to name-only validation. - context._symbol_types = _build_job_parameter_symbol_types(values) if expr_enabled else {} # type: ignore[attr-defined] return _validate_model_template_variable_references( cls, @@ -369,18 +398,27 @@ def _validate_model_template_variable_references( # If the node doesn't modify the variable prefix, then symbol_prefix will be the empty string symbol_prefix += variable_defs.symbol_prefix - # Recursively collect all of the variable definitions at this node and its child nodes. + expr_enabled = bool( + context is not None and getattr(context, "_prevalidation_expr_enabled", False) + ) + + # Recursively collect all of the variable definitions at this node and its + # child nodes. Symbol EXPR types are collected only when the EXPR + # extension is active, keeping the Rust expr surface off the non-EXPR + # parse path. value_symbols = _collect_variable_definitions( - model, value, current_scope, symbol_prefix, recursive_pruning=False + model, + value, + current_scope, + symbol_prefix, + recursive_pruning=False, + collect_types=expr_enabled, ) # Validate EXPR `let` bindings defined by this model: each binding's RHS # expression is checked against the enclosing scope plus the bindings that # precede it (chained references), and a binding may not shadow a variable # from the enclosing scope (RFC 0007 §3.6). - expr_enabled = bool( - context is not None and getattr(context, "_prevalidation_expr_enabled", False) - ) if expr_enabled: # EXPR-conditional symbol injection (e.g. Step.Name). Added to the # model's own scope so it flows to fields whose sources include @@ -459,33 +497,6 @@ def _validate_model_template_variable_references( _HOST_CONTEXT_FUNCTIONS = {"apply_path_mapping"} -def _build_job_parameter_symbol_types(values: dict) -> dict: - """Map job-parameter symbols (``Param.`` and ``RawParam.``) to - their EXPR type strings, derived from the template's ``parameterDefinitions``. - Used for type-aware expression validation; parameters whose type has no - confident EXPR mapping are omitted (callers fall back to name-only checks). - """ - from openjd.model._format_strings._expr_support import expr_type_for_openjd_type - - result: dict[str, str] = {} - defs = values.get("parameterDefinitions") - if not isinstance(defs, list): - return result - for entry in defs: - if not isinstance(entry, dict): - continue - name = entry.get("name") - type_name = entry.get("type") - if not isinstance(name, str) or not isinstance(type_name, str): - continue - expr_type = expr_type_for_openjd_type(type_name) - if expr_type is None: - continue - result[f"Param.{name}"] = expr_type - result[f"RawParam.{name}"] = expr_type - return result - - def _check_format_string( value: FormatString, current_scope: ResolutionScope, @@ -506,7 +517,12 @@ def _check_format_string( # Improperly formed string. Later validation passes will catch and flag this. return errors - symbol_types = getattr(context, "_symbol_types", None) if context is not None else None + # Symbol EXPR types threaded through the traversal alongside the symbol + # sets (populated only when the EXPR extension is active). Per-step typed + # symbols (Task.Param.*/Task.RawParam.*) are visible only within their + # own step, so same-named task parameters with different types across + # steps validate independently. + symbol_types = symbols.types or None for expr in f_value.expressions: if expr.expression: try: @@ -713,6 +729,39 @@ def _get_model_for_singleton_value( ## ============================================= +def _symbol_expr_type(value: dict, raw: bool) -> Optional[str]: + """The EXPR type string for a symbol defined by a model whose raw + ``value`` dict carries a sibling ``type`` field (e.g. a job- or + task-parameter definition), or ``None`` when no confident mapping exists. + + When ``raw`` is set (RawParam.*/Task.RawParam.*), path types map to their + raw string forms — a raw PATH is a plain string and a raw LIST[PATH] a + list[string] (RFC 0005; openjd-rs build_param_symtab). + + ``expr_type_for_openjd_type`` returns ``None`` for types with no + confident EXPR mapping — notably CHUNK[INT], which openjd-rs types as + RANGE_EXPR in its task-scope symtab (format_strings.rs + build_task_scope_symtab). It is deliberately omitted here (name-only + fallback) because Python's session runtime also skips typing CHUNK task + parameters; typing it at validation without runtime support would be a + Rust-parity question to revisit alongside the runtime. + """ + type_field_value = value.get("type") + if not isinstance(type_field_value, str): + return None + # Imported lazily: only reached when the EXPR extension is active, so the + # non-EXPR parse path never loads the Rust expr surface. + from openjd.model._format_strings._expr_support import expr_type_for_openjd_type + + expr_type = expr_type_for_openjd_type(type_field_value) + if raw: + if expr_type == "path": + expr_type = "string" + elif expr_type == "list[path]": + expr_type = "list[string]" + return expr_type + + def _collect_variable_definitions( # noqa: C901 (suppress: too complex) model: Type, value: Any, @@ -720,6 +769,7 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) symbol_prefix: str, recursive_pruning: bool = True, discriminator: Union[str, Discriminator, None] = None, + collect_types: bool = False, ) -> dict[str, ScopedSymtabs]: """Collects the names of variables that each field of this model object provides. @@ -743,6 +793,7 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) current_scope, symbol_prefix, discriminator=discriminator, + collect_types=collect_types, ) # Unwrap the Annotated type, and get the discriminator while doing so @@ -752,7 +803,12 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) if isinstance(annotation, FieldInfo): discriminator = annotation.discriminator return _collect_variable_definitions( - model_args[0], value, current_scope, symbol_prefix, discriminator=discriminator + model_args[0], + value, + current_scope, + symbol_prefix, + discriminator=discriminator, + collect_types=collect_types, ) # Aggregate all the collected variable definitions from a list @@ -764,9 +820,9 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) item_model = typing.get_args(model)[0] for item in value: symtab.update_self( - _collect_variable_definitions(item_model, item, current_scope, symbol_prefix)[ - "__export__" - ] + _collect_variable_definitions( + item_model, item, current_scope, symbol_prefix, collect_types=collect_types + )["__export__"] ) return {"__export__": symtab} @@ -775,9 +831,9 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) symtab = ScopedSymtabs() for sub_type in typing.get_args(model): symtab.update_self( - _collect_variable_definitions(sub_type, value, current_scope, symbol_prefix)[ - "__export__" - ] + _collect_variable_definitions( + sub_type, value, current_scope, symbol_prefix, collect_types=collect_types + )["__export__"] ) return {"__export__": symtab} @@ -790,6 +846,7 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) value, current_scope, symbol_prefix, + collect_types=collect_types, ) else: return {"__export__": ScopedSymtabs()} @@ -837,6 +894,16 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) else: symbol_name = f"{symbol_prefix}{vardef.prefix}{name}" _add_symbol(symbols["__self__"], vardef.resolves, symbol_name) + if collect_types: + # Record the symbol's EXPR type from the defining + # model's sibling `type` field (job- and task-parameter + # definitions), honoring the raw-variable string + # semantics (RFC 0005) — per-step Task.Param.* types + # stay local to their step because they ride the same + # per-field symbol threading as the names. + symbol_type = _symbol_expr_type(value, vardef.raw) + if symbol_type is not None: + symbols["__self__"].set_type(symbol_name, symbol_type) # If this object injects any template variables then those are injected at the # current model's scope. @@ -870,7 +937,12 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) discriminator = field_info.discriminator symbols[field_name] = _collect_variable_definitions( - field_model, field_value, current_scope, symbol_prefix, discriminator=discriminator + field_model, + field_value, + current_scope, + symbol_prefix, + discriminator=discriminator, + collect_types=collect_types, )["__export__"] # Collect the exported symbols as specified by the metadata diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index f731ccac..e4174f66 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -197,6 +197,11 @@ class TemplateVariableDef: prefix: str resolves: ResolutionScope + # Whether the variable holds the raw, unprocessed template value + # (RawParam.* / Task.RawParam.*). Raw path-typed values are plain strings + # (RFC 0005): type-aware validation maps a raw PATH to "string" and a raw + # LIST[PATH] to "list[string]", matching openjd-rs's build_param_symtab. + raw: bool = False class DefinesTemplateVariables: @@ -284,6 +289,19 @@ class JobCreationMetadata: is shared with the ``create_as`` callable (see JobCreateAsMetadata). """ + typed_resolve_coerce: Optional[Callable[["OpenJDModel", str, Any], Any]] = field(default=None) + """Optional element-coercion hook for typed whole-field resolutions + (RFC 0006). When set, ``instantiate_model`` calls it for each field whose + typed resolution succeeded — arguments are the model, the field name, and + the raw resolution result (the engine's list ``ExprValue`` with element + type variants preserved, or a native Python list on the non-engine + fallback path) — and uses its return value as the instantiated field + value. The hook may raise ``ValueError`` to reject an element (e.g. a + bool in an INT task-parameter range); the error is aggregated into the + model's validation errors. When unset, the raw result is unwrapped to a + native Python list unchanged. + """ + create_as: Optional[JobCreateAsMetadata] = field(default=None) """The class to create the model as during instantiation. Options: 1. None -- create as the model class itself. diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index e9ff6386..2e0432b2 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1330,6 +1330,85 @@ def _range_task_param_target(model: Any, typed_values: dict) -> Type[OpenJDModel return RangeListTaskParameterDefinition +def _range_element_type_name(elem: Any) -> str: + """The engine type name of a range element for error messages, in the + shape openjd-rs uses (nested lists report as plain "list").""" + type_str = str(getattr(elem, "type", type(elem).__name__)) + return "list" if type_str.startswith("list[") else type_str + + +def _coerce_typed_range_elements(model: Any, field_name: str, raw: Any) -> list: + """``typed_resolve_coerce`` hook for task-parameter ``range`` fields. + + Enforces element/target type agreement on RFC 0006 typed whole-field + range resolutions, mirroring openjd-rs's per-variant checks in create_job + (ranges.rs): an INT (or CHUNK[INT]) range accepts only int elements, a + FLOAT range accepts int or float, and STRING/PATH ranges take every + element's spec display form (a bool renders ``true``/``false``, a nested + list ``[1, 2]``) — without this, unwrapping the engine list erases the + variants and e.g. a LIST[BOOL] parameter silently becomes 1/0 task values + that the Rust implementation rejects outright. + + ``raw`` is the engine's list ``ExprValue`` (elements keep their typed + variants) or, on the non-engine fallback path, a native Python list. + """ + target = model.type + out: list = [] + for elem in raw: + elem_type = getattr(elem, "type", None) + if elem_type is not None: + # Engine element ExprValue. + type_str = str(elem_type) + if target in (TaskParameterType.INT, TaskParameterType.CHUNK_INT): + if type_str != "int": + raise ValueError(f"Expected int in range, got {_range_element_type_name(elem)}") + out.append(elem.item()) + elif target == TaskParameterType.FLOAT: + if type_str not in ("int", "float"): + raise ValueError( + f"Expected float in range, got {_range_element_type_name(elem)}" + ) + out.append(elem.item()) + else: # STRING / PATH: spec display coercion. + out.append(str(elem)) + else: + # Native Python element (non-engine fallback). bool subclasses + # int, so it is checked first. + if target in (TaskParameterType.INT, TaskParameterType.CHUNK_INT): + if isinstance(elem, bool) or not isinstance(elem, int): + raise ValueError( + f"Expected int in range, got {_native_element_type_name(elem)}" + ) + out.append(elem) + elif target == TaskParameterType.FLOAT: + if isinstance(elem, bool) or not isinstance(elem, (int, float)): + raise ValueError( + f"Expected float in range, got {_native_element_type_name(elem)}" + ) + out.append(elem) + else: + if isinstance(elem, bool): + out.append("true" if elem else "false") + else: + out.append(str(elem)) + return out + + +def _native_element_type_name(elem: Any) -> str: + """Engine-style type name for a native Python range element.""" + if isinstance(elem, bool): + return "bool" + if isinstance(elem, int): + return "int" + if isinstance(elem, float): + return "float" + if isinstance(elem, str): + return "string" + if isinstance(elem, list): + return "list" + return type(elem).__name__ + + def _validate_int_range_elements(value: Any) -> Any: """Shared ``range`` post-validator for the INT and CHUNK[INT] task-parameter definitions: a literal range-expression string must parse @@ -1371,7 +1450,7 @@ class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Task.Param.", resolves=ResolutionScope.TASK), - TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK), + TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK, raw=True), }, field="name", ) @@ -1381,6 +1460,7 @@ class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 create_as=JobCreateAsMetadata(callable=_range_task_param_target), resolve_fields={"range"}, typed_resolve_fields={"range"}, + typed_resolve_coerce=_coerce_typed_range_elements, exclude_fields={"name"}, ) @@ -1439,7 +1519,7 @@ class FloatTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_ _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Task.Param.", resolves=ResolutionScope.TASK), - TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK), + TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK, raw=True), }, field="name", ) @@ -1448,6 +1528,7 @@ class FloatTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_ create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, typed_resolve_fields={"range"}, + typed_resolve_coerce=_coerce_typed_range_elements, exclude_fields={"name"}, ) @@ -1484,7 +1565,7 @@ class StringTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023 _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Task.Param.", resolves=ResolutionScope.TASK), - TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK), + TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK, raw=True), }, field="name", ) @@ -1493,6 +1574,7 @@ class StringTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023 create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, typed_resolve_fields={"range"}, + typed_resolve_coerce=_coerce_typed_range_elements, exclude_fields={"name"}, ) @@ -1518,7 +1600,7 @@ class PathTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_0 _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Task.Param.", resolves=ResolutionScope.TASK), - TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK), + TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK, raw=True), }, field="name", ) @@ -1527,6 +1609,7 @@ class PathTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_0 create_as=JobCreateAsMetadata(model=RangeListTaskParameterDefinition), resolve_fields={"range"}, typed_resolve_fields={"range"}, + typed_resolve_coerce=_coerce_typed_range_elements, exclude_fields={"name"}, ) @@ -1571,7 +1654,7 @@ class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v20 _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Task.Param.", resolves=ResolutionScope.TASK), - TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK), + TemplateVariableDef(prefix="|Task.RawParam.", resolves=ResolutionScope.TASK, raw=True), }, field="name", ) @@ -1581,6 +1664,7 @@ class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v20 create_as=JobCreateAsMetadata(callable=_range_task_param_target), resolve_fields={"range"}, typed_resolve_fields={"range"}, + typed_resolve_coerce=_coerce_typed_range_elements, exclude_fields={"name"}, ) @@ -1999,7 +2083,7 @@ class JobStringParameterDefinition( _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -2246,7 +2330,7 @@ class JobPathParameterDefinition( _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.SESSION), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -2474,7 +2558,7 @@ class JobIntParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09) _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -2751,7 +2835,7 @@ def _validate_decimals_requires_spin_box(self) -> Self: _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -3658,7 +3742,7 @@ class JobBoolParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 _template_variable_definitions = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -3851,7 +3935,7 @@ def _normalize_parameter_type_case(value: Any, info: ValidationInfo) -> Any: _LIST_PARAM_VARS = DefinesTemplateVariables( defines={ TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), - TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), }, field="name", ) @@ -3958,6 +4042,21 @@ class JobListPathParameterDefinition(_JobListParameterDefinitionBase): dataFlow: Optional[JobPathParameterDefinitionDataFlow] = None # noqa: N815 item: Optional[_ListStringItemConstraint] = None + # Path-typed parameters follow the scalar PATH scoping contract, not the + # shared _LIST_PARAM_VARS: the processed Param.* value (list[path], with + # path mapping applied) only exists at host/session scope, while the raw + # RawParam.* value is a template-scope list[string] (RFC 0005 "Job + # Parameter Types"; Template Schemas §2.12/§7.3.1). Mirrors openjd-rs's + # build_template_scope_symtab, which excludes Param.* for PATH and + # LIST[PATH] alike. + _template_variable_definitions = DefinesTemplateVariables( + defines={ + TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.SESSION), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE, raw=True), + }, + field="name", + ) + def _check_item(self, item: Any) -> None: _check_string_item(self.name, item, self.item) diff --git a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py index 7cbb69b8..70e71f7c 100644 --- a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py +++ b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py @@ -108,7 +108,7 @@ class TestTypedPathRangeEmptyValues: matching openjd-rs's resolve-time check in create_job (ranges.rs). """ - def _create_with_typed_range(self, list_type, task_type, default): + def _create_with_typed_range(self, list_type, task_type, default, *, ref="Param"): jt = decode_job_template( template={ "specificationVersion": "jobtemplate-2023-09", @@ -120,7 +120,7 @@ def _create_with_typed_range(self, list_type, task_type, default): "name": "S", "parameterSpace": { "taskParameterDefinitions": [ - {"name": "V", "type": task_type, "range": "{{Param.Items}}"} + {"name": "V", "type": task_type, "range": f"{{{{{ref}.Items}}}}"} ] }, "script": { @@ -137,11 +137,13 @@ def test_empty_path_in_typed_range_rejected(self): # A LIST[PATH] parameter containing "" must not flow into the # instantiated Job's PATH task-parameter range. with pytest.raises(DecodeValidationError, match=r"must not be an empty string"): - self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "", "/b"]) + # LIST[PATH] has no template-scope Param.* (RFC 0005): the range + # forwards the raw list[string] value via RawParam, as in Rust. + self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "", "/b"], ref="RawParam") def test_valid_paths_in_typed_range_accepted(self): # Positive polarity: valid paths still instantiate. - job = self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "/b"]) + job = self._create_with_typed_range("LIST[PATH]", "PATH", ["/a", "/b"], ref="RawParam") steps = job.steps step = steps["S"] if isinstance(steps, dict) else steps[0] assert step.parameterSpace is not None @@ -161,3 +163,119 @@ def test_empty_string_in_typed_string_range_accepted(self): tpd = step.parameterSpace.taskParameterDefinitions tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] assert [str(v) for v in tp.range] == ["a", "", "b"] + + +class TestTypedRangeElementSemantics: + """RFC 0006 typed whole-field ranges must preserve element type variants + and enforce element/target agreement, matching openjd-rs's per-variant + checks in create_job (ranges.rs): an INT range accepts only int elements, + a FLOAT range accepts int or float, and STRING/PATH ranges take each + element's spec display form (bool renders true/false, a nested list + "[1, 2]"). Without this, unwrapping the engine list erased the variants — + a LIST[BOOL] parameter silently produced 1/0 task values that the Rust + implementation rejects outright. + """ + + def _create(self, list_type, task_type, default): + jt = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "extensions": ["EXPR"], + "parameterDefinitions": [{"name": "Items", "type": list_type, "default": default}], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "V", "type": task_type, "range": "{{Param.Items}}"} + ] + }, + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.V}}"]}} + }, + } + ], + }, + supported_extensions=["EXPR"], + ) + return create_job(job_template=jt, job_parameter_values={}) + + def _range_of(self, job): + steps = job.steps + step = steps["S"] if isinstance(steps, dict) else steps[0] + tpd = step.parameterSpace.taskParameterDefinitions + tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] + return [str(v) for v in tp.range] + + @pytest.mark.parametrize( + "list_type, task_type, default, message", + [ + pytest.param( + "LIST[BOOL]", + "INT", + [True, False], + r"Expected int in range, got bool", + id="bool-into-int", + ), + pytest.param( + "LIST[BOOL]", + "FLOAT", + [True, False], + r"Expected float in range, got bool", + id="bool-into-float", + ), + pytest.param( + "LIST[STRING]", + "INT", + ["1", "2"], + r"Expected int in range, got string", + id="string-into-int", + ), + pytest.param( + "LIST[FLOAT]", + "INT", + [1.5, 2.5], + r"Expected int in range, got float", + id="float-into-int", + ), + pytest.param( + "LIST[LIST[INT]]", + "INT", + [[1, 2], [3]], + r"Expected int in range, got list", + id="nested-list-into-int", + ), + ], + ) + def test_mismatched_elements_rejected(self, list_type, task_type, default, message): + with pytest.raises(DecodeValidationError, match=message): + self._create(list_type, task_type, default) + + def test_bool_into_string_range_uses_display_form(self): + # Matches Rust to_display_string: true/false, not Python's True/False + # or the int 1/0 the erased-variant path produced. + assert self._range_of(self._create("LIST[BOOL]", "STRING", [True, False])) == [ + "true", + "false", + ] + + def test_nested_list_into_string_range_uses_display_form(self): + assert self._range_of(self._create("LIST[LIST[INT]]", "STRING", [[1, 2], [3]])) == [ + "[1, 2]", + "[3]", + ] + + def test_int_into_float_range_accepted(self): + # Rust's resolve_float_range accepts Int elements. + assert self._range_of(self._create("LIST[INT]", "FLOAT", [1, 2])) == ["1", "2"] + + @pytest.mark.parametrize( + "list_type, task_type, default, expected", + [ + pytest.param("LIST[INT]", "INT", [1, 2, 3], ["1", "2", "3"], id="int-int"), + pytest.param("LIST[STRING]", "STRING", ["a", "b"], ["a", "b"], id="string-string"), + ], + ) + def test_matching_elements_accepted(self, list_type, task_type, default, expected): + assert self._range_of(self._create(list_type, task_type, default)) == expected diff --git a/test/openjd/model_v0/v2023_09/test_typed_symbol_contracts.py b/test/openjd/model_v0/v2023_09/test_typed_symbol_contracts.py new file mode 100644 index 00000000..f321e705 --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_typed_symbol_contracts.py @@ -0,0 +1,191 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""PATH/LIST[PATH] symbol contracts and per-step typed expression validation. + +RFC 0005 "Job Parameter Types": processed ``Param.*`` values for path-typed +parameters are ``path``/``list[path]`` with path mapping applied and exist +only at host/session scope; ``RawParam.*`` values are template-scope +``string``/``list[string]``. Template validation types symbols per defining +model (per step for task parameters), mirroring openjd-rs's +``build_param_symtab``/``build_task_scope_symtab`` (format_strings.rs). +""" + +import pytest + +from openjd.model import DecodeValidationError, create_job, decode_job_template + +EXT = ["EXPR"] +_RUN = {"actions": {"onRun": {"command": "echo"}}} + + +def _decode(t): + return decode_job_template(template=t, supported_extensions=EXT) + + +def _base(**overrides): + t = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": EXT, + "name": "T", + "steps": [{"name": "S", "script": _RUN}], + } + t.update(overrides) + return t + + +def _step_with_task_param(name, tp_type, arg_expr, param_name="File", range_=None): + if range_ is None: + range_ = [1, 2] if tp_type == "INT" else (["/a"] if tp_type == "PATH" else ["a"]) + return { + "name": name, + "parameterSpace": { + "taskParameterDefinitions": [{"name": param_name, "type": tp_type, "range": range_}] + }, + "script": {"actions": {"onRun": {"command": "echo", "args": [arg_expr]}}}, + } + + +class TestPerStepTaskParameterTyping: + """Task.Param.*/Task.RawParam.* references are type-checked per step.""" + + def test_raw_path_property_access_rejected(self): + # Task.RawParam. is a string (raw, unmapped) — path properties + # are unavailable. Rust rejects this at check; previously Python only + # failed at session evaluation. + t = _base(steps=[_step_with_task_param("S", "PATH", "{{ Task.RawParam.File.name }}")]) + with pytest.raises(DecodeValidationError, match=r"'name' property is not available"): + _decode(t) + + def test_processed_path_property_access_accepted(self): + t = _base(steps=[_step_with_task_param("S", "PATH", "{{ Task.Param.File.name }}")]) + _decode(t) + + def test_int_task_param_path_property_rejected(self): + t = _base( + steps=[ + _step_with_task_param("S", "INT", "{{ Task.Param.Frame.name }}", param_name="Frame") + ] + ) + with pytest.raises(DecodeValidationError, match=r"not available for int"): + _decode(t) + + def test_same_name_different_types_validate_per_step(self): + # Two steps define task param "File": PATH in step A (property access + # valid), STRING in step B (invalid). Only step B may error — + # per-step type separation, as in Rust. + t = _base( + steps=[ + _step_with_task_param("A", "PATH", "{{ Task.Param.File.name }}"), + _step_with_task_param("B", "STRING", "{{ Task.Param.File.name }}"), + ] + ) + with pytest.raises(DecodeValidationError) as exc_info: + _decode(t) + message = str(exc_info.value) + assert "steps[1]" in message + assert "steps[0]" not in message + + def test_let_name_mixed_with_typed_task_param_falls_back(self): + # An expression touching an untyped (let-bound) name falls back to + # name-only validation — no false rejection. + t = _base( + steps=[ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Frame", "type": "INT", "range": [1, 2]} + ] + }, + "script": { + "let": ["msg = 'x'"], + "actions": { + "onRun": { + "command": "echo", + "args": ["{{ msg + Task.Param.Frame }}"], + } + }, + }, + } + ] + ) + _decode(t) + + +class TestJobParameterRawPathTyping: + """RawParam. is a plain string at template scope.""" + + def _job(self, name_expr): + return _base( + name=name_expr, + parameterDefinitions=[{"name": "In", "type": "PATH", "default": "/x/y.exr"}], + ) + + def test_raw_path_property_access_rejected(self): + with pytest.raises(DecodeValidationError, match=r"'name' property is not available"): + _decode(self._job("{{ RawParam.In.name }}")) + + def test_raw_path_string_method_accepted(self): + # Previously falsely rejected: RawParam. was mistyped "path". + _decode(self._job("{{ RawParam.In.upper() }}")) + + +class TestListPathScopeContract: + """Param. is session-scope only; RawParam is list[string].""" + + def _job(self, **overrides): + return _base( + parameterDefinitions=[ + {"name": "Paths", "type": "LIST[PATH]", "default": ["/a/x.exr", "/b/y.exr"]} + ], + **overrides, + ) + + def test_processed_rejected_at_template_scope(self): + # The job name resolves at template scope, where the processed + # (path-mapped) value cannot exist. Rust: "Undefined variable ... + # Did you mean: RawParam.Paths". + t = self._job(name="{{ Param.Paths[0].name }}") + with pytest.raises(DecodeValidationError, match=r"Param\.Paths does not exist"): + _decode(t) + + def test_processed_accepted_at_session_scope(self): + t = self._job( + steps=[ + { + "name": "S", + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{ Param.Paths[0] }}"]}} + }, + } + ] + ) + _decode(t) + + def test_raw_list_path_items_are_strings(self): + # string methods available; path properties not. + _decode(self._job(name="{{ RawParam.Paths[0].upper() }}")) + with pytest.raises(DecodeValidationError, match=r"'name' property is not available"): + _decode(self._job(name="{{ RawParam.Paths[0].name }}")) + + def test_create_job_seeds_raw_only(self): + # create_job must not seed a template-scope Param.* for LIST[PATH]; + # a typed range forwards the raw list[string] via RawParam. + t = self._job( + steps=[ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "P", "type": "PATH", "range": "{{RawParam.Paths}}"} + ] + }, + "script": _RUN, + } + ] + ) + job = create_job(job_template=_decode(t), job_parameter_values={}) + steps = job.steps + step = steps["S"] if isinstance(steps, dict) else steps[0] + tpd = step.parameterSpace.taskParameterDefinitions + tp = tpd["P"] if isinstance(tpd, dict) else tpd[0] + assert len(tp.range) == 2 From 7e47e467f5ba30fed3723b766af5fc55836b2136 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:52:32 -0700 Subject: [PATCH 12/13] fix: Type built-in symbols statically; coerce submitted EXPR values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two parity fixes from external review feedback, both confirmed with two-sided repros against openjd-rs: - Runtime-injected built-in symbols now carry static EXPR types (_BUILTIN_SYMBOL_EXPR_TYPES, applied at every injection site and threaded into `let`-binding validation), mirroring openjd-rs's typed validation symtabs (format_strings.rs build_task_scope_symtab and add_wrapped_action_scope, including the nullable int?/string? unions for the WrappedAction forwarding symbols). Previously expressions such as "{{ Job.Name + 1 }}", "{{ Session.WorkingDirectory + 1 }}", and "{{ WrappedAction.Command + 1 }}" passed decode, survived create_job, and only failed on a worker; Rust rejects them at check. Valid usage (string concat, path properties, the RFC 0008 round-trip forwarding fields) is unaffected; expressions touching untyped names keep the name-only fallback. - Submitted values for EXPR-typed job parameters are now coerced from their string forms, mirroring openjd-rs's coerce_from_str (job/create_job/parameters.rs): BOOL accepts true/false, 1/0, yes/no, on/off (case-insensitive) and LIST[*] accepts JSON strings — the public input type is dict[str, str], so string forms must work. Previously BOOL "no" was stored verbatim and "[1,2]" was rejected with "value must be a list" while the Rust CLI accepted both. Native values still pass through; error messages match the Rust shapes. One existing test updated: test_expr_param_constraints.py pinned the pre-coercion rejection message for a string list value; split into string-input (JSON-coercion error) and native-input (list-type error) cases. Tests: 19 new (test_builtin_symbol_types.py, test_expr_param_coercion.py). Suite: 5457 passed; coverage 94%; mypy/black/ruff clean; sessions suite against this tree unchanged (2 known timing flakes only, pass in isolation). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/_create_job.py | 42 +++++- .../_variable_reference_validation.py | 48 ++++++- .../model_v0/test_builtin_symbol_types.py | 120 ++++++++++++++++++ .../model_v0/test_expr_param_coercion.py | 99 +++++++++++++++ .../v2023_09/test_expr_param_constraints.py | 11 +- 5 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 test/openjd/model_v0/test_builtin_symbol_types.py create mode 100644 test/openjd/model_v0/test_expr_param_coercion.py diff --git a/src/openjd/model/_create_job.py b/src/openjd/model/_create_job.py index 1ae7c585..99bc5ff2 100644 --- a/src/openjd/model/_create_job.py +++ b/src/openjd/model/_create_job.py @@ -1,8 +1,9 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import json from os.path import normpath from pathlib import Path -from typing import Optional, cast +from typing import Any, Optional, cast from pydantic import ValidationError @@ -34,6 +35,38 @@ _LEGACY_SCALAR_TYPE_NAMES = frozenset({"STRING", "INT", "FLOAT", "PATH"}) +def _coerce_expr_param_value(param_type_name: str, value: Any) -> Any: + """Coerce a SUBMITTED string value for an EXPR-typed job parameter to its + native form, mirroring openjd-rs's ``coerce_from_str`` + (job/create_job/parameters.rs): BOOL accepts the spec's boolean strings, + and LIST[*] values may be supplied as JSON — the public input type is + ``dict[str, str]``, so string forms must be accepted. Native values + (bool, list) pass through unchanged. + + Raises: + ValueError: If a string value cannot be coerced (message shapes match + the Rust implementation). + """ + if param_type_name == "BOOL" and isinstance(value, str): + lowered = value.lower() + if lowered in ("true", "yes", "on", "1"): + return True + if lowered in ("false", "no", "off", "0"): + return False + raise ValueError( + f"Value '{value}' is not a valid boolean. Accepted: true/false, 1/0, yes/no, on/off." + ) + if param_type_name.startswith("LIST") and isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"Value '{value}' is not valid JSON for a list parameter.") + if not isinstance(parsed, list): + raise ValueError(f"Value '{value}' is not valid JSON for a list parameter.") + return parsed + return value + + def _is_uri(value: str) -> bool: """Whether ``value`` is a URI (``scheme://...`` with an RFC 3986 scheme). Mirrors openjd-rs's ``uri_path::is_uri``: the scheme must start with an @@ -167,7 +200,12 @@ def _collect_defaults_2023_09( # Check the parameter against the constraints value = job_parameter_values[param.name] if not is_legacy_scalar: - # EXPR types: carry the provided native value through. + # EXPR types: coerce submitted string forms (BOOL strings, + # JSON lists — the public input type is dict[str, str]) to + # their native values, then carry through; mirrors + # openjd-rs's coerce_from_str. + # Raises ValueError (collected by the caller) on bad input. + value = _coerce_expr_param_value(param.type.name, value) return_value[param.name] = ParameterValue( type=ParameterValueType(param.type), value=value ) diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index 78c2545a..0bb6c41f 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -426,6 +426,8 @@ def _validate_model_template_variable_references( for symbol in variable_defs.expr_inject: sym_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" _add_symbol(value_symbols["__self__"], current_scope, sym_name) + if sym_name in _BUILTIN_SYMBOL_EXPR_TYPES: + value_symbols["__self__"].set_type(sym_name, _BUILTIN_SYMBOL_EXPR_TYPES[sym_name]) errors.extend(_validate_let_bindings(value, current_scope, symbols, value_symbols, loc)) @@ -459,6 +461,8 @@ def _validate_model_template_variable_references( for symbol in model._template_field_inject.get(field_name, set()): symbol_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" _add_symbol(validation_symbols, ResolutionScope.TEMPLATE, symbol_name) + if expr_enabled and symbol_name in _BUILTIN_SYMBOL_EXPR_TYPES: + validation_symbols.set_type(symbol_name, _BUILTIN_SYMBOL_EXPR_TYPES[symbol_name]) # Per-field SESSION-scoped symbols (e.g. the RFC 0008 # WrappedEnv.Name / WrappedStep.Name variables): visible to the @@ -468,6 +472,8 @@ def _validate_model_template_variable_references( for symbol in model._template_field_inject_session.get(field_name, set()): symbol_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" _add_symbol(validation_symbols, ResolutionScope.SESSION, symbol_name) + if expr_enabled and symbol_name in _BUILTIN_SYMBOL_EXPR_TYPES: + validation_symbols.set_type(symbol_name, _BUILTIN_SYMBOL_EXPR_TYPES[symbol_name]) # Per-field scope override: fields such as an Action's # timeout/cancelation validate at their declared (template) scope @@ -496,6 +502,29 @@ def _validate_model_template_variable_references( # submission/template-resolution (TEMPLATE) time. _HOST_CONTEXT_FUNCTIONS = {"apply_path_mapping"} +# EXPR types of the runtime-injected built-in symbols, mirroring the typed +# symtabs openjd-rs validates against (format_strings.rs +# build_task_scope_symtab and add_wrapped_*_scope): without these, +# expressions such as ``Job.Name + 1`` or ``Session.WorkingDirectory + 1`` +# fall back to name-only validation and only fail on a worker. The nullable +# wrap-forwarding symbols use the engine's ``T?`` union spelling, exactly as +# Rust types them (``ExprType::union(vec![INT, NULLTYPE])``). +_BUILTIN_SYMBOL_EXPR_TYPES: dict[str, str] = { + "Job.Name": "string", + "Step.Name": "string", + "Session.WorkingDirectory": "path", + "Session.HasPathMappingRules": "bool", + "Session.PathMappingRulesFile": "path", + "WrappedEnv.Name": "string", + "WrappedStep.Name": "string", + "WrappedAction.Command": "string", + "WrappedAction.Args": "list[string]", + "WrappedAction.Environment": "list[string]", + "WrappedAction.Timeout": "int?", + "WrappedAction.Cancelation.Mode": "string?", + "WrappedAction.Cancelation.NotifyPeriodInSeconds": "int?", +} + def _check_format_string( value: FormatString, @@ -599,6 +628,21 @@ def _validate_let_bindings( # symbols + its embedded-file symbols, minus the let names themselves # (added progressively below). accumulated: set = (enclosing | self_symbols | file_symbols) - let_names + # Known EXPR types for the visible symbols (enclosing + this model's own + # + embedded files) so a binding's RHS is type-checked like any other + # expression (e.g. `x = Step.Name + 1` is rejected, matching openjd-rs). + # Let-bound names themselves stay untyped: any expression touching one + # falls back to name-only validation (no false rejects). + accumulated_types: dict[str, str] = dict(symbols.types) + for source_symtab in ( + value_symbols.get("__self__", None), + value_symbols.get("embeddedFiles", None), + ): + if source_symtab is not None: + for type_name, expr_type in source_symtab.types.items(): + accumulated_types.setdefault(type_name, expr_type) + for let_name in let_names: + accumulated_types.pop(let_name, None) for index, binding in enumerate(let_value): if not isinstance(binding, str): continue @@ -654,7 +698,7 @@ def _validate_let_bindings( accumulated.add(name) continue try: - node.validate_symbol_refs(symbols=accumulated) + node.validate_symbol_refs(symbols=accumulated, symbol_types=accumulated_types or None) except ValueError as exc: errors.append( InitErrorDetails( @@ -913,6 +957,8 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) else: symbol_name = f"{symbol_prefix}{symbol}" _add_symbol(symbols["__self__"], current_scope, symbol_name) + if collect_types and symbol_name in _BUILTIN_SYMBOL_EXPR_TYPES: + symbols["__self__"].set_type(symbol_name, _BUILTIN_SYMBOL_EXPR_TYPES[symbol_name]) # EXPR `let` bindings (RFC 0007 §3.6) define bare-name symbols at this # model's scope, available to the model's other fields and to subsequent diff --git a/test/openjd/model_v0/test_builtin_symbol_types.py b/test/openjd/model_v0/test_builtin_symbol_types.py new file mode 100644 index 00000000..04f14fa4 --- /dev/null +++ b/test/openjd/model_v0/test_builtin_symbol_types.py @@ -0,0 +1,120 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Static EXPR typing of runtime-injected built-in symbols. + +openjd-rs validates expressions against typed symtabs containing the +built-ins (format_strings.rs build_task_scope_symtab: Job.Name/Step.Name are +strings, Session.WorkingDirectory a path, ...; add_wrapped_action_scope types +the nullable forwarding symbols as ``int?``/``string?`` unions). Without the +same typing, Python accepted expressions like ``Job.Name + 1`` that only +failed on a worker. +""" + +import pytest + +from openjd.model import DecodeValidationError, decode_job_template + +EXT = ["EXPR", "WRAP_ACTIONS", "FEATURE_BUNDLE_1"] +RUN = {"actions": {"onRun": {"command": "echo"}}} + + +def _decode(t): + return decode_job_template(template=t, supported_extensions=EXT) + + +def _job(**overrides): + t = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": EXT, + "name": "T", + "steps": [{"name": "S", "script": RUN}], + } + t.update(overrides) + return t + + +def _step_arg(expr): + return [{"name": "S", "script": {"actions": {"onRun": {"command": "echo", "args": [expr]}}}}] + + +class TestBuiltinSymbolTyping: + @pytest.mark.parametrize( + "expr", + [ + pytest.param("{{ Job.Name + 1 }}", id="job-name-plus-int"), + pytest.param("{{ Session.WorkingDirectory + 1 }}", id="workdir-plus-int"), + pytest.param("{{ Step.Name + 1 }}", id="step-name-plus-int"), + ], + ) + def test_invalid_builtin_expressions_rejected(self, expr): + with pytest.raises(DecodeValidationError, match=r"Cannot use '\+' operator"): + _decode(_job(steps=_step_arg(expr))) + + def test_wrapped_action_command_plus_int_rejected(self): + t = _job( + jobEnvironments=[ + { + "name": "W", + "script": { + "actions": { + "onWrapEnvEnter": { + "command": "echo", + "args": ["{{ WrappedAction.Command + 1 }}"], + }, + "onWrapTaskRun": {"command": "echo"}, + "onWrapEnvExit": {"command": "echo"}, + } + }, + } + ] + ) + with pytest.raises(DecodeValidationError, match=r"Cannot use '\+' operator"): + _decode(t) + + def test_step_let_binding_rhs_is_type_checked(self): + t = _job(steps=[{"name": "S", "let": ["x = Step.Name + 1"], "script": RUN}]) + with pytest.raises(DecodeValidationError, match=r"Cannot use '\+' operator"): + _decode(t) + + @pytest.mark.parametrize( + "expr", + [ + pytest.param("{{ Job.Name + '-suffix' }}", id="string-concat"), + pytest.param("{{ Session.WorkingDirectory }}/out.exr", id="workdir-plain"), + pytest.param("{{ Session.WorkingDirectory.name }}", id="workdir-path-property"), + ], + ) + def test_valid_builtin_expressions_accepted(self, expr): + _decode(_job(steps=_step_arg(expr))) + + def test_full_wrap_forwarding_still_accepted(self): + # The nullable typed symbols (int?/string?) must not over-reject the + # RFC 0008 round-trip forwarding case. + t = _job( + jobEnvironments=[ + { + "name": "W", + "script": { + "actions": { + "onWrapEnvEnter": { + "command": "{{WrappedAction.Command}}", + "args": ["{{ WrappedEnv.Name }}"], + "timeout": "{{WrappedAction.Timeout}}", + "cancelation": { + "mode": "{{WrappedAction.Cancelation.Mode}}", + "notifyPeriodInSeconds": "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}", + }, + }, + "onWrapTaskRun": { + "command": "run", + "args": [ + "{{WrappedStep.Name}}", + "{{ repr_sh(WrappedAction.Args) }}", + ], + }, + "onWrapEnvExit": {"command": "exit"}, + } + }, + } + ] + ) + _decode(t) diff --git a/test/openjd/model_v0/test_expr_param_coercion.py b/test/openjd/model_v0/test_expr_param_coercion.py new file mode 100644 index 00000000..99fc6cbc --- /dev/null +++ b/test/openjd/model_v0/test_expr_param_coercion.py @@ -0,0 +1,99 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Coercion of SUBMITTED string values for EXPR-typed job parameters. + +The public input type is ``dict[str, str]``, so string forms must be +accepted: BOOL takes the spec's boolean strings and LIST[*] takes JSON, +mirroring openjd-rs's ``coerce_from_str`` (job/create_job/parameters.rs). +Previously BOOL ``"no"`` was stored verbatim and JSON list strings were +rejected outright while the Rust CLI accepted both. +""" + +import pytest + +from openjd.model import create_job, decode_job_template, preprocess_job_parameters + +_TEMPLATE = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": [ + {"name": "Flag", "type": "BOOL"}, + {"name": "Values", "type": "LIST[INT]"}, + {"name": "Nested", "type": "LIST[LIST[INT]]"}, + ], + "steps": [ + { + "name": "S", + "script": { + "actions": { + "onRun": { + "command": "echo", + "args": [ + "{{ Param.Flag }}", + "{{ Param.Values[0] + 1 }}", + "{{ Param.Nested[0][1] }}", + ], + } + } + }, + } + ], +} + + +@pytest.fixture +def template(): + return decode_job_template(template=_TEMPLATE, supported_extensions=["EXPR"]) + + +def _preprocess(template, values, tmp_path): + return preprocess_job_parameters( + job_template=template, + job_parameter_values=values, + job_template_dir=tmp_path, + current_working_dir=tmp_path, + ) + + +class TestExprParameterValueCoercion: + @pytest.mark.parametrize( + "submitted, expected", + [ + pytest.param("true", True, id="true"), + pytest.param("no", False, id="no"), + pytest.param("ON", True, id="on-case-insensitive"), + pytest.param("0", False, id="zero"), + pytest.param(True, True, id="native-bool-passthrough"), + ], + ) + def test_bool_string_forms_coerced(self, template, tmp_path, submitted, expected): + pv = _preprocess(template, {"Flag": submitted, "Values": [1], "Nested": [[1, 2]]}, tmp_path) + assert pv["Flag"].value is expected + + def test_json_list_strings_coerced(self, template, tmp_path): + pv = _preprocess( + template, {"Flag": "true", "Values": "[1,2]", "Nested": "[[1,2],[3]]"}, tmp_path + ) + assert pv["Values"].value == [1, 2] + assert pv["Nested"].value == [[1, 2], [3]] + # And the coerced values evaluate as their native types end-to-end. + create_job(job_template=template, job_parameter_values=pv) + + def test_native_lists_pass_through(self, template, tmp_path): + pv = _preprocess(template, {"Flag": False, "Values": [3, 4], "Nested": [[5]]}, tmp_path) + assert pv["Values"].value == [3, 4] + + def test_invalid_bool_rejected_with_rust_message(self, template, tmp_path): + with pytest.raises(ValueError, match=r"not a valid boolean\. Accepted: true/false"): + _preprocess(template, {"Flag": "nope", "Values": [1], "Nested": [[1]]}, tmp_path) + + @pytest.mark.parametrize( + "bad", + [ + pytest.param("[1,2", id="malformed-json"), + pytest.param('{"a": 1}', id="json-but-not-a-list"), + ], + ) + def test_invalid_list_json_rejected_with_rust_message(self, template, tmp_path, bad): + with pytest.raises(ValueError, match=r"not valid JSON for a list parameter"): + _preprocess(template, {"Flag": "true", "Values": bad, "Nested": [[1]]}, tmp_path) diff --git a/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py b/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py index 8e97768c..57322d03 100644 --- a/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py +++ b/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py @@ -84,10 +84,17 @@ def test_wrong_item_type_rejected(self): with pytest.raises(ValueError, match=r"list items must be integers"): _preprocess({"name": "Nums", "type": "LIST[INT]"}, {"Nums": ["not-an-int"]}) - def test_non_list_value_rejected(self): - with pytest.raises(ValueError, match=r"value must be a list"): + def test_non_list_string_value_rejected(self): + # String inputs are JSON-coerced (dict[str, str] public input type, + # mirroring openjd-rs coerce_from_str); JSON that is not a list is + # rejected there. + with pytest.raises(ValueError, match=r"not valid JSON for a list parameter"): _preprocess({"name": "Nums", "type": "LIST[INT]"}, {"Nums": "5"}) + def test_non_list_native_value_rejected(self): + with pytest.raises(ValueError, match=r"value must be a list"): + _preprocess({"name": "Nums", "type": "LIST[INT]"}, {"Nums": 5}) + def test_list_list_int_inner_constraint_enforced(self): with pytest.raises(ValueError, match=r"item 99 is above item\.maxValue 10"): _preprocess( From 7a62f06d5036975c4c483d844df735a36c034173 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:04:01 -0700 Subject: [PATCH 13/13] fix: Raw PATH range values are not separator-normalized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The typed-range positive test expected Windows separator normalization, a leftover from when its fixture used the processed {{Param.Items}} reference. The LIST[PATH] scope fix switched it to {{RawParam.Items}}, and raw values are the original unmapped strings (RFC 0005) — no path-separator normalization applies on any platform. Expect the verbatim values everywhere. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../model_v0/v2023_09/test_create_job_expr_params.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py index 70e71f7c..b2c88271 100644 --- a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py +++ b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py @@ -9,8 +9,6 @@ symbol table can coerce them. """ -import sys - import pytest from openjd.model import ( @@ -149,10 +147,11 @@ def test_valid_paths_in_typed_range_accepted(self): assert step.parameterSpace is not None tpd = step.parameterSpace.taskParameterDefinitions tp = tpd["V"] if isinstance(tpd, dict) else tpd[0] - # PATH range values are normalized to the host's native separator - # (same convention as test_lists.py's path coercion tests). - expected = ["\\a", "\\b"] if sys.platform == "win32" else ["/a", "/b"] - assert [str(v) for v in tp.range] == expected + # The range forwards the RAW list[string] value (RawParam, RFC 0005): + # raw values are the original unmapped strings, so no path-separator + # normalization applies on any platform (unlike processed Param.* + # path values, whose normalization test_lists.py covers). + assert [str(v) for v in tp.range] == ["/a", "/b"] def test_empty_string_in_typed_string_range_accepted(self): # No over-rejection: STRING task parameters legitimately allow "".