diff --git a/rust-bindings/src/expr/mod.rs b/rust-bindings/src/expr/mod.rs index 5a08eda4..8b11ac22 100644 --- a/rust-bindings/src/expr/mod.rs +++ b/rust-bindings/src/expr/mod.rs @@ -26,5 +26,5 @@ pub(crate) use path_mapping::PyPathMappingRule; pub(crate) use profile::{PyExprExtension, PyExprProfile, PyExprRevision, PyHostContext}; pub(crate) use range_expr::{PyIntRange, PyRangeExpr}; pub(crate) use symbol_table::{ - _reconstruct_serialized_symtab, PySerializedSymbolTable, PySymbolTable, + _reconstruct_serialized_symtab, build_symbol_table, PySerializedSymbolTable, PySymbolTable, }; diff --git a/rust-bindings/src/expr/parsed_expression.rs b/rust-bindings/src/expr/parsed_expression.rs index 8121ced2..d61755d5 100644 --- a/rust-bindings/src/expr/parsed_expression.rs +++ b/rust-bindings/src/expr/parsed_expression.rs @@ -92,6 +92,37 @@ impl PyParsedExpression { Ok(PyExprValue { inner: value }) } + /// Type-check the expression against a symbol table of (typically + /// unresolved) typed placeholders, without extracting a concrete value. + /// + /// Succeeds when the expression is well-typed for the given symbol types — + /// including when the result is an unresolved value that merely depends on + /// a runtime symbol — and raises only on a genuine type/evaluation error. + /// Unlike :meth:`evaluate`, it discards the result, so it never raises the + /// "cannot extract value from unresolved" boundary error; callers no longer + /// need to sniff the error message to tell a real type error from a + /// runtime-dependent one. + #[pyo3(signature = (*, values=None, profile=None))] + fn typecheck( + &self, + values: Option<&Bound<'_, pyo3::PyAny>>, + profile: Option<&PyExprProfile>, + ) -> PyResult<()> { + let symtab; + let symtab_refs: Vec<&SymbolTable> = if let Some(v) = values { + symtab = extract_symtab(v)?; + vec![&symtab] + } else { + vec![] + }; + let lib = profile_for_call(profile); + self.inner + .with_library(&lib) + .evaluate(&symtab_refs) + .map_err(expr_err_to_py)?; + Ok(()) + } + /// Evaluate the expression and return an :class:`EvalResult` with the /// resulting value alongside the per-call resource-usage metrics /// (``peak_memory`` in bytes, ``operation_count``). diff --git a/rust-bindings/src/expr/symbol_table.rs b/rust-bindings/src/expr/symbol_table.rs index 13762f44..f2fa3570 100644 --- a/rust-bindings/src/expr/symbol_table.rs +++ b/rust-bindings/src/expr/symbol_table.rs @@ -2,13 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyString}; #[cfg(feature = "stub-gen")] use pyo3_stub_gen::derive::*; +use openjd_expr::path_mapping::PathFormat; use openjd_expr::symbol_table::SymbolTable; +use openjd_expr::types::ExprType; +use openjd_expr::value::ExprValue; use crate::expr::expr_value::{py_to_expr_value, PyExprValue}; +use crate::expr::path_format::PyPathFormat; #[cfg_attr(feature = "stub-gen", gen_stub_pyclass(module = "openjd._openjd_rs"))] #[pyclass(module = "openjd.expr", name = "SymbolTable", from_py_object)] @@ -324,3 +328,79 @@ pub(crate) fn _reconstruct_serialized_symtab(json: &str) -> PyResult, + target: Option<&ExprType>, + pf: PathFormat, +) -> PyResult { + let Some(target) = target else { + // No confident type — let the engine infer from the native value. + return py_to_expr_value(value); + }; + if let Ok(s) = value.cast::() { + return ExprValue::from_str_coerce(&s.to_cow()?, target, pf) + .map_err(pyo3::exceptions::PyValueError::new_err); + } + py_to_expr_value(value)? + .coerce(target, pf) + .map_err(pyo3::exceptions::PyValueError::new_err) +} + +/// Build a typed :class:`SymbolTable` from a flat dotted-key value map and an +/// optional per-key EXPR type-spec map, coercing string values to their +/// declared type and nesting dotted keys (``"Param.Frame"``) into subtables. +/// +/// ``values`` maps dotted symbol names to their (typically string) values, as +/// the v0 model stores them. ``types`` maps the same dotted names to EXPR +/// type spec strings (``"int"``, ``"list[int]"``, ``"path"``, …); names absent +/// from ``types`` are inferred from the value. ``path_format`` controls how +/// PATH-typed values are interpreted (defaults to the host OS). +#[cfg_attr( + feature = "stub-gen", + gen_stub_pyfunction(module = "openjd._openjd_rs") +)] +#[pyfunction] +#[pyo3(signature = (values, types=None, *, path_format=None))] +pub(crate) fn build_symbol_table( + values: &Bound<'_, PyDict>, + types: Option<&Bound<'_, PyDict>>, + path_format: Option, +) -> PyResult { + let pf = path_format + .map(PathFormat::from) + .unwrap_or_else(PathFormat::host); + let mut st = SymbolTable::new(); + for (key, value) in values.iter() { + let dotted: String = key.extract()?; + let target: Option = match types { + Some(t) => match t.get_item(dotted.as_str())? { + Some(spec_obj) => { + let spec: String = spec_obj.extract()?; + Some(ExprType::parse(&spec).map_err(pyo3::exceptions::PyValueError::new_err)?) + } + None => None, + }, + None => None, + }; + let ev = coerce_symbol_value(&value, target.as_ref(), pf)?; + st.set(&dotted, ev) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + } + Ok(PySymbolTable { inner: st }) +} diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 2130c486..d20db372 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -81,6 +81,7 @@ fn openjd_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(evaluate_expression, m)?)?; m.add_function(wrap_pyfunction!(parse_expression, m)?)?; m.add_function(wrap_pyfunction!(escape_format_string, m)?)?; + m.add_function(wrap_pyfunction!(build_symbol_table, m)?)?; m.add_function(wrap_pyfunction!(_reconstruct_expr_value, m)?)?; m.add_function(wrap_pyfunction!(_reconstruct_serialized_symtab, m)?)?; @@ -231,6 +232,7 @@ fn openjd_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(validate_attribute_capability_name, m)?)?; m.add_function(wrap_pyfunction!(standard_amount_capability_names, m)?)?; m.add_function(wrap_pyfunction!(standard_attribute_capability_names, m)?)?; + m.add_function(wrap_pyfunction!(job_parameter_type_expr_spec, m)?)?; m.add_function(wrap_pyfunction!(standard_attribute_capabilities, m)?)?; register_renamed_exception( diff --git a/rust-bindings/src/model/mod.rs b/rust-bindings/src/model/mod.rs index e6a624a3..e3aa4f30 100644 --- a/rust-bindings/src/model/mod.rs +++ b/rust-bindings/src/model/mod.rs @@ -72,8 +72,8 @@ pub(crate) use template_types::{ PyStepScript as PyTemplateStepScript, PyStepTemplate, }; pub(crate) use types::{ - PyDocumentType, PyJobParameterType, PyJobParameterValue, PyTaskParameterType, - PyTaskParameterValue, PyTemplateSpecificationVersion, + job_parameter_type_expr_spec, PyDocumentType, PyJobParameterType, PyJobParameterValue, + PyTaskParameterType, PyTaskParameterValue, PyTemplateSpecificationVersion, }; pub(crate) use user_interfaces::{ PyBoolUserInterface, PyFileFilter, PyFloatUserInterface, PyHiddenOnlyUserInterface, diff --git a/rust-bindings/src/model/types.rs b/rust-bindings/src/model/types.rs index 8a7b68fc..6a58cd75 100644 --- a/rust-bindings/src/model/types.rs +++ b/rust-bindings/src/model/types.rs @@ -304,6 +304,22 @@ impl PyJobParameterType { } } +/// Map an OpenJD job-parameter type spec name (e.g. ``"INT"``, ``"LIST[INT]"``, +/// ``"RANGE_EXPR"``; case-insensitive) to its EXPR type spec string +/// (``"int"``, ``"list[int]"``, ``"range_expr"``), or ``None`` when the name is +/// not a recognized job-parameter type. Single-sources both the +/// (case-insensitive) type-name parsing and the OpenJD-type -> EXPR-type +/// mapping in the Rust ``openjd-model`` crate so the Python model does not +/// hand-maintain a parallel table. +#[cfg_attr( + feature = "stub-gen", + gen_stub_pyfunction(module = "openjd._openjd_rs") +)] +#[pyfunction] +pub(crate) fn job_parameter_type_expr_spec(type_name: &str) -> Option { + JobParameterType::from_spec_str(type_name.trim()).map(|t| t.expr_type().to_string()) +} + impl From for JobParameterType { fn from(v: PyJobParameterType) -> Self { match v { diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index d44835d7..48c73542 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -1780,6 +1780,12 @@ class ParsedExpression: @property def expr(self) -> builtins.str: ... def __repr__(self) -> builtins.str: ... + def typecheck( + self, + *, + values: typing.Optional[typing.Any] = None, + profile: typing.Optional[ExprProfile] = None, + ) -> None: ... def evaluate( self, *, @@ -3567,6 +3573,15 @@ def deserialize_step(step_dict: dict) -> Step: """ def escape_format_string(value: builtins.str) -> builtins.str: ... +def job_parameter_type_expr_spec( + type_name: builtins.str, +) -> typing.Optional[builtins.str]: ... +def build_symbol_table( + values: builtins.dict, + types: typing.Optional[builtins.dict] = None, + *, + path_format: typing.Optional[PathFormat] = None, +) -> SymbolTable: ... def evaluate_expression( expr: builtins.str, *, diff --git a/src/openjd/model/_create_job.py b/src/openjd/model/_create_job.py index c4ea3f67..4f8ca38d 100644 --- a/src/openjd/model/_create_job.py +++ b/src/openjd/model/_create_job.py @@ -26,6 +26,12 @@ __all__ = ("preprocess_job_parameters",) +# The original scalar job-parameter type names whose values are carried as +# strings through preprocessing. EXPR-extension types (BOOL, RANGE_EXPR, and +# the LIST[*] variants) are carried natively instead so the typed EXPR symbol +# table can coerce them. +_LEGACY_SCALAR_TYPE_NAMES = frozenset({"STRING", "INT", "FLOAT", "PATH"}) + # ======================================================================= # ================ Preprocessing Job Parameters ========================= @@ -70,8 +76,18 @@ def _collect_defaults_2023_09( return_value: JobParameterValues = dict[str, ParameterValue]() # Collect defaults for param in job_parameter_definitions: + is_legacy_scalar = param.type.name in _LEGACY_SCALAR_TYPE_NAMES if param.name not in job_parameter_values: if param.default is not None: + if not is_legacy_scalar: + # EXPR types (BOOL / RANGE_EXPR / LIST[*]): carry the native + # default through so the typed symbol-table builder can + # coerce it. The PATH-relative-default handling below only + # applies to the scalar PATH type. + return_value[param.name] = ParameterValue( + type=ParameterValueType(param.type), value=param.default + ) + continue default = str(param.default) # Make PATH defaults relative to job_template_dir, and # enforce the `allow_job_template_dir_walk_up` parameter request. @@ -104,6 +120,12 @@ def _collect_defaults_2023_09( else: # Check the parameter against the constraints value = job_parameter_values[param.name] + if not is_legacy_scalar: + # EXPR types: carry the provided native value through. + return_value[param.name] = ParameterValue( + type=ParameterValueType(param.type), value=value + ) + continue # Join any provided relative PATH parameter value with the current_working_directory (except the empty value "") if param.type.name == "PATH" and value != "" and not Path(value).is_absolute(): value = str(current_working_dir / value) @@ -123,8 +145,16 @@ def _check_2023_09( for param in job_parameter_definitions: if param.name in job_parameter_values: param_value = job_parameter_values[param.name] + # The EXPR-extension LIST[*]/RANGE_EXPR definitions don't implement + # _check_constraints (BOOL and the original scalars do). Their + # template defaults are validated at decode time, and their values + # are type-checked when coerced into the typed EXPR symbol table, so + # skip the create-time constraint check when it isn't available. + check_constraints = getattr(param, "_check_constraints", None) + if check_constraints is None: + continue try: - param._check_constraints(param_value.value) + check_constraints(param_value.value) except ValueError as err: errors.append(str(err)) @@ -312,14 +342,21 @@ def create_job( if job_template.specificationVersion == TemplateSpecificationVersion.JOBTEMPLATE_v2023_09: from .v2023_09 import ValueReferenceConstants as ValueReferenceConstants_2023_09 + # EXPR-extension typed params (BOOL / RANGE_EXPR / LIST[*]) carry native + # values; record their OpenJD type so the typed symbol-table builder + # coerces them to the right ExprType during expression evaluation. The + # original scalar types keep their existing string-based handling. + expr_types: dict[str, str] = {} for name, param in all_job_parameter_values.items(): + prefix = ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value + raw_prefix = ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value if param.type != "PATH": - symtab[f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{name}"] = ( - all_job_parameter_values[name].value - ) - symtab[f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value}.{name}"] = ( - all_job_parameter_values[name].value - ) + symtab[f"{prefix}.{name}"] = all_job_parameter_values[name].value + symtab[f"{raw_prefix}.{name}"] = all_job_parameter_values[name].value + if param.type.name not in _LEGACY_SCALAR_TYPE_NAMES: + expr_types[f"{prefix}.{name}"] = param.type.value + expr_types[f"{raw_prefix}.{name}"] = param.type.value + symtab.expr_types.update(expr_types) else: raise NotImplementedError( f"Spec version {job_template.specificationVersion} not implemented." diff --git a/src/openjd/model/_format_strings/_expr_support.py b/src/openjd/model/_format_strings/_expr_support.py new file mode 100644 index 00000000..4b5838b1 --- /dev/null +++ b/src/openjd/model/_format_strings/_expr_support.py @@ -0,0 +1,184 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Support for the EXPR extension (RFCs 0005/0006/0007) in the pure-Python +(v0) model. + +The pure-Python model does not implement the EXPR expression grammar. When a +template declares the ``EXPR`` extension, format-string expressions are parsed +and evaluated by the Rust ``openjd-expr`` engine through the ``openjd._openjd_rs`` +bindings. This module is the thin bridge between the two. + +The typed symbol-table construction (coercing the v0 model's flat, stringly +typed values into a typed EXPR ``SymbolTable``) is performed by the Rust +``build_symbol_table`` binding; this module supplies the OpenJD-type → EXPR +type-spec mapping and re-raises Rust errors as the model's own +``ExpressionError`` at the boundary. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from .._errors import ExpressionError as _ModelExpressionError +from .._symbol_table import SymbolTable + +# The EXPR engine lives in the compiled extension. These module-level imports +# are reached only by code that handles an EXPR template: callers import this +# module lazily (inside functions), and the EXPR_EXTENSION gate constant lives +# in ._parser so that merely importing the parser does not load the Rust expr +# surface. The non-EXPR parse path therefore never imports this module. +from openjd._openjd_rs import ( # type: ignore[import-not-found] + build_symbol_table, + job_parameter_type_expr_spec, +) +from openjd.expr import ( # type: ignore[import-not-found] + ExprProfile, + ExprType, + ExprValue, + ExpressionError as RustExpressionError, + ExpressionTypeError as RustExpressionTypeError, + FormatString as _RustFormatString, + FormatStringValidationError as RustFormatStringValidationError, + HostContext, + RangeExprError as RustRangeExprError, + parse_expression, +) + +# Errors raised by the Rust expr engine. All subclass ValueError but are +# distinct classes from the model's own ExpressionError, so they must be +# caught explicitly at the binding boundary and re-raised as model errors. +_RUST_EXPR_ERRORS = ( + RustExpressionError, + RustExpressionTypeError, + RustFormatStringValidationError, + RustRangeExprError, +) + + +def symtab_to_expr_values( + symtab: SymbolTable, + *, + types: Optional[dict[str, str]] = None, + path_format: Any = None, +) -> Any: + """Build a typed EXPR ``SymbolTable`` from the v0 (flat dotted-key) + ``SymbolTable``. + + The typed coercion (e.g. a stored ``"10"`` of type INT → a real integer) + and the dotted-key → nested-subtable construction are delegated to the + Rust ``build_symbol_table`` binding, so the value typing lives next to the + engine and stays consistent with it (PR #285 review, C3/C5). + + ``types`` maps dotted symbol names to OpenJD type names (e.g. ``"INT"``, + ``"LIST[INT]"``); each is translated to the EXPR type spec the engine + expects (``"int"``, ``"list[int]"``) via :func:`expr_type_for_openjd_type`. + Names whose type has no confident EXPR mapping (e.g. ``RANGE_EXPR``) are + omitted so the engine infers them from the value. + """ + flat = {name: symtab[name] for name in symtab.symbols} + expr_types: dict[str, str] = {} + for name, openjd_type in (types or {}).items(): + spec = expr_type_for_openjd_type(openjd_type) + if spec is not None: + expr_types[name] = spec + return build_symbol_table(flat, expr_types or None, path_format=path_format) + + +def parse_expr_or_raise(expr: str) -> Any: + """Parse an EXPR expression string with the Rust engine, re-raising parse + failures as the model's ``ExpressionError``. + """ + try: + return parse_expression(expr.strip()) + except _RUST_EXPR_ERRORS as exc: + raise _ModelExpressionError(str(exc)) + + +def _static_validation_profile() -> ExprProfile: + """Profile for static (parse-time) expression validation. + + Uses ``HostContext.unresolved()`` so host-context-gated functions such as + ``apply_path_mapping`` resolve to unresolved values (valid) rather than + raising "Unknown function", while genuinely unknown functions still fail. + """ + return ExprProfile.current().with_host_context(HostContext.unresolved()) + + +def static_validate_symbol_free(expr: str) -> None: + """Statically validate an expression that references no free symbols. + + Catches literal/semantic errors that the Rust engine only raises at + evaluation time — int64 overflow, division by zero, unknown functions, + type mismatches, slice-step-zero, null-in-list, etc. — so that + ``openjd check`` rejects them. Safe only for symbol-free expressions: + with no symbols there is no risk of a wrong placeholder type causing a + false rejection. Symbol-referencing expressions are validated once their + symbol types are known (RFC 0007 parameter types — follow-up). + """ + try: + _RustFormatString("{{ " + expr + " }}").validate_expressions( + {}, profile=_static_validation_profile() + ) + except _RUST_EXPR_ERRORS as exc: + raise _ModelExpressionError(str(exc)) + + +def map_eval_error(exc: BaseException) -> _ModelExpressionError: + """Wrap a Rust evaluation error as the model's ``ExpressionError``.""" + return _ModelExpressionError(str(exc)) + + +def expr_type_for_openjd_type(openjd_type: str) -> Optional[str]: + """Map an OpenJD parameter type name (e.g. ``PATH``, ``LIST[INT]``, + ``RANGE_EXPR``; case-insensitive) to the EXPR engine's spec-form type + string (``path``, ``list[int]``, ``range_expr``), or ``None`` if the name + is not a recognized job-parameter type. + + Delegates to the Rust ``job_parameter_type_expr_spec`` binding so the + OpenJD-type → EXPR-type mapping (including ``LIST[...]`` nesting) lives in a + single place — the ``openjd-model`` crate — rather than a parallel + hand-maintained Python table that could drift. + """ + return job_parameter_type_expr_spec(openjd_type) + + +def longest_defined_prefix(name: str, defined: set) -> Optional[str]: + """Return the longest dotted prefix of ``name`` that is a defined symbol, + treating any remaining segments as method/property access. E.g. for + ``"Param.File.name"`` with ``Param.File`` defined, returns ``"Param.File"``. + """ + segments = name.split(".") + for end in range(len(segments), 0, -1): + prefix = ".".join(segments[:end]) + if prefix in defined: + return prefix + return None + + +def validate_typed_expression(parsed: Any, *, typed_symbols: dict[str, str]) -> None: + """Statically validate a parsed expression against symbols of known EXPR + type (provided as ``{dotted_name: expr_type_string}``). Catches type + mismatches and invalid method/property access. Raises the model's + ``ExpressionError`` on failure. + + Symbols are supplied as ``ExprValue.unresolved()`` placeholders so the + engine type-checks without concrete values. + """ + values: dict[str, Any] = {} + for dotted, type_str in typed_symbols.items(): + ev = ExprValue.unresolved(ExprType(type_str)) + cursor = values + segs = dotted.split(".") + for seg in segs[:-1]: + cursor = cursor.setdefault(seg, {}) + cursor[segs[-1]] = ev + try: + # typecheck() evaluates without extracting the result, so an expression + # that is well-typed but depends on an unresolved runtime symbol passes + # (no "cannot extract value from unresolved" boundary error to sniff for). + parsed.typecheck( + values=values, + profile=ExprProfile.current().with_host_context(HostContext.unresolved()), + ) + except _RUST_EXPR_ERRORS as exc: + raise _ModelExpressionError(str(exc)) diff --git a/src/openjd/model/_format_strings/_expression.py b/src/openjd/model/_format_strings/_expression.py index 85d10a8b..7968eff9 100644 --- a/src/openjd/model/_format_strings/_expression.py +++ b/src/openjd/model/_format_strings/_expression.py @@ -1,7 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. import numbers -from typing import Union +from typing import Any, Optional, Union from .._errors import ExpressionError from .._symbol_table import SymbolTable @@ -29,35 +29,79 @@ def __init__(self, expr: str, *, context: ModelParsingContextInterface) -> None: # Raises: ExpressionError, TokenError self._expresion_tree = parse_format_string_expr(expr, context=context) - def validate_symbol_refs(self, *, symbols: set[str]) -> None: + def validate_symbol_refs(self, *, symbols: set[str], symbol_types: Any = None) -> None: """Check whether this expression can be evaluated correctly given a set of symbol names. Args: symbols (set[str]): The names of symbols visible to this expression. + symbol_types: Optional mapping of symbol name -> EXPR type string, + enabling type-aware validation when available. Raises: ValueError: If the expression cannot be evaluated with the given symbol names """ - self._expresion_tree.validate_symbol_refs(symbols=symbols) - - def evaluate(self, *, symtab: SymbolTable) -> Union[numbers.Real, str]: + self._expresion_tree.validate_symbol_refs(symbols=symbols, symbol_types=symbol_types) + + @property + def called_functions(self) -> set: + """Names of functions invoked by this expression (empty for the legacy + name-only parser).""" + return getattr(self._expresion_tree, "called_functions", set()) + + @property + def accessed_symbols(self) -> set: + """Free symbol references in this expression in full dotted spelling + (empty for the legacy name-only parser, which exposes no such set).""" + return getattr(self._expresion_tree, "accessed_symbols", set()) + + def evaluate( + self, *, symtab: SymbolTable, path_format: Optional[Any] = None + ) -> Union[numbers.Real, str, Any]: """Evaluate the expression given a SymbolTable. Args: symtab (SymbolTable): A symbol table containing values to use in the evaluation. + path_format (Any): Optional EXPR PathFormat for PATH-typed values. Raises: ExpressionError: If the expression could not be evaluated. Returns: - Union[numbers.Real, str]: Resulting value. + The resulting value. For legacy (non-EXPR) expressions this is a + ``numbers.Real`` or ``str``; EXPR expressions may also yield + ``bool``, ``list``, or ``None``. """ try: - result = self._expresion_tree.evaluate(symtab=symtab) + result = self._expresion_tree.evaluate(symtab=symtab, path_format=path_format) except ValueError as exc: raise ExpressionError(f"Expression failed validation: {str(exc)}") + # EXPR-backed nodes may legitimately return non-scalar values; the + # legacy name path is still restricted to Real/str to preserve its + # exact behavior. + if getattr(self._expresion_tree, "allows_nonscalar", False): + return result if isinstance(result, (numbers.Real, str)): return result raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}") + + def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> str: + """Evaluate the expression and coerce the result to the string form that + is substituted into the surrounding format string. + + For EXPR expressions this uses the engine's spec-defined string coercion + (RFC 0005) rather than Python's ``str()``, so booleans, null, and lists + render per the specification rather than as Python reprs. + + Args: + symtab (SymbolTable): A symbol table containing values to use in the evaluation. + path_format (Any): Optional EXPR PathFormat for PATH-typed values. + + Raises: + ExpressionError: If the expression could not be evaluated. + """ + try: + return self._expresion_tree.evaluate_to_str(symtab=symtab, path_format=path_format) + except ValueError as exc: + raise ExpressionError(f"Expression failed validation: {str(exc)}") diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index 364a137b..5ee6348c 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -1,7 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. from dataclasses import dataclass -from numbers import Real from typing import Optional, Union from .._errors import ExpressionError, TokenError @@ -16,7 +15,9 @@ class ExpressionInfo: start_pos: int end_pos: int expression: Optional[InterpolationExpression] = None - resolved_value: Optional[Union[Real, str]] = None + # The string form substituted into the surrounding format string, produced + # by InterpolationExpression.evaluate_to_str during resolve(). + resolved_value: Optional[str] = None class FormatStringError(ValueError): @@ -75,7 +76,7 @@ def expressions(self) -> list[ExpressionInfo]: """ return [expr for expr in self._processed_list if isinstance(expr, ExpressionInfo)] - def resolve(self, *, symtab: SymbolTable) -> str: + def resolve(self, *, symtab: SymbolTable, path_format: Optional[object] = None) -> str: """ Uses a given symbol table to resolve an interpolated string. Each interpolation expression in the original string is replaced @@ -108,7 +109,12 @@ def resolve(self, *, symtab: SymbolTable) -> str: assert element.expression is not None try: - element.resolved_value = element.expression.evaluate(symtab=symtab) + # evaluate_to_str applies the engine's spec-defined string + # coercion for EXPR results (RFC 0005), so booleans/null/lists + # render per the specification rather than as Python reprs. + element.resolved_value = element.expression.evaluate_to_str( + symtab=symtab, path_format=path_format + ) except ExpressionError as exc: raise FormatStringError( string=self.original_value, @@ -118,7 +124,7 @@ def resolve(self, *, symtab: SymbolTable) -> str: details=str(exc), ) - resolved_list.append(str(element.resolved_value)) + resolved_list.append(element.resolved_value) return "".join(resolved_list) diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 1cc5ac9b..0bcb5f3a 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -16,7 +16,9 @@ class Node(ABC): """ @abstractmethod - def validate_symbol_refs(self, *, symbols: set[str]) -> None: # pragma: no cover + def validate_symbol_refs( + self, *, symbols: set[str], symbol_types: Any = None + ) -> None: # pragma: no cover """Verifies that the expression rooted at this node is valid given the definitions of symbols in a symbol table. @@ -32,8 +34,12 @@ def validate_symbol_refs(self, *, symbols: set[str]) -> None: # pragma: no cove """ pass + # Whether evaluate() may return a non-scalar (list/bool/path) value. + # The legacy FullNameNode only ever yields str/Real; EXPR can yield more. + allows_nonscalar: bool = False + @abstractmethod - def evaluate(self, *, symtab: SymbolTable) -> Any: # pragma: no cover + def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: # pragma: no cover """Evaluate the expression rooted at this node given definitions of symbols in a symbol table. @@ -43,12 +49,26 @@ def evaluate(self, *, symtab: SymbolTable) -> Any: # pragma: no cover Args: symtab (SymbolTable): Symbol definitions. + path_format (Any): Optional EXPR PathFormat used for PATH-typed + values; ignored by nodes that do not evaluate expressions. Returns: Any: Value of the expression. """ pass + def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str: + """Evaluate the expression and coerce the result to its format-string + string form (the value substituted into the surrounding string). + + The default implementation applies Python ``str()`` to the evaluated + value, which is correct for the legacy scalar (``str``/``Real``) nodes. + EXPR-backed nodes override this to use the engine's own spec-defined + coercion (RFC 0005), so e.g. ``true``/``false``/``null`` and lists + render per the specification rather than as Python reprs. + """ + return str(self.evaluate(symtab=symtab, path_format=path_format)) + @abstractmethod def __repr__(self) -> str: # pragma: no cover """String representation of the node for printing.""" @@ -60,6 +80,21 @@ def __repr__(self) -> str: # pragma: no cover MAX_MATCH_DISTANCE_THRESHOLD = 5 +def _missing_symbol_error(name: str, symbols: set[str]) -> ValueError: + """Build the "Variable ... does not exist" error for an unresolved symbol, + appending an edit-distance "Did you mean ...?" hint when a close match + exists. Shared by the legacy and EXPR symbol-reference checks so the + diagnostic stays identical across both.""" + msg = f"Variable {name} does not exist at this location." + distance, closest_matches = closest(symbols, name) + if distance < MAX_MATCH_DISTANCE_THRESHOLD: + if len(closest_matches) == 1: + msg += f" Did you mean: {''.join(closest_matches)}" + elif len(closest_matches) > 1: + msg += f" Did you mean one of: {', '.join(sorted(closest_matches))}" + return ValueError(msg) + + @dataclass class FullNameNode(Node): """Expression tree node representing a fully qualified identifier name. @@ -68,21 +103,143 @@ class FullNameNode(Node): name: str - def validate_symbol_refs(self, *, symbols: set[str]) -> None: + def validate_symbol_refs(self, *, symbols: set[str], symbol_types: Any = None) -> None: if self.name not in symbols: - msg = f"Variable {self.name} does not exist at this location." - distance, closest_matches = closest(symbols, self.name) - if distance < MAX_MATCH_DISTANCE_THRESHOLD: - if len(closest_matches) == 1: - msg += f" Did you mean: {''.join(closest_matches)}" - elif len(closest_matches) > 1: - msg += f" Did you mean one of: {', '.join(sorted(closest_matches))}" - raise ValueError(msg) - - def evaluate(self, *, symtab: SymbolTable) -> Any: + raise _missing_symbol_error(self.name, symbols) + + def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: if self.name not in symtab: raise ValueError(f"{self.name} has no value") return symtab[self.name] def __repr__(self): return f"FullName({self.name})" + + +class ExprNode(Node): + """Expression tree node backed by the Rust ``openjd-expr`` engine. + + Used in place of :class:`FullNameNode` when a template declares the + ``EXPR`` extension. Wraps a Rust ``ParsedExpression`` and satisfies the + same ``Node`` interface so the rest of the model is unaware of which + backend answered. + """ + + allows_nonscalar = True + + def __init__(self, expr: str) -> None: + # Import here to keep the Rust expr surface off the non-EXPR path. + from ._expr_support import parse_expr_or_raise, static_validate_symbol_free + + self.expr = expr + # Raises: model ExpressionError on parse failure. + self._parsed = parse_expr_or_raise(expr) + # Statically validate expressions with no free symbol references so + # that literal/semantic errors (overflow, division by zero, unknown + # function, type mismatch, ...) are caught at `check` time rather than + # only at evaluation. Symbol-referencing expressions are validated once + # their types are known (RFC 0007 parameter types). + if not self._parsed.accessed_symbols: + static_validate_symbol_free(expr) + + @property + def called_functions(self) -> set: + """Names of functions invoked by this expression.""" + return set(self._parsed.called_functions) + + @property + def accessed_symbols(self) -> set: + """Free symbol references in this expression, in full dotted spelling + (e.g. ``"Param.X"``). Local ``let``/comprehension-bound names are + excluded by the engine.""" + return set(self._parsed.accessed_symbols) + + def validate_symbol_refs(self, *, symbols: set[str], symbol_types: Any = None) -> None: + accessed = set(self._parsed.accessed_symbols) + + # Typed validation: if every accessed symbol resolves to a defined + # symbol prefix whose EXPR type is known (e.g. job parameters), validate + # the whole expression against those types. This catches type mismatches + # and resolves method/property access (e.g. Param.File.name on a PATH). + # Only attempted when ALL accessed prefixes are typed-known, so an + # unknown-typed symbol (e.g. a `let` name) safely falls back to the + # name-only check below rather than risking a wrong-type rejection. + if symbol_types and accessed: + from ._expr_support import ( + longest_defined_prefix, + validate_typed_expression, + ) + + prefix_types: dict = {} + fully_typed = True + for name in accessed: + prefix = longest_defined_prefix(name, symbols) + if prefix is None or prefix not in symbol_types: + fully_typed = False + break + prefix_types[prefix] = symbol_types[prefix] + if fully_typed: + # Raises model ExpressionError on a type/method error. + validate_typed_expression(self._parsed, typed_symbols=prefix_types) + self._check_comprehension_shadowing(symbols) + return + + # accessed_symbols are the free variable references in full dotted + # spelling (e.g. "Param.X"); local let-bound names are excluded by the + # engine. Compare against the set of symbols visible at this location. + missing = accessed - symbols + if missing: + raise _missing_symbol_error(sorted(missing)[0], symbols) + self._check_comprehension_shadowing(symbols) + + def _check_comprehension_shadowing(self, symbols: set) -> None: + # Comprehension/local-binding variables may not shadow a variable that + # is in scope (RFC 0007 §3.6). `local_bindings` are the names bound by + # comprehensions/let-expressions inside this expression. + shadowed = set(self._parsed.local_bindings) & symbols + if shadowed: + raise ValueError( + f"Variable {sorted(shadowed)[0]!r} bound by a comprehension shadows " + "a variable that already exists at this location." + ) + + def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + """Evaluate the expression and return the engine's ``ExprValue``. + + Shared by :meth:`evaluate` (which unwraps to the native Python value) + and :meth:`evaluate_to_str` (which uses the engine's spec-defined string + coercion). Re-raises engine errors as the model's ``ExpressionError``. + """ + from ._expr_support import ( + ExprProfile, + map_eval_error, + symtab_to_expr_values, + ) + + values = symtab_to_expr_values( + symtab, types=symtab.expr_types or None, path_format=path_format + ) + try: + return self._parsed.evaluate( + values=values, profile=ExprProfile.current(), path_format=path_format + ) + except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error + raise map_eval_error(exc) + + def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + return self._evaluate_raw(symtab=symtab, path_format=path_format).item() + + def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str: + # ``str(ExprValue)`` applies the engine's RFC 0005 format-string + # coercion (e.g. ``true``/``false``, double-quoted list items, preserved + # Decimal trailing zeros) rather than Python's ``str()`` of the native + # value, which would emit Python reprs (``True``/``None``/``['a', 'b']``). + # A null result interpolates as the empty string, matching the Rust + # engine's FormatString resolution (RFC 0005). + value = self._evaluate_raw(symtab=symtab, path_format=path_format) + if getattr(value, "is_null", False): + return "" + return str(value) + + def __repr__(self): + return f"Expr({self.expr})" diff --git a/src/openjd/model/_format_strings/_parser.py b/src/openjd/model/_format_strings/_parser.py index 1a49a36f..e4725620 100644 --- a/src/openjd/model/_format_strings/_parser.py +++ b/src/openjd/model/_format_strings/_parser.py @@ -4,16 +4,27 @@ from .._errors import ExpressionError, TokenError from .._tokenstream import Token, TokenStream, TokenType -from ._nodes import FullNameNode, Node +from ._nodes import ExprNode, FullNameNode, Node from ._tokens import DotToken, NameToken from .._types import ModelParsingContextInterface +# Defined here (rather than imported from ._expr_support) so that importing the +# parser does not transitively load the Rust expr surface. ExprNode itself +# imports the Rust bindings lazily, so the non-EXPR parse path never touches +# them. +EXPR_EXTENSION = "EXPR" + _tokens: dict[TokenType, Type[Token]] = {TokenType.NAME: NameToken, TokenType.DOT: DotToken} def parse_format_string_expr(expr: str, *, context: ModelParsingContextInterface) -> Node: """Generate an expression tree for the given string interpolation expression. + When the template declares the ``EXPR`` extension, parsing is delegated to + the Rust ``openjd-expr`` engine (via :class:`ExprNode`); otherwise the + legacy ``Name.Dot.Name`` parser is used. Templates that do not declare + ``EXPR`` are byte-for-byte unaffected. + Args: expr (str): A string interpolation expression @@ -24,6 +35,9 @@ def parse_format_string_expr(expr: str, *, context: ModelParsingContextInterface Returns: Node: Root of the expression tree. """ + if EXPR_EXTENSION in context.extensions: + # Raises: model ExpressionError on parse failure. + return ExprNode(expr) return FormatStringExprParser_v2023_09().parse(expr) diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index 4ee8c48e..076e8f51 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -153,6 +153,25 @@ def prevalidate_model_template_variable_references( _template_variable_sources """ + # The EXPR extension is "active" for this template only if it is declared in + # the template's `extensions` (the raw root values) AND supported. During + # prevalidation `context.extensions` still holds the *supported* set (the + # `extensions` field validator that narrows it to the declared set runs + # later), so we read the declared set from the raw values here and stash the + # result on the context for the EXPR-conditional checks below to consult. + declared = values.get("extensions") + expr_enabled = ( + isinstance(declared, list) + and "EXPR" in declared + and (context is None or "EXPR" in context.extensions) + ) + if context is not None: + context._prevalidation_expr_enabled = expr_enabled # type: ignore[attr-defined] + # Build a map of job-parameter symbol -> EXPR type (for type-aware + # expression validation). Only job parameters are mapped; expressions + # referencing other symbols fall back to name-only validation. + context._symbol_types = _build_job_parameter_symbol_types(values) if expr_enabled else {} # type: ignore[attr-defined] + return _validate_model_template_variable_references( cls, _internal_deepcopy(values), @@ -351,6 +370,23 @@ def _validate_model_template_variable_references( model, value, current_scope, symbol_prefix, recursive_pruning=False ) + # Validate EXPR `let` bindings defined by this model: each binding's RHS + # expression is checked against the enclosing scope plus the bindings that + # precede it (chained references), and a binding may not shadow a variable + # from the enclosing scope (RFC 0007 §3.6). + expr_enabled = bool( + context is not None and getattr(context, "_prevalidation_expr_enabled", False) + ) + if expr_enabled: + # EXPR-conditional symbol injection (e.g. Step.Name). Added to the + # model's own scope so it flows to fields whose sources include + # "__self__"; only present when the EXPR extension is enabled. + for symbol in variable_defs.expr_inject: + sym_name = symbol[1:] if symbol.startswith("|") else f"{symbol_prefix}{symbol}" + _add_symbol(value_symbols["__self__"], current_scope, sym_name) + + errors.extend(_validate_let_bindings(value, current_scope, symbols, value_symbols, loc)) + # Recursively validate the contents of FormatStrings within the model. for field_name, field_info in model.model_fields.items(): field_value = value.get(field_name) @@ -388,6 +424,39 @@ def _validate_model_template_variable_references( return errors +# EXPR function-library functions that require host context (path-mapping +# rules / host OS), so they are only available at runtime — not at +# submission/template-resolution (TEMPLATE) time. +_HOST_CONTEXT_FUNCTIONS = {"apply_path_mapping"} + + +def _build_job_parameter_symbol_types(values: dict) -> dict: + """Map job-parameter symbols (``Param.`` and ``RawParam.``) to + their EXPR type strings, derived from the template's ``parameterDefinitions``. + Used for type-aware expression validation; parameters whose type has no + confident EXPR mapping are omitted (callers fall back to name-only checks). + """ + from openjd.model._format_strings._expr_support import expr_type_for_openjd_type + + result: dict[str, str] = {} + defs = values.get("parameterDefinitions") + if not isinstance(defs, list): + return result + for entry in defs: + if not isinstance(entry, dict): + continue + name = entry.get("name") + type_name = entry.get("type") + if not isinstance(name, str) or not isinstance(type_name, str): + continue + expr_type = expr_type_for_openjd_type(type_name) + if expr_type is None: + continue + result[f"Param.{name}"] = expr_type + result[f"RawParam.{name}"] = expr_type + return result + + def _check_format_string( value: FormatString, current_scope: ResolutionScope, @@ -408,14 +477,140 @@ def _check_format_string( # Improperly formed string. Later validation passes will catch and flag this. return errors + symbol_types = getattr(context, "_symbol_types", None) if context is not None else None for expr in f_value.expressions: if expr.expression: try: - expr.expression.validate_symbol_refs(symbols=scoped_symbols) + expr.expression.validate_symbol_refs( + symbols=scoped_symbols, symbol_types=symbol_types + ) except ValueError as exc: errors.append( InitErrorDetails(type="value_error", loc=loc, ctx={"error": exc}, input=value) ) + # Host-context functions (e.g. apply_path_mapping) are only available + # at runtime, not at submission/template-resolution time (RFC 0005). + if current_scope == ResolutionScope.TEMPLATE: + host_fns = expr.expression.called_functions & _HOST_CONTEXT_FUNCTIONS + if host_fns: + errors.append( + InitErrorDetails( + type="value_error", + loc=loc, + ctx={ + "error": ValueError( + f"The function {sorted(host_fns)[0]!r} is only available " + "at runtime (host context), not at submission time." + ) + }, + input=value, + ) + ) + return errors + + +def _validate_let_bindings( + value: dict, + current_scope: ResolutionScope, + symbols: ScopedSymtabs, + value_symbols: dict, + loc: tuple, +) -> list[InitErrorDetails]: + """Validate the RHS expressions of a model's EXPR ``let`` bindings. + + Each binding's expression is validated against the enclosing scope, the + symbols this model itself injects/defines (e.g. ``Session.WorkingDirectory``), + and the names bound by earlier bindings in the same list (chained + references). A binding may not shadow a variable already visible in the + enclosing scope. Strict format/duplicate/limit rules are enforced by the + `let` field validator; this pass covers symbol references and shadowing. + """ + errors = list[InitErrorDetails]() + let_value = value.get("let") + if not isinstance(let_value, list): + return errors + + # Import here to avoid a module import cycle and to keep the Rust expr + # surface off the non-EXPR path. + from openjd.model._format_strings._nodes import ExprNode + + # Names bound by this `let` list (so we can seed `accumulated` with the + # model's own non-let symbols and then add bindings progressively). + let_names: set = set() + for binding in let_value: + if isinstance(binding, str): + n, sep, _ = binding.partition("=") + if sep and n.strip(): + let_names.add(n.strip()) + + enclosing: set = set(symbols[current_scope]) + self_symbols: set = set(value_symbols.get("__self__", ScopedSymtabs())[current_scope]) + # Visible to the bindings: enclosing scope + this model's injected/defined + # symbols, minus the let names themselves (added progressively below). + accumulated: set = (enclosing | self_symbols) - let_names + for index, binding in enumerate(let_value): + if not isinstance(binding, str): + continue + name, sep, rhs = binding.partition("=") + name = name.strip() + rhs = rhs.strip() + if not sep or not name or not rhs: + # Malformed binding; the `let` field validator reports the details. + continue + binding_loc = (*loc, "let", index) + if name in enclosing: + errors.append( + InitErrorDetails( + type="value_error", + loc=binding_loc, + ctx={ + "error": ValueError( + f"The 'let' binding {name!r} shadows a variable that already " + "exists in the enclosing scope." + ) + }, + input=binding, + ) + ) + try: + node = ExprNode(rhs) + except ValueError as exc: + # Parse/static-validation failure in the binding expression. + errors.append( + InitErrorDetails( + type="value_error", loc=binding_loc, ctx={"error": exc}, input=binding + ) + ) + accumulated.add(name) + continue + # A binding may not reference its own name on its RHS: it is not yet + # bound at its own definition point (only earlier bindings are in + # scope). Reported explicitly so the diagnostic says "references + # itself" rather than the misleading "does not exist". Mirrors the + # openjd-rs `validate_let_bindings` self-reference check + # (`'' references itself.`). + if name in node.accessed_symbols: + errors.append( + InitErrorDetails( + type="value_error", + loc=binding_loc, + ctx={ + "error": ValueError(f"The 'let' binding {name!r} cannot reference itself.") + }, + input=binding, + ) + ) + accumulated.add(name) + continue + try: + node.validate_symbol_refs(symbols=accumulated) + except ValueError as exc: + errors.append( + InitErrorDetails( + type="value_error", loc=binding_loc, ctx={"error": exc}, input=binding + ) + ) + accumulated.add(name) return errors @@ -613,6 +808,19 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) symbol_name = f"{symbol_prefix}{symbol}" _add_symbol(symbols["__self__"], current_scope, symbol_name) + # EXPR `let` bindings (RFC 0007 §3.6) define bare-name symbols at this + # model's scope, available to the model's other fields and to subsequent + # bindings. Names are parsed leniently here; the `let` field validator and + # the let-reference validator enforce the strict rules. + let_value = value.get("let") + if isinstance(let_value, list): + for binding in let_value: + if isinstance(binding, str): + bind_name, sep, _ = binding.partition("=") + bind_name = bind_name.strip() + if sep and bind_name: + _add_symbol(symbols["__self__"], current_scope, bind_name) + # Collect the variable definitions exported by the fields of the model for field_name, field_info in model.model_fields.items(): field_value = value.get(field_name) diff --git a/src/openjd/model/_merge_job_parameter.py b/src/openjd/model/_merge_job_parameter.py index 34f6699f..619bb155 100644 --- a/src/openjd/model/_merge_job_parameter.py +++ b/src/openjd/model/_merge_job_parameter.py @@ -20,6 +20,20 @@ JobIntParameterDefinition, ) +# The four original scalar parameter types whose allowedValues / length / +# value-range constraints are cross-merged by +# merge_job_parameter_definitions_for_one. The EXPR-extension types (BOOL, +# RANGE_EXPR, LIST[*]) carry different constraint shapes that are validated +# per-definition at decode time and are not merged here. +_LEGACY_CONSTRAINT_TYPES = frozenset( + { + JobParameterType.STRING, + JobParameterType.INT, + JobParameterType.FLOAT, + JobParameterType.PATH, + } +) + class SourcedParamDefinition(NamedTuple): source: str @@ -167,37 +181,65 @@ def merge_job_parameter_definitions_for_one( errors = list[str]() - # The set of allowedValues for a parameter definition must be a subset as we proceed down the list. - av_ret, err = _merge_allowed_values(params) - if av_ret is not None: - merged_properties["allowedValues"] = av_ret - if err: - errors.extend(err) - - if param_type == JobParameterType.PATH: - ret, err = _merge_path_param_types(params) - if ret: - merged_properties.update(**ret) - if err: - errors.extend(err) - - if param_type in (JobParameterType.STRING, JobParameterType.PATH): - ret, err = _merge_string_kind_param_constraints(params) - if ret: - merged_properties.update(**ret) - if err: - errors.extend(err) - else: - ret, err = _merge_number_kind_param_constraints(params) - if ret: - merged_properties.update(**ret) + # The allowedValues / path / string / number constraint shapes below + # belong only to the four original scalar parameter types. The EXPR + # extension (RFC 0007) types (BOOL, RANGE_EXPR, and the LIST[*] variants) + # carry their own constraint shapes (item:, minLength on lists, etc.) that + # are validated per-definition at decode time and are not cross-merged + # here, so skip the legacy constraint merge for them and just carry + # name/type/default forward. + if param_type in _LEGACY_CONSTRAINT_TYPES: + # The set of allowedValues for a parameter definition must be a subset as we proceed down the list. + av_ret, err = _merge_allowed_values(params) + if av_ret is not None: + merged_properties["allowedValues"] = av_ret if err: errors.extend(err) - if errors: - raise CompatibilityError("\n".join(errors)) - - return parse_model(model=params[0].definition.__class__, obj=merged_properties) + if param_type == JobParameterType.PATH: + ret, err = _merge_path_param_types(params) + if ret: + merged_properties.update(**ret) + if err: + errors.extend(err) + + if param_type in (JobParameterType.STRING, JobParameterType.PATH): + ret, err = _merge_string_kind_param_constraints(params) + if ret: + merged_properties.update(**ret) + if err: + errors.extend(err) + else: + ret, err = _merge_number_kind_param_constraints(params) + if ret: + merged_properties.update(**ret) + if err: + errors.extend(err) + + if errors: + raise CompatibilityError("\n".join(errors)) + + return parse_model(model=params[0].definition.__class__, obj=merged_properties) + + # EXPR-extension types (BOOL, RANGE_EXPR, LIST[*]): not cross-merged. Return + # the last-defined definition with the last-defined default applied. + # ``model_copy`` avoids re-validation, which would otherwise re-trigger the + # EXPR extension gate (the merge has no parsing context to satisfy it). But + # because ``model_copy`` skips validators, re-validate the merged default + # against the base definition's own constraints explicitly so a default + # carried over from another source that violates them is still rejected + # (mirrors the ``parse_model`` re-validation the legacy path performs). + base = params[-1].definition + merged_default = merged_properties.get("default") + if merged_default is not None and merged_default is not base.default: + check_constraints = getattr(base, "_check_constraints", None) + if check_constraints is not None: + try: + check_constraints(merged_default) + except ValueError as err: + raise CompatibilityError(str(err)) + return base.model_copy(update={"default": merged_default}) + return base def _merge_allowed_values( @@ -208,18 +250,19 @@ def _merge_allowed_values( for param in params: definition = param.definition - if not definition.allowedValues: + # Only the original scalar definitions carry allowedValues, and this + # helper only runs for them; getattr keeps it type-safe over the full + # JobParameterDefinition union (the EXPR list/bool/range types have no + # allowedValues and are never merged here). + allowed = getattr(definition, "allowedValues", None) + if not allowed: # If this definition doesn't have a set of allowedValues, then it's unconstrained. # Thus, it's happy with any values and we can move on to the next one. continue if not return_value: - return_value = cast( - Union[set[str], set[int], set[Decimal]], set(definition.allowedValues) - ) + return_value = cast(Union[set[str], set[int], set[Decimal]], set(allowed)) else: - param_as_set = cast( - Union[set[str], set[int], set[Decimal]], set(definition.allowedValues) - ) + param_as_set = cast(Union[set[str], set[int], set[Decimal]], set(allowed)) return_value.intersection_update(param_as_set) if return_value is not None and not return_value: diff --git a/src/openjd/model/_symbol_table.py b/src/openjd/model/_symbol_table.py index 5c5fbcf2..c709c5f9 100644 --- a/src/openjd/model/_symbol_table.py +++ b/src/openjd/model/_symbol_table.py @@ -14,6 +14,12 @@ class SymbolTable: """ _table: dict[str, Any] + # Optional mapping of symbol name -> OpenJD type name (e.g. "INT", + # "LIST[INT]") for EXPR-typed symbols. Used by the EXPR expression engine to + # coerce stored values to their declared type. Declared as a real field + # (rather than stashed dynamically) so it is preserved across copies and + # unions; empty for non-EXPR symbol tables. + expr_types: dict[str, str] def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = None): """Initialize the SymbolTable @@ -23,9 +29,11 @@ def __init__(self, *, source: Optional[Union[SymbolTable, dict[str, Any]]] = Non gets initialized with the contents of the given source. Defaults to None. """ self._table = dict() + self.expr_types = dict() if source is not None: if isinstance(source, SymbolTable): self._table.update(source._table) + self.expr_types.update(source.expr_types) elif isinstance(source, dict): self._table.update(source) else: @@ -65,9 +73,11 @@ def union(self, *symtabs: Union[SymbolTable, dict[str, Any]]) -> SymbolTable: """ retval = SymbolTable() retval._table.update(self._table) + retval.expr_types.update(self.expr_types) for symtab in symtabs: if isinstance(symtab, SymbolTable): retval._table.update(symtab._table) + retval.expr_types.update(symtab.expr_types) elif isinstance(symtab, dict): retval._table.update(symtab) else: diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index fa6499d3..ff65bb7b 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -24,6 +24,14 @@ JobFloatParameterDefinition, JobStringParameterDefinition, JobPathParameterDefinition, + JobBoolParameterDefinition, + JobListStringParameterDefinition, + JobListPathParameterDefinition, + JobListIntParameterDefinition, + JobListFloatParameterDefinition, + JobListBoolParameterDefinition, + JobListListIntParameterDefinition, + JobRangeExprParameterDefinition, ) EnvironmentTemplate = EnvironmentTemplate_2023_09 @@ -34,6 +42,14 @@ JobFloatParameterDefinition, JobStringParameterDefinition, JobPathParameterDefinition, + JobBoolParameterDefinition, + JobListStringParameterDefinition, + JobListPathParameterDefinition, + JobListIntParameterDefinition, + JobListFloatParameterDefinition, + JobListBoolParameterDefinition, + JobListListIntParameterDefinition, + JobRangeExprParameterDefinition, ] StepParameterSpace = StepParameterSpace_2023_09 Step = Step_2023_09 @@ -77,15 +93,31 @@ class ParameterValueType(str, Enum): PATH = "PATH" # This type is only used for task parameters, not job parameters CHUNK_INT = "CHUNK[INT]" + # EXPR extension (RFC 0007) job-parameter value types. These tag a job + # parameter value through preprocessing/create_job; the underlying value is + # carried natively (lists/bools) for these types so the typed EXPR + # SymbolTable builder can coerce them, rather than the stringified form used + # by the original scalar types. + BOOL = "BOOL" + RANGE_EXPR = "RANGE_EXPR" + LIST_STRING = "LIST[STRING]" + LIST_PATH = "LIST[PATH]" + LIST_INT = "LIST[INT]" + LIST_FLOAT = "LIST[FLOAT]" + LIST_BOOL = "LIST[BOOL]" + LIST_LIST_INT = "LIST[LIST[INT]]" @dataclass(frozen=True, **dataclass_kwargs) class ParameterValue: type: ParameterValueType - # All values are strings regardless of types. - # We typecast as needed during processing based on the type - # of the parameter value. - value: str + # For the original scalar types (STRING/INT/FLOAT/PATH) the value is a + # string, typecast as needed during processing. For the EXPR-extension + # types (RFC 0007: BOOL and the LIST[*] variants) the value is carried as + # its native Python type (bool / list) so the typed EXPR SymbolTable + # builder can coerce it — a stringified list cannot be re-parsed back to a + # typed list. + value: Any TaskParameterSet = dict[str, ParameterValue] @@ -194,14 +226,19 @@ def __init__( defines: set[TemplateVariableDef] = set(), field: str = "", inject: set[str] = set(), + expr_inject: set[str] = set(), ): self.symbol_prefix = symbol_prefix self.defines = defines self.field = field self.inject = inject + # Symbols injected only when the EXPR extension is enabled. Applied by + # the variable-reference prevalidator (which has the parsing context); + # the static `inject` set above cannot be made conditional. + self.expr_inject = expr_inject def __repr__(self) -> str: - return f"DefinesTemplateVariables(symbol_prefix={self.symbol_prefix!r}, defines={self.defines!r}, field={self.field!r}, inject={self.inject!r})" + return f"DefinesTemplateVariables(symbol_prefix={self.symbol_prefix!r}, defines={self.defines!r}, field={self.field!r}, inject={self.inject!r}, expr_inject={self.expr_inject!r})" @dataclass(frozen=True, eq=False, **dataclass_kwargs) diff --git a/src/openjd/model/v2023_09/__init__.py b/src/openjd/model/v2023_09/__init__.py index c2f70ff7..13f81813 100644 --- a/src/openjd/model/v2023_09/__init__.py +++ b/src/openjd/model/v2023_09/__init__.py @@ -50,13 +50,21 @@ IntTaskParameterDefinition, Job, JobEnvironmentsList, + JobBoolParameterDefinition, JobFloatParameterDefinition, JobIntParameterDefinition, + JobListBoolParameterDefinition, + JobListFloatParameterDefinition, + JobListIntParameterDefinition, + JobListListIntParameterDefinition, + JobListPathParameterDefinition, + JobListStringParameterDefinition, JobName, JobParameter, JobParameterDefinitionList, JobParameterType, JobPathParameterDefinition, + JobRangeExprParameterDefinition, JobStringParameterDefinition, JobTemplate, JobTemplateName, @@ -138,6 +146,14 @@ "Job", "JobIntParameterDefinition", "JobFloatParameterDefinition", + "JobBoolParameterDefinition", + "JobListBoolParameterDefinition", + "JobListFloatParameterDefinition", + "JobListIntParameterDefinition", + "JobListListIntParameterDefinition", + "JobListPathParameterDefinition", + "JobListStringParameterDefinition", + "JobRangeExprParameterDefinition", "JobName", "JobEnvironmentsList", "JobParameter", diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 83a03821..f7d21254 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -14,6 +14,7 @@ from pydantic import ( field_validator, model_validator, + ConfigDict, StringConstraints, Field, PositiveInt, @@ -110,6 +111,13 @@ class ExtensionName(str, Enum): # Extension for increased limits, format strings in timeout/min/max/notifyPeriodInSeconds, # endOfLine control, and script interpreter syntax sugar FEATURE_BUNDLE_1 = "FEATURE_BUNDLE_1" + # Expression Language (RFCs 0005/0006/0007): rich {{ }} expressions, function + # library, and extended job-parameter types. Evaluated by the Rust openjd-expr + # engine via the openjd._openjd_rs bindings. + EXPR = "EXPR" + # Environment Wrap Actions (RFC 0008): onWrapEnvEnter/onWrapTaskRun/onWrapEnvExit + # on . Requires EXPR. + WRAP_ACTIONS = "WRAP_ACTIONS" ExtensionNameList = Annotated[list[str], Field(min_length=1)] @@ -256,9 +264,12 @@ def __new__(cls, value: str, *, context: ModelParsingContextInterface = ModelPar class ArgString(FormatString): - # All unicode except the [Cc] (control characters) category - # Allow CR, LF, and TAB. - _regex = f"(?-m:^[^{_Cc_characters}]*\\Z)" + # All unicode except the [Cc] (control characters) category, plus the line + # breaks LF (\n) and CR (\r) so multi-line inline scripts can be passed as + # arguments (e.g. `python -c ""`). Matches the openjd-rs + # reference, which accepts LF/CR (but not TAB or other control chars) in + # args with no extension required. + _regex = f"(?-m:^(?:[^{_Cc_characters}]|[\r\n])*\\Z)" def __new__(cls, value: str, *, context: ModelParsingContextInterface = ModelParsingContext()): return super().__new__(cls, value, context=context) @@ -348,6 +359,72 @@ class CancelationMethodTerminate(OpenJDModel_v2023_09): ArgListType = Annotated[list[ArgString], Field(min_length=1)] +# WRAP_ACTIONS (RFC 0008) wrap-hook field names on EnvironmentActions. +_WRAP_ACTION_FIELDS = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") + +# RFC 0008 single-wrap-layer rule. Mirrors the message emitted by openjd-rs +# (validate_v2023_09/wrap_actions.rs::SINGLE_WRAP_LAYER_MSG). +_SINGLE_WRAP_LAYER_MSG = ( + "only one environment in the session stack may define any of onWrapEnvEnter, " + "onWrapTaskRun, onWrapEnvExit (RFC 0008)." +) + + +def _env_defines_wrap_hook(env: Any) -> bool: + """True if the given Environment's script defines any WRAP_ACTIONS hook. + + Used by the single-wrap-layer validation to count the wrap-defining + environments reachable in a session stack. + """ + script = getattr(env, "script", None) + actions = getattr(script, "actions", None) if script is not None else None + if actions is None: + return False + return any(getattr(actions, name, None) is not None for name in _WRAP_ACTION_FIELDS) + + +# RFC 0008 wrapped-context variable namespaces and where each may be +# referenced. `WrappedAction.*` is available in all three wrap hooks; +# `WrappedEnv.*` only in the env-enter/exit hooks; `WrappedStep.*` only in the +# task-run hook. None of them may be referenced outside the wrap hooks. +_WRAPPED_NAMESPACES = ("WrappedAction", "WrappedEnv", "WrappedStep") +_WRAP_HOOK_ALLOWED_NAMESPACES = { + "onWrapEnvEnter": {"WrappedAction", "WrappedEnv"}, + "onWrapEnvExit": {"WrappedAction", "WrappedEnv"}, + "onWrapTaskRun": {"WrappedAction", "WrappedStep"}, +} + + +def _action_referenced_namespaces(action: Any) -> set[str]: + """Collect the set of wrapped-context namespaces (``WrappedAction`` / + ``WrappedEnv`` / ``WrappedStep``) referenced by an Action's format strings. + + Inspects every FormatString-bearing field of the Action: ``command``, each + of ``args``, and ``timeout`` (which is a FormatString under the + FEATURE_BUNDLE_1 extension, and a plain int otherwise — non-FormatString + values are skipped). + """ + referenced: set[str] = set() + if action is None: + return referenced + format_strings = [ + getattr(action, "command", None), + *(getattr(action, "args", None) or []), + getattr(action, "timeout", None), + ] + for fs in format_strings: + if not isinstance(fs, FormatString): + continue + for expr_info in fs.expressions: + expr = expr_info.expression + if expr is None: + continue + for symbol in expr.accessed_symbols: + namespace = symbol.split(".", 1)[0] + if namespace in _WRAPPED_NAMESPACES: + referenced.add(namespace) + return referenced + class Action(OpenJDModel_v2023_09): """An Action to run. @@ -414,26 +491,116 @@ class EnvironmentActions(OpenJDModel_v2023_09): as part of a Session. onExit (Optional[Action]): Action to run when exiting the environment in a Session. - - Note: Must define at least one of onEnter or onExit + onWrapEnvEnter (Optional[Action]): WRAP_ACTIONS — runs instead of the + onEnter of every inner environment while this environment is active. + onWrapTaskRun (Optional[Action]): WRAP_ACTIONS — runs instead of every + task's onRun while this environment is active. + onWrapEnvExit (Optional[Action]): WRAP_ACTIONS — runs instead of the + onExit of every inner environment while this environment is active. + + Note: Must define at least one of onEnter or onExit (or, with the + WRAP_ACTIONS extension, the wrap actions). The three wrap actions are + all-or-nothing and require the WRAP_ACTIONS extension (which itself + requires EXPR). """ onEnter: Optional[Action] = Field(None) # noqa: N815 onExit: Optional[Action] = Field(None) # noqa: N815 + # WRAP_ACTIONS extension (RFC 0008). Gated in the validator below. + onWrapEnvEnter: Optional[Action] = Field(None) # noqa: N815 + onWrapTaskRun: Optional[Action] = Field(None) # noqa: N815 + onWrapEnvExit: Optional[Action] = Field(None) # noqa: N815 @model_validator(mode="before") @classmethod - def _requires_oneof(cls, values: dict[str, Any]) -> dict[str, Any]: - """A validator that runs on the model data before parsing.""" + def _requires_oneof(cls, values: dict[str, Any], info: ValidationInfo) -> dict[str, Any]: + """A validator that runs on the model data before parsing. + + Enforces, for the WRAP_ACTIONS extension (RFC 0008): + - wrap actions require the WRAP_ACTIONS extension to be declared; + - WRAP_ACTIONS requires the EXPR extension (hard prerequisite); + - the three wrap actions are all-or-nothing. + Otherwise preserves the legacy "must define onEnter or onExit" rule. + """ if not isinstance(values, dict): raise ValueError("Expected a dictionary of values") + context = cast(Optional[ModelParsingContext], info.context) if info else None + extensions = context.extensions if context else set() + + wrap_values = {name: values.get(name) for name in _WRAP_ACTION_FIELDS} + any_wrap = any(v is not None for v in wrap_values.values()) + + if any_wrap: + # Extension gating is a template-decode concern and only applies + # when a parsing context is present. During job instantiation + # (create_job re-validates the model without a ModelParsingContext) + # the template has already been validated at decode time, so the + # extension-requirement checks are skipped then -- mirroring the + # `if context` guard the other extension gates in this module use. + if context is not None: + if "WRAP_ACTIONS" not in extensions: + raise ValueError( + "The onWrapEnvEnter, onWrapTaskRun, and onWrapEnvExit actions " + "require the WRAP_ACTIONS extension." + ) + if "EXPR" not in extensions: + raise ValueError("The WRAP_ACTIONS extension requires the EXPR extension.") + if not all(v is not None for v in wrap_values.values()): + raise ValueError( + "When any wrap action is defined, all of onWrapEnvEnter, " + "onWrapTaskRun, and onWrapEnvExit must be defined." + ) + # A wrap environment with the three hooks satisfies the + # "at least one action" requirement. + return values + on_enter = values.get("onEnter") on_exit = values.get("onExit") if on_enter is None and on_exit is None: raise ValueError("Must define one of: onEnter or onExit") return values + @model_validator(mode="after") + def _validate_wrapped_variable_scope(self, info: ValidationInfo) -> Self: + # RFC 0008 scope rule: WrappedAction.* may be referenced only in the + # three wrap hooks; WrappedEnv.* only in onWrapEnvEnter/onWrapEnvExit; + # WrappedStep.* only in onWrapTaskRun. None of them may appear in the + # ordinary onEnter/onExit actions. Schedulers must reject templates + # that violate this; mirrors openjd-rs (which enforces it via its + # per-scope function/symbol library split). + context = cast(Optional[ModelParsingContext], info.context) if info else None + extensions = context.extensions if context else set() + if "EXPR" not in extensions: + # Without EXPR the wrapped variables are never resolvable symbols, + # and the format strings carry no accessed-symbol set to inspect. + return self + + errors = list[InitErrorDetails]() + for field_name in ("onEnter", "onExit", *_WRAP_ACTION_FIELDS): + action = getattr(self, field_name, None) + if action is None: + continue + allowed = _WRAP_HOOK_ALLOWED_NAMESPACES.get(field_name, set()) + referenced = _action_referenced_namespaces(action) + for namespace in sorted(referenced - allowed): + errors.append( + InitErrorDetails( + type="value_error", + loc=(field_name,), + ctx={ + "error": ValueError( + f"The {namespace}.* variables may not be referenced in " + f"{field_name} (RFC 0008)." + ) + }, + input=action, + ) + ) + if errors: + raise ValidationError.from_exception_data(self.__class__.__name__, errors) + return self + # --------------------- Embedded Files type ------------------------- @@ -550,6 +717,59 @@ class ScriptInterpreter(str, Enum): NODE = "node" +LET_MAX_BINDINGS = 50 +_LET_NAME_RE = re.compile(r"^[a-z_][A-Za-z0-9_]*$") + + +def parse_let_bindings(value: Any) -> list[tuple[str, str]]: + """Parse a ``let`` field value (list of ``"name = expression"`` strings) + into ``(name, expression)`` pairs. Raises ValueError on malformed input. + """ + if not isinstance(value, list): + raise ValueError("'let' must be a list of 'name = expression' bindings.") + result: list[tuple[str, str]] = [] + for binding in value: + if not isinstance(binding, str): + raise ValueError("Each 'let' binding must be a 'name = expression' string.") + name, sep, expr = binding.partition("=") + if not sep: + raise ValueError( + f"A 'let' binding must be of the form 'name = expression': {binding!r}" + ) + name = name.strip() + expr = expr.strip() + if not _LET_NAME_RE.match(name): + raise ValueError(f"A 'let' binding name must be a valid identifier: {name!r}") + if not expr: + raise ValueError(f"A 'let' binding must define an expression: {binding!r}") + result.append((name, expr)) + return result + + +def validate_let_field(value: Any, info: ValidationInfo, *, simple_action: bool = False) -> Any: + """Validate a ``let`` field: EXPR (and FEATURE_BUNDLE_1 for SimpleAction) + gating, non-empty, at most ``LET_MAX_BINDINGS``, each binding parses, and + unique names. Symbol-reference and shadow rules are validated separately by + the variable-reference prevalidator. + """ + if value is None: + return value + context = cast(Optional[ModelParsingContext], info.context) + if context and "EXPR" not in context.extensions: + raise ValueError("The 'let' field requires the EXPR extension.") + if simple_action and context and "FEATURE_BUNDLE_1" not in context.extensions: + raise ValueError("A SimpleAction 'let' requires the FEATURE_BUNDLE_1 extension.") + if isinstance(value, list) and len(value) == 0: + raise ValueError("'let' must define at least one binding.") + if isinstance(value, list) and len(value) > LET_MAX_BINDINGS: + raise ValueError(f"'let' must define at most {LET_MAX_BINDINGS} bindings.") + bindings = parse_let_bindings(value) + names = [name for name, _ in bindings] + if len(names) != len(set(names)): + raise ValueError("'let' binding names must be unique.") + return value + + class SimpleAction(OpenJDModel_v2023_09): """Syntax sugar for a script action with a specific interpreter. @@ -569,6 +789,28 @@ class SimpleAction(OpenJDModel_v2023_09): cancelation: Optional[ Union[CancelationMethodNotifyThenTerminate, CancelationMethodTerminate] ] = Field(None, discriminator="mode") + let: Optional[list[str]] = None + + # SimpleAction is syntax sugar that resolves to a StepScript (TASK scope), + # so it sees the same session-scope variables, and its `let` names (in + # __self__) are visible to its script and args. + _template_variable_scope = ResolutionScope.TASK + _template_variable_definitions = DefinesTemplateVariables( + inject={ + f"|{ValueReferenceConstants.WORKING_DIRECTORY.value}", + f"|{ValueReferenceConstants.HAS_PATH_MAPPING_RULES.value}", + f"|{ValueReferenceConstants.PATH_MAPPING_RULES_FILE.value}", + }, + ) + _template_variable_sources = { + "script": {"__self__"}, + "args": {"__self__"}, + } + + @field_validator("let") + @classmethod + def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: + return validate_let_field(v, info, simple_action=True) @field_validator("timeout", mode="before") @classmethod @@ -599,6 +841,7 @@ class StepScript(OpenJDModel_v2023_09): actions: StepActions embeddedFiles: Optional[EmbeddedFiles] = None # noqa: N815 + let: Optional[list[str]] = None _template_variable_scope = ResolutionScope.TASK _template_variable_definitions = DefinesTemplateVariables( @@ -614,6 +857,11 @@ class StepScript(OpenJDModel_v2023_09): "embeddedFiles": {"embeddedFiles", "__self__"}, } + @field_validator("let") + @classmethod + def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: + return validate_let_field(v, info) + @field_validator("embeddedFiles") @classmethod def _unique_names(cls, v: Optional[EmbeddedFiles]) -> Optional[EmbeddedFiles]: @@ -636,6 +884,7 @@ class EnvironmentScript(OpenJDModel_v2023_09): actions: EnvironmentActions embeddedFiles: Optional[EmbeddedFiles] = None # noqa: N815 + let: Optional[list[str]] = None _template_variable_definitions = DefinesTemplateVariables( symbol_prefix="|Env.", @@ -643,6 +892,19 @@ class EnvironmentScript(OpenJDModel_v2023_09): f"|{ValueReferenceConstants.WORKING_DIRECTORY.value}", f"|{ValueReferenceConstants.HAS_PATH_MAPPING_RULES.value}", f"|{ValueReferenceConstants.PATH_MAPPING_RULES_FILE.value}", + # WRAP_ACTIONS (RFC 0008) variables, visible inside the wrap hooks. + # First-cut note: injected at the script scope, so they are visible + # to all of this environment's actions rather than only the wrap + # hooks. Precise per-hook scoping (and rejecting WrappedAction.* / + # WrappedEnv.* / WrappedStep.* outside their hook) is deferred and + # belongs with the sessions runtime that produces these variables; + # the conformance scope-restriction cases are runtime `jobs/` tests. + "|WrappedAction.Command", + "|WrappedAction.Args", + "|WrappedAction.Environment", + "|WrappedAction.Timeout", + "|WrappedEnv.Name", + "|WrappedStep.Name", }, ) _template_variable_sources = { @@ -650,6 +912,11 @@ class EnvironmentScript(OpenJDModel_v2023_09): "embeddedFiles": {"embeddedFiles", "__self__"}, } + @field_validator("let") + @classmethod + def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: + return validate_let_field(v, info) + @field_validator("embeddedFiles") @classmethod def _unique_names(cls, v: Optional[EmbeddedFiles]) -> Optional[EmbeddedFiles]: @@ -817,6 +1084,19 @@ def _validate_range_elements(cls, value: Any) -> Any: return value +def _validate_range_expr_requires_expr(value: Any, info: ValidationInfo) -> Any: + """Shared ``range`` field validator for task-parameter definitions: a range + expression (a ``RangeString`` format string, RFC 0007) requires the EXPR + extension to be declared. Used by the Float/String/Path task-parameter + definitions so the gate and its message are single-sourced. + """ + if isinstance(value, RangeString): + context = cast(Optional[ModelParsingContext], info.context) + if context and "EXPR" not in context.extensions: + raise ValueError("A range expression (format string) requires the EXPR extension.") + return value + + class FloatTaskParameterDefinition(OpenJDModel_v2023_09): """Definition of a float-typed Task Parameter and its value range. @@ -828,7 +1108,7 @@ class FloatTaskParameterDefinition(OpenJDModel_v2023_09): name: Identifier type: Literal[TaskParameterType.FLOAT] - range: FloatRangeList + range: Union[FloatRangeList, RangeString] = Field(union_mode="left_to_right") _template_variable_definitions = DefinesTemplateVariables( defines={ @@ -855,6 +1135,11 @@ def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: return validate_list_field(value, validate_float_fmtstring_field, context=context) return value + @field_validator("range") + @classmethod + def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: + return _validate_range_expr_requires_expr(value, info) + class StringTaskParameterDefinition(OpenJDModel_v2023_09): """Definition of a string-typed Task Parameter and its value range. @@ -867,7 +1152,7 @@ class StringTaskParameterDefinition(OpenJDModel_v2023_09): name: Identifier type: Literal[TaskParameterType.STRING] - range: StringRangeList + range: Union[StringRangeList, RangeString] = Field(union_mode="left_to_right") _template_variable_definitions = DefinesTemplateVariables( defines={ @@ -883,6 +1168,11 @@ class StringTaskParameterDefinition(OpenJDModel_v2023_09): exclude_fields={"name"}, ) + @field_validator("range") + @classmethod + def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: + return _validate_range_expr_requires_expr(value, info) + class PathTaskParameterDefinition(OpenJDModel_v2023_09): """Definition of a path-typed Task Parameter and its value range. @@ -895,7 +1185,7 @@ class PathTaskParameterDefinition(OpenJDModel_v2023_09): name: Identifier type: Literal[TaskParameterType.PATH] - range: StringRangeList + range: Union[StringRangeList, RangeString] = Field(union_mode="left_to_right") _template_variable_definitions = DefinesTemplateVariables( defines={ @@ -911,6 +1201,11 @@ class PathTaskParameterDefinition(OpenJDModel_v2023_09): exclude_fields={"name"}, ) + @field_validator("range") + @classmethod + def _range_expr_requires_expr(cls, value: Any, info: ValidationInfo) -> Any: + return _validate_range_expr_requires_expr(value, info) + class ChunkIntTaskParameterDefinition(OpenJDModel_v2023_09): """Definition of an integer-typed Task Parameter, that is processed as @@ -1257,6 +1552,16 @@ class JobParameterType(str, Enum): PATH = "PATH" INT = "INT" FLOAT = "FLOAT" + # EXPR extension (RFC 0007) — gated on the EXPR extension in the + # per-definition validators. + BOOL = "BOOL" + LIST_STRING = "LIST[STRING]" + LIST_PATH = "LIST[PATH]" + LIST_INT = "LIST[INT]" + LIST_FLOAT = "LIST[FLOAT]" + LIST_BOOL = "LIST[BOOL]" + LIST_LIST_INT = "LIST[LIST[INT]]" + RANGE_EXPR = "RANGE_EXPR" AllowedParameterStringValueList = Annotated[list[ParameterStringValue], Field(min_length=1)] @@ -1280,7 +1585,10 @@ class JobParameterType(str, Enum): # Target model for a job parameter when instantiating a job. class JobParameter(OpenJDModel_v2023_09): type: JobParameterType - value: str + # For the original scalar types the value is a string. For the EXPR + # extension types (RFC 0007: BOOL and the LIST[*] variants) the value is the + # native Python value (bool / list) produced by create_job. + value: Any description: Optional[Description] = None @@ -2687,6 +2995,16 @@ class StepTemplate(OpenJDModel_v2023_09): cmd: Optional[SimpleAction] = None powershell: Optional[SimpleAction] = None node: Optional[SimpleAction] = None + let: Optional[list[str]] = None + + # Step.Name is available to the step's script and step environments, but + # only when the EXPR extension is enabled (RFC 0007). + _template_variable_definitions = DefinesTemplateVariables(expr_inject={"|Step.Name"}) + + @field_validator("let") + @classmethod + def _validate_let(cls, v: Any, info: ValidationInfo) -> Any: + return validate_let_field(v, info) _template_variable_sources = { "script": {"__self__", "parameterSpace"}, @@ -2699,7 +3017,7 @@ class StepTemplate(OpenJDModel_v2023_09): } _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=Step), - exclude_fields={"python", "bash", "cmd", "powershell", "node"}, + exclude_fields={"python", "bash", "cmd", "powershell", "node", "let"}, transform=lambda t: cast("StepTemplate", t).resolve_syntax_sugar(), ) @@ -2797,6 +3115,15 @@ def resolve_syntax_sugar(self) -> "StepTemplate": StepTemplate: A new StepTemplate with de-sugared script, or self if no sugar. """ if self.script: + # Step-level `let` (RFC 0007) is excluded from the instantiated Step + # by the job-creation metadata, so fold it into the script's own + # `let` (step bindings first, then the script's) so it survives into + # the Job and the runtime resolves it. The model has already + # validated reference/shadowing rules across both scopes at decode. + if self.let: + merged_let = [*self.let, *(self.script.let or [])] + new_script = self.script.model_copy(update={"let": merged_let}) + return self.model_copy(update={"script": new_script, "let": None}) return self for name, (command, ext, arg_prefix) in _INTERPRETER_MAP.items(): @@ -2833,6 +3160,10 @@ def resolve_syntax_sugar(self) -> "StepTemplate": cancelation=simple_action.cancelation, ) ), + # Carry step-level `let` (RFC 0007) and the SimpleAction's own + # `let` onto the de-sugared script (step bindings first) so they + # are preserved into the Job and resolved at runtime. + let=([*(self.let or []), *(simple_action.let or [])] or None), embeddedFiles=[ EmbeddedFileText.model_construct( name=embedded_name, @@ -2851,6 +3182,480 @@ def resolve_syntax_sugar(self) -> "StepTemplate": StepTemplateList = Annotated[list[StepTemplate], Field(min_length=1)] +# ----- EXPR extension (RFC 0007) BOOL job parameter ----- + + +class BoolUserInterfaceControl(str, Enum): + CHECK_BOX = "CHECK_BOX" + HIDDEN = "HIDDEN" + + +class JobBoolParameterDefinitionUserInterface(OpenJDModel_v2023_09): + """User interface attributes for a job bool parameter.""" + + control: Optional[BoolUserInterfaceControl] = None + label: Optional[UserInterfaceLabelStringValue] = None + groupLabel: Optional[UserInterfaceLabelStringValue] = None # noqa: N815 + + +# Accepted string spellings for boolean defaults/values (case-insensitive), +# per RFC 0007 (BOOL parameter type). +_BOOL_TRUE_STRINGS = frozenset({"true", "yes", "on", "1"}) +_BOOL_FALSE_STRINGS = frozenset({"false", "no", "off", "0"}) + + +def _coerce_bool_value(value: Any) -> bool: + """Coerce an RFC 0007 BOOL value to a Python bool, raising ValueError for + anything outside the accepted set (bool, int 0/1, float 0.0/1.0, or a + case-insensitive true/false/yes/no/on/off/1/0 string). + """ + if isinstance(value, bool): + return value + if isinstance(value, int): # bool already handled above + if value in (0, 1): + return bool(value) + raise ValueError("BOOL value as an integer must be 0 or 1.") + if isinstance(value, float): + if value in (0.0, 1.0): + return bool(value) + raise ValueError("BOOL value as a float must be 0.0 or 1.0.") + if isinstance(value, str): + low = value.lower() + if low in _BOOL_TRUE_STRINGS: + return True + if low in _BOOL_FALSE_STRINGS: + return False + raise ValueError( + "BOOL value as a string must be one of (case-insensitive): " + "true, false, yes, no, on, off, 1, 0." + ) + raise ValueError("BOOL value must be a boolean, 0/1, 0.0/1.0, or a boolean string.") + + +class JobBoolParameterDefinition(OpenJDModel_v2023_09): + """A Job Parameter of type bool (EXPR extension, RFC 0007). + + Attributes: + name (Identifier): A name by which the parameter is referenced. + type (JobParameterType.BOOL): discriminator to identify the type. + userInterface (Optional[JobBoolParameterDefinitionUserInterface]): + User interface properties for this parameter. + description (Optional[Description]): Free-form description. + default (Optional[bool]): Default value if one is not provided. + """ + + name: Identifier + type: Literal[JobParameterType.BOOL] + userInterface: Optional[JobBoolParameterDefinitionUserInterface] = None + description: Optional[Description] = None + default: Optional[bool] = None + + _template_variable_definitions = DefinesTemplateVariables( + defines={ + TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + }, + field="name", + ) + _template_variable_sources = {"__export__": {"__self__"}} + _job_creation_metadata = JobCreationMetadata( + create_as=JobCreateAsMetadata(model=JobParameter), + exclude_fields={"name", "userInterface", "default"}, + adds_fields=lambda this, symtab: { + "value": symtab[f"RawParam.{cast(JobBoolParameterDefinition, this).name}"] + }, + ) + + @field_validator("type") + @classmethod + def _requires_expr_extension( + cls, value: JobParameterType, info: ValidationInfo + ) -> JobParameterType: + context = cast(Optional[ModelParsingContext], info.context) + if context and "EXPR" not in context.extensions: + raise ValueError("The BOOL job parameter type requires the EXPR extension.") + return value + + @field_validator("default", mode="before") + @classmethod + def _validate_default(cls, value: Optional[Any]) -> Optional[bool]: + if value is None: + return None + # Raises ValueError for values outside the accepted boolean set. + return _coerce_bool_value(value) + + # override + def _check_constraints(self, value: Any) -> None: + if value is None: + raise ValueError(f"No value given for {self.name}.") + try: + _coerce_bool_value(value) + except ValueError as exc: + raise ValueError(f"Value ({value}) for parameter {self.name}: {exc}") + + +# ----- EXPR extension (RFC 0007) LIST[*] and RANGE_EXPR job parameters ----- + + +class _ListParameterUserInterface(OpenJDModel_v2023_09): + # Permissive: list parameter UI controls vary by element type + # (LINE_EDIT_LIST, SPIN_BOX_LIST, CHECK_BOX_LIST, HIDDEN, ...) and carry + # type-specific fields (singleStepDelta, decimals, fileFilters, ...). + # The UI block is non-functional metadata and not exercised by the + # conformance .invalid fixtures, so accept any control name and ignore + # extra UI fields rather than enumerating every variant. + model_config = ConfigDict(extra="ignore", frozen=True) + + control: Optional[str] = None + label: Optional[UserInterfaceLabelStringValue] = None + groupLabel: Optional[UserInterfaceLabelStringValue] = None # noqa: N815 + + +class _ListStringItemConstraint(OpenJDModel_v2023_09): + """`item:` constraints for LIST[STRING] / LIST[PATH] elements.""" + + allowedValues: Optional[list[ParameterStringValue]] = None # noqa: N815 + minLength: Optional[StrictInt] = None # noqa: N815 + maxLength: Optional[StrictInt] = None # noqa: N815 + + +class _ListIntItemConstraint(OpenJDModel_v2023_09): + """`item:` constraints for LIST[INT] elements (and the inner items of + LIST[LIST[INT]]).""" + + allowedValues: Optional[list[StrictInt]] = None # noqa: N815 + minValue: Optional[StrictInt] = None # noqa: N815 + maxValue: Optional[StrictInt] = None # noqa: N815 + + +class _ListFloatItemConstraint(OpenJDModel_v2023_09): + """`item:` constraints for LIST[FLOAT] elements.""" + + allowedValues: Optional[list[Decimal]] = None # noqa: N815 + minValue: Optional[Decimal] = None # noqa: N815 + maxValue: Optional[Decimal] = None # noqa: N815 + + +class _ListListIntInnerConstraint(OpenJDModel_v2023_09): + """`item:` constraints for the inner lists of LIST[LIST[INT]].""" + + minLength: Optional[StrictInt] = None # noqa: N815 + maxLength: Optional[StrictInt] = None # noqa: N815 + item: Optional[_ListIntItemConstraint] = None + + +def _check_list_length( + name: str, values: list[Any], min_len: Optional[int], max_len: Optional[int] +) -> None: + if min_len is not None and len(values) < min_len: + raise ValueError( + f"Parameter {name}: list length {len(values)} is below minLength {min_len}." + ) + if max_len is not None and len(values) > max_len: + raise ValueError( + f"Parameter {name}: list length {len(values)} exceeds maxLength {max_len}." + ) + + +def _check_string_item(name: str, item: Any, c: Optional[_ListStringItemConstraint]) -> None: + if not isinstance(item, str): + raise ValueError( + f"Parameter {name}: list items must be strings, got {type(item).__name__}." + ) + if c is None: + return + if c.minLength is not None and len(item) < c.minLength: + raise ValueError( + f"Parameter {name}: item {item!r} is shorter than item.minLength {c.minLength}." + ) + if c.maxLength is not None and len(item) > c.maxLength: + raise ValueError( + f"Parameter {name}: item {item!r} is longer than item.maxLength {c.maxLength}." + ) + if c.allowedValues is not None and item not in c.allowedValues: + raise ValueError(f"Parameter {name}: item {item!r} is not in item.allowedValues.") + + +def _check_numeric_item_bounds( + name: str, + item: Any, + compare: Any, + min_value: Any, + max_value: Any, + allowed_values: Any, +) -> None: + """Shared minValue / maxValue / allowedValues check for the numeric LIST + item types. ``item`` is used only in error messages; ``compare`` is the + value the bounds are tested against (the int itself for LIST[INT], the + Decimal form for LIST[FLOAT]).""" + if min_value is not None and compare < min_value: + raise ValueError(f"Parameter {name}: item {item} is below item.minValue {min_value}.") + if max_value is not None and compare > max_value: + raise ValueError(f"Parameter {name}: item {item} is above item.maxValue {max_value}.") + if allowed_values is not None and compare not in allowed_values: + raise ValueError(f"Parameter {name}: item {item} is not in item.allowedValues.") + + +def _check_int_item(name: str, item: Any, c: Optional[_ListIntItemConstraint]) -> None: + if isinstance(item, bool) or not isinstance(item, int): + raise ValueError( + f"Parameter {name}: list items must be integers, got {type(item).__name__}." + ) + if c is None: + return + _check_numeric_item_bounds(name, item, item, c.minValue, c.maxValue, c.allowedValues) + + +def _check_float_item(name: str, item: Any, c: Optional[_ListFloatItemConstraint]) -> None: + if isinstance(item, bool) or not isinstance(item, (int, float, Decimal)): + raise ValueError( + f"Parameter {name}: list items must be numbers, got {type(item).__name__}." + ) + val = Decimal(str(item)) + if c is None: + return + _check_numeric_item_bounds(name, item, val, c.minValue, c.maxValue, c.allowedValues) + + +def _expr_param_gate(value: JobParameterType, info: ValidationInfo) -> JobParameterType: + context = cast(Optional[ModelParsingContext], info.context) + if context and "EXPR" not in context.extensions: + raise ValueError(f"The {value.value} job parameter type requires the EXPR extension.") + return value + + +def _normalize_parameter_type_case(value: Any, info: ValidationInfo) -> Any: + """Uppercase the ``type`` discriminator of each parameter definition when + the EXPR extension is enabled (RFC 0007 makes parameter type names + case-insensitive, e.g. ``int`` == ``INT``, ``list[int]`` == ``LIST[INT]``). + Runs before discriminated-union resolution. + """ + context = cast(Optional[ModelParsingContext], info.context) + if not (context and "EXPR" in context.extensions): + return value + if not isinstance(value, list): + return value + normalized: list[Any] = [] + for item in value: + if isinstance(item, dict) and isinstance(item.get("type"), str): + item = {**item, "type": item["type"].upper()} + normalized.append(item) + return normalized + + +_LIST_PARAM_VARS = DefinesTemplateVariables( + defines={ + TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE), + TemplateVariableDef(prefix="|RawParam.", resolves=ResolutionScope.TEMPLATE), + }, + field="name", +) + +# Shared create-job metadata for the EXPR LIST[*]/RANGE_EXPR job-parameter +# definitions (RFC 0007). Like the scalar definitions, instantiation produces a +# JobParameter carrying the resolved value taken from the symbol table. The +# native value is supplied by create_job (RawParam.); the type-specific +# definition fields (constraints, userInterface) are dropped. The exclude set +# is a superset covering every constraint field used across the list/range +# definitions, so the same metadata applies to all of them. +_LIST_RANGE_JOB_CREATION_METADATA = JobCreationMetadata( + create_as=JobCreateAsMetadata(model=JobParameter), + exclude_fields={ + "name", + "userInterface", + "default", + "minLength", + "maxLength", + "item", + "objectType", + "dataFlow", + }, + adds_fields=lambda this, symtab: {"value": symtab[f"RawParam.{getattr(this, 'name')}"]}, +) + + +class _JobListParameterDefinitionBase(OpenJDModel_v2023_09, JobParameterInterface): + """Shared base for the EXPR (RFC 0007) ``LIST[*]`` job-parameter definitions. + + Collects the fields and machinery common to every list parameter type — the + name/type-gate, the shared create-job metadata and template-variable + definitions, list-length checking, and the EXPR-extension gate. Each concrete + subclass declares only its ``type`` literal, its ``item`` constraint (if any), + and overrides :meth:`_check_item` to validate a single element. + + Both the template ``default`` (via :meth:`_validate_default`) and any + user-supplied value at create time (via :meth:`_check_constraints`) are + validated through the same length + per-item checks, so list constraints are + enforced consistently in both paths. + """ + + name: Identifier + userInterface: Optional[_ListParameterUserInterface] = None + description: Optional[Description] = None + minLength: Optional[StrictInt] = None # noqa: N815 + maxLength: Optional[StrictInt] = None # noqa: N815 + default: Optional[list[Any]] = None + + _job_creation_metadata = _LIST_RANGE_JOB_CREATION_METADATA + _template_variable_definitions = _LIST_PARAM_VARS + _template_variable_sources = {"__export__": {"__self__"}} + + @field_validator("type", check_fields=False) + @classmethod + def _validate_type_gate(cls, value: JobParameterType, info: ValidationInfo) -> JobParameterType: + return _expr_param_gate(value, info) + + def _check_item(self, item: Any) -> None: # pragma: no cover - overridden + """Validate a single list element. Overridden per element type.""" + raise NotImplementedError + + def _check_list_value(self, value: list[Any]) -> None: + """Length + per-item validation shared by default- and value-checking.""" + _check_list_length(self.name, value, self.minLength, self.maxLength) + for it in value: + self._check_item(it) + + @model_validator(mode="after") + def _validate_default(self) -> Self: + if self.default is not None: + self._check_list_value(self.default) + return self + + # override (JobParameterInterface) — enforce list constraints on a + # user-supplied value at create time, mirroring the scalar definitions. + def _check_constraints(self, value: Any) -> None: + if value is None: + raise ValueError(f"No value given for {self.name}.") + if not isinstance(value, list): + raise ValueError( + f"Parameter {self.name}: value must be a list, got {type(value).__name__}." + ) + self._check_list_value(value) + + +class JobListStringParameterDefinition(_JobListParameterDefinitionBase): + """LIST[STRING] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_STRING] + item: Optional[_ListStringItemConstraint] = None + + def _check_item(self, item: Any) -> None: + _check_string_item(self.name, item, self.item) + + +class JobListPathParameterDefinition(_JobListParameterDefinitionBase): + """LIST[PATH] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_PATH] + objectType: Optional[JobPathParameterDefinitionObjectType] = None # noqa: N815 + dataFlow: Optional[JobPathParameterDefinitionDataFlow] = None # noqa: N815 + item: Optional[_ListStringItemConstraint] = None + + def _check_item(self, item: Any) -> None: + _check_string_item(self.name, item, self.item) + + +class JobListIntParameterDefinition(_JobListParameterDefinitionBase): + """LIST[INT] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_INT] + item: Optional[_ListIntItemConstraint] = None + + def _check_item(self, item: Any) -> None: + _check_int_item(self.name, item, self.item) + + +class JobListFloatParameterDefinition(_JobListParameterDefinitionBase): + """LIST[FLOAT] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_FLOAT] + item: Optional[_ListFloatItemConstraint] = None + + def _check_item(self, item: Any) -> None: + _check_float_item(self.name, item, self.item) + + +class JobListBoolParameterDefinition(_JobListParameterDefinitionBase): + """LIST[BOOL] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_BOOL] + + def _check_item(self, item: Any) -> None: + # Reuse the BOOL coercion: each item must be boolean-like. + try: + _coerce_bool_value(item) + except ValueError as exc: + raise ValueError(f"Parameter {self.name}: {exc}") + + +class JobListListIntParameterDefinition(_JobListParameterDefinitionBase): + """LIST[LIST[INT]] job parameter (EXPR extension, RFC 0007).""" + + type: Literal[JobParameterType.LIST_LIST_INT] + item: Optional[_ListListIntInnerConstraint] = None + + def _check_item(self, item: Any) -> None: + inner_c = self.item + inner_item_c = inner_c.item if inner_c else None + if not isinstance(item, list): + raise ValueError( + f"Parameter {self.name}: every element of LIST[LIST[INT]] must be a list." + ) + if inner_c is not None: + _check_list_length(self.name, item, inner_c.minLength, inner_c.maxLength) + for it in item: + _check_int_item(self.name, it, inner_item_c) + + +class JobRangeExprParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): + """RANGE_EXPR job parameter (EXPR extension, RFC 0007). + + The value is an integer range expression string (e.g. ``"1-100:10"``) + validated against the existing ``IntRangeExpr`` grammar. + """ + + name: Identifier + type: Literal[JobParameterType.RANGE_EXPR] + _job_creation_metadata = _LIST_RANGE_JOB_CREATION_METADATA + userInterface: Optional[_ListParameterUserInterface] = None + description: Optional[Description] = None + minLength: Optional[StrictInt] = None # noqa: N815 + maxLength: Optional[StrictInt] = None # noqa: N815 + default: Optional[ParameterStringValue] = None + + _template_variable_definitions = _LIST_PARAM_VARS + _template_variable_sources = {"__export__": {"__self__"}} + + _validate_type_gate = field_validator("type")( + classmethod(lambda cls, v, info: _expr_param_gate(v, info)) + ) + + @field_validator("default") + @classmethod + def _validate_default(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return value + # Raises: ExpressionError / TokenError on a malformed range expression. + IntRangeExpr.from_str(value) + return value + + # override (JobParameterInterface) — validate a user-supplied range + # expression string at create time against the IntRangeExpr grammar. + def _check_constraints(self, value: Any) -> None: + if value is None: + raise ValueError(f"No value given for {self.name}.") + if not isinstance(value, str): + raise ValueError( + f"Parameter {self.name}: RANGE_EXPR value must be a string, " + f"got {type(value).__name__}." + ) + try: + # Raises: ExpressionError / TokenError on a malformed range expression. + IntRangeExpr.from_str(value) + except (ExpressionError, TokenError) as exc: + raise ValueError(f"Value ({value}) for parameter {self.name}: {exc}") + + JobParameterDefinitionList = Annotated[ list[ Annotated[ @@ -2859,6 +3664,14 @@ def resolve_syntax_sugar(self) -> "StepTemplate": JobFloatParameterDefinition, JobStringParameterDefinition, JobPathParameterDefinition, + JobBoolParameterDefinition, + JobListStringParameterDefinition, + JobListPathParameterDefinition, + JobListIntParameterDefinition, + JobListFloatParameterDefinition, + JobListBoolParameterDefinition, + JobListListIntParameterDefinition, + JobRangeExprParameterDefinition, ], Field(..., discriminator="type"), ] @@ -2991,6 +3804,11 @@ def _permitted_extension_names( def _unique_step_names(cls, v: StepTemplateList) -> StepTemplateList: return validate_unique_elements(v, item_value=lambda v: v.name, property="name") + @field_validator("parameterDefinitions", mode="before") + @classmethod + def _normalize_parameter_type_case(cls, v: Any, info: ValidationInfo) -> Any: + return _normalize_parameter_type_case(v, info) + @field_validator("parameterDefinitions") @classmethod def _validate_parameter_definitions( @@ -3119,6 +3937,61 @@ def _validate_env_names_dont_match_step_env_names(self) -> Self: return self + @model_validator(mode="after") + def _validate_single_wrap_layer(self, info: ValidationInfo) -> Self: + # RFC 0008 single-wrap-layer rule: at most one environment in any + # session's stack may define wrap hooks. A session's stack is the + # job's jobEnvironments plus exactly one step's stepEnvironments, so + # "one wrap layer per session" reduces to: for every step, + # (wrap envs in jobEnvironments) + (wrap envs in that step's + # stepEnvironments) must be <= 1. Mirrors openjd-rs + # validate_v2023_09/wrap_actions.rs. + context = cast(Optional[ModelParsingContext], info.context) if info else None + extensions = context.extensions if context else set() + if "WRAP_ACTIONS" not in extensions: + return self + + job_envs = self.jobEnvironments or [] + job_env_wrap_count = sum(1 for env in job_envs if _env_defines_wrap_hook(env)) + + errors = list[InitErrorDetails]() + + # Multiple wrap layers in jobEnvironments are reachable from every + # session, independent of any step's stepEnvironments. + if job_env_wrap_count > 1: + errors.append( + InitErrorDetails( + type="value_error", + loc=("jobEnvironments",), + ctx={"error": ValueError(_SINGLE_WRAP_LAYER_MSG)}, + input=self.jobEnvironments, + ) + ) + + # Each step's session is jobEnvironments + that step's + # stepEnvironments. Only steps that declare stepEnvironments are + # checked (a step with none adds no wrap layer of its own). + for i, step in enumerate(self.steps or []): + if step.stepEnvironments is None: + continue + step_env_wrap_count = sum( + 1 for env in step.stepEnvironments if _env_defines_wrap_hook(env) + ) + if job_env_wrap_count + step_env_wrap_count > 1: + errors.append( + InitErrorDetails( + type="value_error", + loc=("steps", i, "stepEnvironments"), + ctx={"error": ValueError(_SINGLE_WRAP_LAYER_MSG)}, + input=step.stepEnvironments, + ) + ) + + if errors: + raise ValidationError.from_exception_data(self.__class__.__name__, errors) + + return self + class EnvironmentTemplate(OpenJDModel_v2023_09): """Definition of an Open Job Description Environment Template. @@ -3179,6 +4052,11 @@ def _permitted_extension_names( context.extensions = set() return value + @field_validator("parameterDefinitions", mode="before") + @classmethod + def _normalize_parameter_type_case(cls, v: Any, info: ValidationInfo) -> Any: + return _normalize_parameter_type_case(v, info) + @field_validator("parameterDefinitions") @classmethod def _validate_parameter_definitions( diff --git a/test/openjd/model_v0/format_strings/test_expr_node.py b/test/openjd/model_v0/format_strings/test_expr_node.py new file mode 100644 index 00000000..908122b6 --- /dev/null +++ b/test/openjd/model_v0/format_strings/test_expr_node.py @@ -0,0 +1,310 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the EXPR-extension format-string seam (Rust-backed ExprNode). + +These cover the v0 model delegating {{ }} expression parsing/evaluation to the +Rust openjd-expr engine when the EXPR extension is declared, and preserving the +legacy Name.Dot.Name behaviour when it is not. +""" + +import pytest + +from openjd.expr import PathFormat +from openjd.model import ExpressionError +from openjd.model._format_strings._format_string import FormatString +from openjd.model._format_strings._nodes import ExprNode, FullNameNode +from openjd.model._format_strings._parser import parse_format_string_expr +from openjd.model._symbol_table import SymbolTable +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 + + +def _ctx(extensions): + """A parsing context whose *declared* extension set is `extensions`. + + Mirrors the post-`extensions`-field state where context.extensions holds + the template's declared extensions. + """ + ctx = ModelParsingContext_v2023_09(supported_extensions=extensions) + ctx.extensions = set(extensions) + return ctx + + +class TestExprGating: + def test_expr_declared_yields_expr_node(self): + node = parse_format_string_expr("Param.X + 3", context=_ctx(["EXPR"])) + assert isinstance(node, ExprNode) + + def test_expr_not_declared_yields_fullname_node(self): + node = parse_format_string_expr("Param.X", context=_ctx([])) + assert isinstance(node, FullNameNode) + + def test_expr_grammar_rejected_without_extension(self): + # '+' is not part of the legacy Name.Dot.Name grammar. + with pytest.raises(ExpressionError): + parse_format_string_expr("Param.X + 3", context=_ctx([])) + + def test_plain_name_still_parses_with_expr(self): + node = parse_format_string_expr("Param.X", context=_ctx(["EXPR"])) + assert isinstance(node, ExprNode) + + +class TestExprEvaluate: + @pytest.mark.parametrize( + "expr,expected", + [ + ("Param.X + 3", 13), + ("Param.X - 3", 7), + ("Param.X * 2", 20), + ("Param.X // 3", 3), + ("2 ** 3", 8), + ("-Param.X", -10), + ("(Param.X + 1) * 2", 22), + ], + ) + def test_arithmetic(self, expr, expected): + node = parse_format_string_expr(expr, context=_ctx(["EXPR"])) + st = SymbolTable() + st["Param.X"] = 10 + assert node.evaluate(symtab=st) == expected + + def test_function_library_available(self): + # repr_sh is part of the standard EXPR function library and must be + # reachable through the default profile. + node = parse_format_string_expr("repr_sh('a b')", context=_ctx(["EXPR"])) + assert node.evaluate(symtab=SymbolTable()) == "'a b'" + + def test_list_result_allowed(self): + node = parse_format_string_expr("[1, 2, 3]", context=_ctx(["EXPR"])) + assert node.evaluate(symtab=SymbolTable()) == [1, 2, 3] + + def test_typed_string_symbol_coerced_via_types(self): + # When a types map is supplied, a string-valued symbol is coerced to + # the declared EXPR type (INT) so arithmetic works. + node = parse_format_string_expr("Param.X + 1", context=_ctx(["EXPR"])) + st = SymbolTable() + st["Param.X"] = "10" # stringly-typed, as create_job stores it + st.expr_types = {"Param.X": "INT"} + assert node.evaluate(symtab=st) == 11 + + def test_native_list_symbol_inferred(self): + # A native list value is inferred as a typed list by the Rust + # build_symbol_table bridge, so aggregate functions work without an + # explicit type entry. + node = parse_format_string_expr("sum(Param.Items)", context=_ctx(["EXPR"])) + st = SymbolTable() + st["Param.Items"] = [1, 2, 3] + assert node.evaluate(symtab=st) == 6 + + +class TestExprSymbolRefValidation: + def test_missing_symbol_raises(self): + node = parse_format_string_expr("Param.X + 1", context=_ctx(["EXPR"])) + with pytest.raises(ValueError): + node.validate_symbol_refs(symbols={"Param.Y"}) + + def test_present_symbol_ok(self): + node = parse_format_string_expr("Param.X + 1", context=_ctx(["EXPR"])) + node.validate_symbol_refs(symbols={"Param.X"}) + + def test_let_local_not_treated_as_free_symbol(self): + # Comprehension-local names must not be reported as missing free + # symbols; only the free reference (Param.Items) is validated. + node = parse_format_string_expr("[x * 2 for x in Param.Items]", context=_ctx(["EXPR"])) + node.validate_symbol_refs(symbols={"Param.Items"}) + + +class TestExprRangeExprTypedValidation: + """RANGE_EXPR job-parameter symbols now type-check (the OpenJD-type → EXPR + mapping resolves RANGE_EXPR to the engine's ``range_expr`` type instead of + the former ``None`` that fell back to name-only validation). Mirrors the + openjd-rs range_expr behaviour. + """ + + def _validate(self, expr): + node = parse_format_string_expr(expr, context=_ctx(["EXPR"])) + node.validate_symbol_refs( + symbols={"Param.Frames"}, symbol_types={"Param.Frames": "range_expr"} + ) + + @pytest.mark.parametrize( + "expr", + [ + "Param.Frames[0]", # subscript -> int + "len(Param.Frames)", # length of the range + "list(Param.Frames)", # convert to list[int] + ], + ) + def test_valid_range_expr_ops_typecheck(self, expr): + self._validate(expr) # does not raise + + def test_type_mismatch_rejected(self): + # Arithmetic with a range_expr is a genuine type error, now caught at + # validation time rather than slipping through name-only validation. + with pytest.raises(ExpressionError, match=r"range_expr"): + self._validate("Param.Frames + 1") + + def test_invalid_method_rejected(self): + with pytest.raises(ExpressionError, match=r"not available for range_expr"): + self._validate("Param.Frames.upper()") + + +class TestExprErrors: + def test_parse_error_is_model_expression_error(self): + with pytest.raises(ExpressionError): + parse_format_string_expr("Param.X +", context=_ctx(["EXPR"])) + + +class TestExprPaths: + """PATH-typed expression coverage mirroring the openjd-rs path tests + (``crates/openjd-expr/tests/integration/test_paths.rs`` and + ``test_rfc_examples.rs``): path construction, the path properties/methods + (``.name``/``.stem``/``.suffix``/``.parent``/``.with_suffix``), POSIX vs + Windows ``path_format`` behavior, and PATH-typed symbol coercion. PR #285 + review C4. + """ + + def _eval(self, expr, *, path_format=PathFormat.POSIX, symtab=None): + node = parse_format_string_expr(expr, context=_ctx(["EXPR"])) + return node.evaluate(symtab=symtab or SymbolTable(), path_format=path_format) + + def test_path_constructor(self): + assert self._eval("path('/tmp/file.txt')") == "/tmp/file.txt" + + def test_path_from_parts_list(self): + # path(list[string]) reconstructs a path from its parts. + assert self._eval("path(['/', 'a', 'b', 'c'])") == "/a/b/c" + + def test_name(self): + assert self._eval("path('/a/b/render.exr').name") == "render.exr" + + def test_stem(self): + assert self._eval("path('/a/b/render.exr').stem") == "render" + + def test_suffix(self): + assert self._eval("path('/a/b/render.exr').suffix") == ".exr" + + def test_parent(self): + assert self._eval("path('/a/b/c').parent") == "/a/b" + + @pytest.mark.parametrize( + "expr,expected", + [ + ( + "path('/projects/shot01/render.exr').with_suffix('.png')", + "/projects/shot01/render.png", + ), + ("path('/projects/shot01/render.exr').with_suffix('')", "/projects/shot01/render"), + ( + "path('/projects/shot01/render.exr').with_name('output.png')", + "/projects/shot01/output.png", + ), + ( + "path('/projects/shot01/render.exr').with_stem('final')", + "/projects/shot01/final.exr", + ), + ], + ) + def test_with_suffix_name_stem(self, expr, expected): + assert self._eval(expr) == expected + + def test_windows_as_posix(self): + # Convert a Windows path to POSIX separators for shell scripts. + assert ( + self._eval(r"path('C:\\renders\\project').as_posix()", path_format=PathFormat.WINDOWS) + == "C:/renders/project" + ) + + def test_join_operator_and_suffix(self): + # RFC example: (OutputDir / InputFile.name).with_suffix('.png') + assert ( + self._eval( + "(path('/output') / path('/in/scene.exr').name).with_suffix('.png')", + ) + == "/output/scene.png" + ) + + def test_path_typed_symbol_method_access(self): + # A PATH-typed job-parameter symbol (stored as a string by create_job) + # is coerced to a path via the types map, so method/property access + # works just like the Rust engine. + st = SymbolTable() + st["Param.File"] = "/a/b/render.exr" + st.expr_types = {"Param.File": "PATH"} + node = parse_format_string_expr("Param.File.name", context=_ctx(["EXPR"])) + assert node.evaluate(symtab=st, path_format=PathFormat.POSIX) == "render.exr" + + +class TestExprStringCoercion: + """An EXPR result substituted into a format string must use the engine's + spec-defined string coercion (RFC 0005), not Python's ``str()`` of the + native value. Booleans render lowercase, null renders as the empty string, + list items are double-quoted, and Decimal trailing zeros are preserved. + Guards against the ``str(value.item())`` regression that emitted Python + reprs (``True``/``None``/``['a', 'b']``). + """ + + @pytest.mark.parametrize( + "expr,expected", + [ + ("true", "true"), + ("false", "false"), + ("[1, 2, 3]", "[1, 2, 3]"), + ('["a", "b"]', '["a", "b"]'), + ("[true, false]", "[true, false]"), + ("1.0 + 2.0", "3.0"), + ], + ) + def test_evaluate_to_str_matches_spec(self, expr, expected): + node = parse_format_string_expr(expr, context=_ctx(["EXPR"])) + assert node.evaluate_to_str(symtab=SymbolTable()) == expected + + def test_null_evaluates_to_empty_string(self): + node = parse_format_string_expr("null", context=_ctx(["EXPR"])) + assert node.evaluate_to_str(symtab=SymbolTable()) == "" + + def test_resolve_uses_spec_coercion(self): + # End-to-end through FormatString.resolve. + fs = FormatString("a={{ true }} b={{ null }} c={{ [1, 2] }}", context=_ctx(["EXPR"])) + assert fs.resolve(symtab=SymbolTable()) == "a=true b= c=[1, 2]" + + def test_evaluate_returns_native_value(self): + # evaluate() (as opposed to evaluate_to_str) still returns the native + # Python value, preserving the existing contract for callers that need + # the typed result rather than its string form. + node = parse_format_string_expr("true", context=_ctx(["EXPR"])) + assert node.evaluate(symtab=SymbolTable()) is True + + +class TestSymbolTableExprTypesPreserved: + """``SymbolTable.expr_types`` is a first-class field, so it survives copies + and unions — type-aware EXPR coercion must not silently revert to inference + when a derived symbol table is used.""" + + def test_copy_preserves_expr_types(self): + st = SymbolTable() + st["Param.X"] = "10" + st.expr_types = {"Param.X": "INT"} + copied = SymbolTable(source=st) + assert copied.expr_types == {"Param.X": "INT"} + + def test_union_preserves_expr_types(self): + st = SymbolTable() + st["Param.X"] = "10" + st.expr_types = {"Param.X": "INT"} + other = SymbolTable() + other["Param.Y"] = "hi" + other.expr_types = {"Param.Y": "STRING"} + merged = st.union(other) + assert merged.expr_types == {"Param.X": "INT", "Param.Y": "STRING"} + + def test_union_coercion_still_works_on_derived_table(self): + # The typed coercion must apply when evaluating against a unioned table. + st = SymbolTable() + st["Param.X"] = "10" + st.expr_types = {"Param.X": "INT"} + derived = st.union(SymbolTable()) + node = parse_format_string_expr("Param.X + 1", context=_ctx(["EXPR"])) + assert node.evaluate(symtab=derived) == 11 + + def test_default_expr_types_is_empty(self): + assert SymbolTable().expr_types == {} diff --git a/test/openjd/model_v0/format_strings/test_lazy_rust_import.py b/test/openjd/model_v0/format_strings/test_lazy_rust_import.py new file mode 100644 index 00000000..beb6f50d --- /dev/null +++ b/test/openjd/model_v0/format_strings/test_lazy_rust_import.py @@ -0,0 +1,35 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression test: importing the v0 format-string parser must NOT eagerly load +the Rust expr surface. + +The EXPR engine bindings (``openjd._openjd_rs`` / ``openjd.expr``) are loaded +lazily, only when an EXPR template is actually parsed (via ``ExprNode``). The +``EXPR_EXTENSION`` gate constant lives in ``_parser`` precisely so that merely +importing the parser does not pull in the compiled extension. This guards +against re-introducing a top-level ``from ._expr_support import ...`` in the +parser, which would force the Rust import on every non-EXPR parse path. + +A subprocess with a fresh interpreter is used because the in-process test runner +has already imported these modules. +""" + +import subprocess +import sys + + +def test_importing_parser_does_not_load_rust_expr_surface(): + code = ( + "import sys\n" + "import openjd.model._format_strings._parser\n" + "loaded = [m for m in ('openjd._openjd_rs', 'openjd.expr') if m in sys.modules]\n" + "print(','.join(loaded))\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=True, + ) + leaked = result.stdout.strip() + assert leaked == "", f"Rust expr modules eagerly imported by the parser: {leaked}" diff --git a/test/openjd/model_v0/test_create_job.py b/test/openjd/model_v0/test_create_job.py index 57b17036..b7a63936 100644 --- a/test/openjd/model_v0/test_create_job.py +++ b/test/openjd/model_v0/test_create_job.py @@ -55,6 +55,7 @@ def fake_template_dir_and_cwd(): [ pytest.param(param_type.value, id=f"{param_type.value} type") for param_type in JobParameterType_2023_09 + if param_type.value in ("STRING", "PATH", "INT", "FLOAT") ], ) def test_preprocess_job_parameters_handles_parameter_type(self, param_type: str) -> None: @@ -94,6 +95,7 @@ def test_preprocess_job_parameters_handles_parameter_type(self, param_type: str) [ pytest.param(param_type.value, id=f"{param_type.value} type") for param_type in JobParameterType_2023_09 + if param_type.value in ("STRING", "PATH", "INT", "FLOAT") ], ) def test_handles_parameter_type_without_path_escape_validation(self, param_type: str) -> None: diff --git a/test/openjd/model_v0/v2023_09/test_bool_parameter.py b/test/openjd/model_v0/v2023_09/test_bool_parameter.py new file mode 100644 index 00000000..78dd2328 --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_bool_parameter.py @@ -0,0 +1,80 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the EXPR-extension BOOL job parameter type (RFC 0007).""" + +import pytest + +from openjd.model import DecodeValidationError, decode_job_template + +_MINIMAL_STEP = { + "name": "S", + "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}, +} + + +def _template(param_def, *, extensions=("EXPR",)): + tmpl = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [param_def], + "steps": [_MINIMAL_STEP], + } + if extensions: + tmpl["extensions"] = list(extensions) + return tmpl + + +def _decode(tmpl): + return decode_job_template(template=tmpl, supported_extensions=["EXPR"]) + + +class TestBoolParameterValid: + @pytest.mark.parametrize( + "default", + [ + True, + False, + 1, + 0, + 1.0, + 0.0, + "true", + "false", + "TRUE", + "False", + "yes", + "no", + "on", + "off", + "1", + "0", + ], + ) + def test_accepts_boolish_defaults(self, default): + _decode(_template({"name": "Flag", "type": "BOOL", "default": default})) + + def test_no_default_ok(self): + _decode(_template({"name": "Flag", "type": "BOOL"})) + + def test_user_interface_checkbox(self): + _decode( + _template( + { + "name": "Flag", + "type": "BOOL", + "default": True, + "userInterface": {"control": "CHECK_BOX", "label": "Enable"}, + } + ) + ) + + +class TestBoolParameterInvalid: + def test_requires_expr_extension(self): + with pytest.raises(DecodeValidationError, match="requires the EXPR extension"): + _decode(_template({"name": "Flag", "type": "BOOL", "default": True}, extensions=())) + + @pytest.mark.parametrize("default", [2, 0.5, "maybe", -1, 42]) + def test_rejects_non_bool_defaults(self, default): + with pytest.raises(DecodeValidationError): + _decode(_template({"name": "Flag", "type": "BOOL", "default": default})) diff --git a/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py new file mode 100644 index 00000000..b9bf5aea --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_create_job_expr_params.py @@ -0,0 +1,97 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for instantiating a Job from a template that declares EXPR-extension +job parameter types (RFC 0007): BOOL, RANGE_EXPR, and the LIST[*] variants. + +These exercise the pure-Python v0 ``create_job`` path, which previously could +only handle the original scalar types. The new types carry their values +natively (lists/bools) into the instantiated ``JobParameter`` so the typed EXPR +symbol table can coerce them. +""" + +import pytest + +from openjd.model import ( + DecodeValidationError, + create_job, + decode_job_template, + model_to_object, +) + + +def _template(param_def, *, on_run_args=("hi",)): + return { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "extensions": ["EXPR"], + "parameterDefinitions": [param_def], + "steps": [ + { + "name": "S", + "script": {"actions": {"onRun": {"command": "echo", "args": list(on_run_args)}}}, + } + ], + } + + +def _create(param_def): + jt = decode_job_template(template=_template(param_def), supported_extensions=["EXPR"]) + return create_job(job_template=jt, job_parameter_values={}) + + +def _stored_value(job, name): + obj = model_to_object(model=job) + return obj["parameters"][name]["value"] + + +class TestCreateJobExprParams: + @pytest.mark.parametrize( + "param_def,expected", + [ + ({"name": "Nums", "type": "LIST[INT]", "default": [1, 2, 3]}, [1, 2, 3]), + ({"name": "Flag", "type": "BOOL", "default": True}, True), + ({"name": "Off", "type": "BOOL", "default": "no"}, False), + ({"name": "Tags", "type": "LIST[STRING]", "default": ["a", "b"]}, ["a", "b"]), + ({"name": "Fs", "type": "LIST[FLOAT]", "default": [1.5, 2.5]}, [1.5, 2.5]), + ({"name": "Bs", "type": "LIST[BOOL]", "default": [True, False]}, [True, False]), + ({"name": "Ps", "type": "LIST[PATH]", "default": ["/a", "/b"]}, ["/a", "/b"]), + ({"name": "M", "type": "LIST[LIST[INT]]", "default": [[1, 2], [3]]}, [[1, 2], [3]]), + ({"name": "R", "type": "RANGE_EXPR", "default": "1-3"}, "1-3"), + ], + ) + def test_create_job_stores_native_value(self, param_def, expected): + job = _create(param_def) + assert _stored_value(job, param_def["name"]) == expected + + def test_create_job_case_insensitive_type(self): + job = _create({"name": "Nums", "type": "list[int]", "default": [1, 2]}) + assert _stored_value(job, "Nums") == [1, 2] + + def test_create_job_scalar_types_unchanged(self): + # The original scalar types still round-trip as strings. + job = _create({"name": "N", "type": "INT", "default": 7}) + assert _stored_value(job, "N") == "7" + + +class TestRangeExprTypedValidation: + """A RANGE_EXPR parameter now carries a typed (``range_expr``) EXPR symbol, + so expressions referencing it are type-validated at decode time rather than + only name-checked. Mirrors openjd-rs. + """ + + def _decode_with_expr(self, expr): + tmpl = _template( + {"name": "Frames", "type": "RANGE_EXPR", "default": "1-10"}, + on_run_args=["{{ " + expr + " }}"], + ) + return decode_job_template(template=tmpl, supported_extensions=["EXPR"]) + + def test_valid_range_expr_use_accepted(self): + # Subscripting a RANGE_EXPR parameter is well-typed and must decode. + self._decode_with_expr("Param.Frames[0]") + + def test_type_mismatch_rejected_at_decode(self): + # Arithmetic on a RANGE_EXPR is a type error; type-aware validation now + # catches it (it previously slipped through name-only validation). + with pytest.raises(DecodeValidationError, match=r"range_expr"): + self._decode_with_expr("Param.Frames + 1") diff --git a/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py b/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py new file mode 100644 index 00000000..8e97768c --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_expr_param_constraints.py @@ -0,0 +1,171 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for create-time and merge-time constraint validation of the +EXPR-extension (RFC 0007) job-parameter types — LIST[*] and RANGE_EXPR. + +These guard against three previously-found gaps: + +* ``preprocess_job_parameters`` did not enforce ``item.*`` / length / range + constraints on a *user-supplied* value for these types (only the template + ``default`` was validated at decode time). +* ``merge_job_parameter_definitions_for_one`` applied a merged default via + ``model_copy``, which skips validators, so a default carried over from one + source that violated another source's constraints went unchecked. +""" + +from pathlib import Path + +import pytest + +from openjd.model import ( + decode_environment_template, + decode_job_template, + preprocess_job_parameters, +) + + +def _template(param_def): + return { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "extensions": ["EXPR"], + "parameterDefinitions": [param_def], + "steps": [{"name": "S", "script": {"actions": {"onRun": {"command": "echo"}}}}], + } + + +def _env_template(param_def): + return { + "specificationVersion": "environment-2023-09", + "extensions": ["EXPR"], + "parameterDefinitions": [param_def], + "environment": {"name": "E", "script": {"actions": {"onEnter": {"command": "echo"}}}}, + } + + +def _preprocess(param_def, values, *, environment_templates=None): + jt = decode_job_template(template=_template(param_def), supported_extensions=["EXPR"]) + return preprocess_job_parameters( + job_template=jt, + job_parameter_values=values, + job_template_dir=Path(), + current_working_dir=Path(), + allow_job_template_dir_walk_up=True, + environment_templates=environment_templates, + ) + + +class TestListParameterValueConstraints: + """A user-supplied value for a LIST[*] parameter is checked against the + definition's length and per-item constraints at preprocess/create time.""" + + def test_item_maxvalue_violation_rejected(self): + with pytest.raises(ValueError, match=r"item 999 is above item\.maxValue 10"): + _preprocess( + {"name": "Nums", "type": "LIST[INT]", "item": {"maxValue": 10}}, + {"Nums": [999]}, + ) + + def test_item_allowedvalues_violation_rejected(self): + with pytest.raises(ValueError, match=r"not in item\.allowedValues"): + _preprocess( + {"name": "Tags", "type": "LIST[STRING]", "item": {"allowedValues": ["a", "b"]}}, + {"Tags": ["z"]}, + ) + + def test_maxlength_violation_rejected(self): + with pytest.raises(ValueError, match=r"list length 3 exceeds maxLength 2"): + _preprocess( + {"name": "Nums", "type": "LIST[INT]", "maxLength": 2}, + {"Nums": [1, 2, 3]}, + ) + + def test_wrong_item_type_rejected(self): + with pytest.raises(ValueError, match=r"list items must be integers"): + _preprocess({"name": "Nums", "type": "LIST[INT]"}, {"Nums": ["not-an-int"]}) + + def test_non_list_value_rejected(self): + with pytest.raises(ValueError, match=r"value must be a list"): + _preprocess({"name": "Nums", "type": "LIST[INT]"}, {"Nums": "5"}) + + def test_list_list_int_inner_constraint_enforced(self): + with pytest.raises(ValueError, match=r"item 99 is above item\.maxValue 10"): + _preprocess( + { + "name": "M", + "type": "LIST[LIST[INT]]", + "item": {"item": {"maxValue": 10}}, + }, + {"M": [[1, 2], [99]]}, + ) + + def test_valid_value_accepted(self): + result = _preprocess( + {"name": "Nums", "type": "LIST[INT]", "item": {"maxValue": 10}}, + {"Nums": [5, 6]}, + ) + assert result["Nums"].value == [5, 6] + + +class TestRangeExprValueConstraints: + """A user-supplied RANGE_EXPR value is validated against the IntRangeExpr + grammar at preprocess/create time.""" + + def test_malformed_range_rejected(self): + with pytest.raises(ValueError, match=r"parameter R"): + _preprocess({"name": "R", "type": "RANGE_EXPR"}, {"R": "1-abc"}) + + def test_non_string_range_rejected(self): + with pytest.raises(ValueError, match=r"must be a string"): + _preprocess({"name": "R", "type": "RANGE_EXPR"}, {"R": [1, 2, 3]}) + + def test_valid_range_accepted(self): + result = _preprocess({"name": "R", "type": "RANGE_EXPR"}, {"R": "1-100:10"}) + assert result["R"].value == "1-100:10" + + +class TestMergedDefaultRevalidation: + """When the same EXPR parameter is defined in more than one template, the + merged (last-defined) default is re-validated against the surviving (job + template) definition's constraints — model_copy alone skips validators.""" + + def test_merged_default_violating_maxlength_rejected(self): + # The environment template supplies default [1,2,3,4,5]; the job template + # constrains the same parameter to maxLength 2 and supplies no default. + # The carried-over default must be rejected. + env = decode_environment_template( + template=_env_template( + {"name": "Nums", "type": "LIST[INT]", "default": [1, 2, 3, 4, 5]} + ), + supported_extensions=["EXPR"], + ) + with pytest.raises(ValueError, match=r"list length 5 exceeds maxLength 2"): + _preprocess( + {"name": "Nums", "type": "LIST[INT]", "maxLength": 2}, + {}, + environment_templates=[env], + ) + + def test_merged_default_violating_item_constraint_rejected(self): + env = decode_environment_template( + template=_env_template({"name": "Nums", "type": "LIST[INT]", "default": [99]}), + supported_extensions=["EXPR"], + ) + with pytest.raises(ValueError, match=r"item 99 is above item\.maxValue 10"): + _preprocess( + {"name": "Nums", "type": "LIST[INT]", "item": {"maxValue": 10}}, + {}, + environment_templates=[env], + ) + + def test_compatible_merged_default_accepted(self): + env = decode_environment_template( + template=_env_template({"name": "Nums", "type": "LIST[INT]", "default": [1, 2]}), + supported_extensions=["EXPR"], + ) + result = _preprocess( + {"name": "Nums", "type": "LIST[INT]", "maxLength": 5}, + {}, + environment_templates=[env], + ) + assert result["Nums"].value == [1, 2] diff --git a/test/openjd/model_v0/v2023_09/test_let_bindings.py b/test/openjd/model_v0/v2023_09/test_let_bindings.py new file mode 100644 index 00000000..ac0ddc6a --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_let_bindings.py @@ -0,0 +1,179 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for EXPR `let` bindings (RFC 0007 §3.6), Step.Name in step +environments, host-context function scoping, and type-aware expression +validation.""" + +import pytest + +from openjd.model import DecodeValidationError, decode_job_template + +_EXTS = ["EXPR", "FEATURE_BUNDLE_1"] + + +def _decode(template): + return decode_job_template(template=template, supported_extensions=_EXTS) + + +def _job(steps, *, params=None, extensions=("EXPR",)): + t = {"specificationVersion": "jobtemplate-2023-09", "name": "T", "steps": steps} + if params is not None: + t["parameterDefinitions"] = params + if extensions: + t["extensions"] = list(extensions) + return t + + +def _onrun(arg): + return {"actions": {"onRun": {"command": "echo", "args": [arg]}}} + + +class TestLetValid: + def test_step_let_visible_in_script(self): + _decode( + _job( + [{"name": "S", "let": ["x = 1", "y = x + 1"], "script": _onrun("{{y}}")}], + ) + ) + + def test_script_let(self): + _decode( + _job([{"name": "S", "script": {"let": ["a = 2"], **_onrun("{{a}}")}}]), + ) + + def test_chained_and_functions(self): + _decode( + _job( + [ + { + "name": "S", + "let": ["total = sum(Param.Items)", "n = len(Param.Items)"], + "script": _onrun("{{total}}-{{n}}"), + } + ], + params=[{"name": "Items", "type": "LIST[INT]", "default": [1, 2, 3]}], + ) + ) + + +class TestLetInvalid: + def test_requires_expr(self): + with pytest.raises(DecodeValidationError, match="requires the EXPR extension"): + _decode(_job([{"name": "S", "let": ["x = 1"], "script": _onrun("hi")}], extensions=())) + + def test_empty(self): + with pytest.raises(DecodeValidationError, match="at least one"): + _decode(_job([{"name": "S", "let": [], "script": _onrun("hi")}])) + + def test_duplicate_name(self): + with pytest.raises(DecodeValidationError, match="unique"): + _decode(_job([{"name": "S", "let": ["x = 1", "x = 2"], "script": _onrun("hi")}])) + + def test_too_many(self): + many = [f"v{i} = {i}" for i in range(51)] + with pytest.raises(DecodeValidationError, match="at most"): + _decode(_job([{"name": "S", "let": many, "script": _onrun("hi")}])) + + def test_uppercase_name(self): + with pytest.raises(DecodeValidationError, match="identifier"): + _decode(_job([{"name": "S", "let": ["Foo = 1"], "script": _onrun("hi")}])) + + def test_shadow_enclosing(self): + with pytest.raises(DecodeValidationError, match="shadows"): + _decode( + _job( + [ + { + "name": "S", + "let": ["x = 1"], + "script": {"let": ["x = 2"], **_onrun("hi")}, + } + ] + ) + ) + + def test_self_reference(self): + # A binding cannot reference its own name on its RHS (not yet bound at + # its definition point). Mirrors openjd-rs "references itself". + with pytest.raises(DecodeValidationError, match="cannot reference itself"): + _decode(_job([{"name": "S", "let": ["x = x + 1"], "script": _onrun("hi")}])) + + def test_comprehension_shadows_let(self): + with pytest.raises(DecodeValidationError, match="shadows"): + _decode( + _job( + [ + { + "name": "S", + "let": ["x = 10"], + "script": _onrun("{{ [x for x in Param.Items] }}"), + } + ], + params=[{"name": "Items", "type": "LIST[INT]", "default": [1, 2]}], + ) + ) + + +class TestStepName: + def test_step_name_in_step_environment_with_expr(self): + _decode( + _job( + [ + { + "name": "Render", + "stepEnvironments": [ + {"name": "Setup", "variables": {"CUR": "{{ Step.Name }}"}} + ], + "script": _onrun("hi"), + } + ] + ) + ) + + def test_step_name_requires_expr(self): + with pytest.raises(DecodeValidationError): + _decode( + _job( + [ + { + "name": "Render", + "stepEnvironments": [ + {"name": "Setup", "variables": {"CUR": "{{ Step.Name }}"}} + ], + "script": _onrun("hi"), + } + ], + extensions=(), + ) + ) + + +class TestHostContextScope: + def test_apply_path_mapping_in_job_name_rejected(self): + with pytest.raises(DecodeValidationError, match="only available at runtime"): + _decode( + _job([{"name": "S", "script": _onrun("hi")}]) + | {"name": "{{ apply_path_mapping('/x') }}"} + ) + + def test_apply_path_mapping_in_task_arg_ok(self): + _decode(_job([{"name": "S", "script": _onrun("{{ apply_path_mapping('/x') }}")}])) + + +class TestSymbolTyping: + def test_path_method_access(self): + _decode( + _job( + [{"name": "S", "script": _onrun("{{ Param.File.name }}")}], + params=[{"name": "File", "type": "PATH", "default": "/a/b.exr"}], + ) + ) + + def test_string_type_mismatch_rejected(self): + with pytest.raises(DecodeValidationError): + _decode( + _job( + [{"name": "S", "script": _onrun("{{ Param.Name + 1 }}")}], + params=[{"name": "Name", "type": "STRING", "default": "hi"}], + ) + ) diff --git a/test/openjd/model_v0/v2023_09/test_list_parameters.py b/test/openjd/model_v0/v2023_09/test_list_parameters.py new file mode 100644 index 00000000..89e66e67 --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_list_parameters.py @@ -0,0 +1,111 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the EXPR-extension LIST[*] and RANGE_EXPR job parameter types +(RFC 0007): structural parsing, list/item constraints, EXPR gating, and +case-insensitive type names.""" + +import pytest + +from openjd.model import DecodeValidationError, decode_job_template + +_STEP = {"name": "S", "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}} + + +def _tmpl(param_def, *, extensions=("EXPR",)): + t = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [param_def], + "steps": [_STEP], + } + if extensions: + t["extensions"] = list(extensions) + return t + + +def _decode(t): + return decode_job_template(template=t, supported_extensions=["EXPR"]) + + +class TestListValid: + @pytest.mark.parametrize( + "param", + [ + {"name": "S", "type": "LIST[STRING]", "default": ["a", "b"]}, + { + "name": "P", + "type": "LIST[PATH]", + "default": ["/a", "/b"], + "objectType": "FILE", + "dataFlow": "IN", + }, + {"name": "I", "type": "LIST[INT]", "default": [1, 2, 3]}, + {"name": "F", "type": "LIST[FLOAT]", "default": [1.0, 2.5]}, + {"name": "B", "type": "LIST[BOOL]", "default": [True, False]}, + {"name": "LL", "type": "LIST[LIST[INT]]", "default": [[1, 2], [3]]}, + {"name": "R", "type": "RANGE_EXPR", "default": "1-100:10"}, + ], + ) + def test_accepts(self, param): + _decode(_tmpl(param)) + + def test_item_constraints_ok(self): + _decode( + _tmpl( + { + "name": "S", + "type": "LIST[STRING]", + "default": ["alpha"], + "minLength": 1, + "maxLength": 5, + "item": {"allowedValues": ["alpha", "beta"], "minLength": 3}, + } + ) + ) + + def test_case_insensitive_type(self): + _decode(_tmpl({"name": "I", "type": "list[int]", "default": [1]})) + + +class TestListInvalid: + def test_requires_expr(self): + with pytest.raises(DecodeValidationError, match="requires the EXPR extension"): + _decode(_tmpl({"name": "I", "type": "LIST[INT]", "default": [1]}, extensions=())) + + @pytest.mark.parametrize( + "param", + [ + {"name": "S", "type": "LIST[STRING]", "default": "notalist"}, # scalar not list + {"name": "I", "type": "LIST[INT]", "default": ["not", "ints"]}, # wrong item type + { + "name": "I", + "type": "LIST[INT]", + "default": [-5], + "item": {"minValue": 0}, + }, # below min + { + "name": "S", + "type": "LIST[STRING]", + "default": [""], + "item": {"minLength": 1}, + }, # too short + { + "name": "S", + "type": "LIST[STRING]", + "default": ["a", "b", "c"], + "maxLength": 2, + }, # too long + {"name": "LL", "type": "LIST[LIST[INT]]", "default": [[1, "a"]]}, # string in inner + {"name": "LL", "type": "LIST[LIST[INT]]", "default": [[1], 3]}, # scalar in outer + { + "name": "LL", + "type": "LIST[LIST[INT]]", + "default": [[999]], + "item": {"item": {"maxValue": 100}}, + }, + {"name": "R", "type": "RANGE_EXPR", "default": "not-a-range"}, # bad range expr + ], + ) + def test_rejects(self, param): + with pytest.raises(DecodeValidationError): + _decode(_tmpl(param)) diff --git a/test/openjd/model_v0/v2023_09/test_task_range_expr.py b/test/openjd/model_v0/v2023_09/test_task_range_expr.py new file mode 100644 index 00000000..ac8820f2 --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_task_range_expr.py @@ -0,0 +1,53 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for EXPR task-parameter range expressions (RFC 0007): a STRING/FLOAT/ +PATH task parameter `range` given as a single `{{ ... }}` expression that +resolves to a list, mirroring the existing INT behaviour. Gated on EXPR.""" + +import pytest + +from openjd.model import DecodeValidationError, decode_job_template + + +def _tmpl(task_param, *, extensions=("EXPR",)): + t = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "Prefix", "type": "STRING", "default": "x"}], + "steps": [ + { + "name": "S", + "parameterSpace": {"taskParameterDefinitions": [task_param]}, + "script": {"actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.P}}"]}}}, + } + ], + } + if extensions: + t["extensions"] = list(extensions) + return t + + +def _decode(t): + return decode_job_template(template=t, supported_extensions=["EXPR"]) + + +class TestTaskRangeExpr: + @pytest.mark.parametrize("ptype", ["STRING", "FLOAT", "PATH"]) + def test_range_expression_accepted_with_expr(self, ptype): + _decode(_tmpl({"name": "P", "type": ptype, "range": "{{ [Param.Prefix] }}"})) + + @pytest.mark.parametrize("ptype", ["STRING", "FLOAT", "PATH"]) + def test_literal_list_range_still_accepted(self, ptype): + rng = ["1.0"] if ptype == "FLOAT" else ["a"] + _decode(_tmpl({"name": "P", "type": ptype, "range": rng})) + + def test_range_expression_requires_expr(self): + # Without EXPR the expression range is rejected (either by the EXPR + # gate or because the `[...]` grammar is invalid in the legacy parser). + with pytest.raises(DecodeValidationError): + _decode( + _tmpl( + {"name": "P", "type": "STRING", "range": "{{ [Param.Prefix] }}"}, + extensions=(), + ) + ) diff --git a/test/openjd/model_v0/v2023_09/test_template_variables.py b/test/openjd/model_v0/v2023_09/test_template_variables.py index 2e6f43ad..cec387ff 100644 --- a/test/openjd/model_v0/v2023_09/test_template_variables.py +++ b/test/openjd/model_v0/v2023_09/test_template_variables.py @@ -578,7 +578,7 @@ def test_job_template_variables_parse_success(data: dict[str, Any]) -> None: }, ], }, - 1, + 2, id="parameter space cannot ref path parameter", ), pytest.param( @@ -731,7 +731,7 @@ def test_job_template_variables_parse_success(data: dict[str, Any]) -> None: }, ], }, - 5, + 10, id="task params cannot see each other, envs, or the session", ), pytest.param( @@ -809,7 +809,7 @@ def test_job_template_variables_parse_success(data: dict[str, Any]) -> None: }, ], }, - 6, + 8, id="task params cannot be seen across steps", ), pytest.param( @@ -883,7 +883,7 @@ def test_job_template_variables_parse_success(data: dict[str, Any]) -> None: }, ], }, - 3, + 6, id="Cannot reference Session values in a Task Parameter", ), pytest.param( diff --git a/test/openjd/model_v0/v2023_09/test_wrap_actions.py b/test/openjd/model_v0/v2023_09/test_wrap_actions.py new file mode 100644 index 00000000..f45ea4e4 --- /dev/null +++ b/test/openjd/model_v0/v2023_09/test_wrap_actions.py @@ -0,0 +1,296 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for WRAP_ACTIONS (RFC 0008) model validation on EnvironmentActions. + +Covers the structural rules enforced at decode/`check` time: extension gating, +the EXPR hard prerequisite, all-or-nothing wrap hooks, and the at-least-one +action rule. Runtime interception and precise per-hook variable scoping live in +the sessions runtime and are out of scope here. +""" + +import pytest + +from openjd.model import ( + DecodeValidationError, + create_job, + decode_environment_template, + decode_job_template, +) + +_ALL_EXTS = ["TASK_CHUNKING", "REDACTED_ENV_VARS", "FEATURE_BUNDLE_1", "EXPR", "WRAP_ACTIONS"] + + +def _env_template(actions, *, extensions): + tmpl = { + "specificationVersion": "environment-2023-09", + "environment": {"name": "WrapEnv", "script": {"actions": actions}}, + } + if extensions: + tmpl["extensions"] = list(extensions) + return tmpl + + +def _cmd(arg): + return {"command": "echo", "args": [arg]} + + +_ALL_THREE = { + "onWrapEnvEnter": _cmd("wrap-enter {{WrappedEnv.Name}}"), + "onWrapTaskRun": _cmd("wrap-task {{WrappedAction.Command}}"), + "onWrapEnvExit": _cmd("wrap-exit {{WrappedEnv.Name}}"), +} + + +def _decode(template, extensions=_ALL_EXTS): + return decode_environment_template(template=template, supported_extensions=extensions) + + +class TestWrapActionsValid: + def test_all_three_hooks_ok(self): + tmpl = _env_template(dict(_ALL_THREE), extensions=["WRAP_ACTIONS", "EXPR"]) + _decode(tmpl) + + def test_wrap_with_onenter_onexit_ok(self): + actions = dict(_ALL_THREE) + actions["onEnter"] = _cmd("entered") + actions["onExit"] = _cmd("exited") + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_non_wrap_env_unchanged(self): + # Legacy behaviour: an env with only onEnter still validates and does + # not require WRAP_ACTIONS/EXPR. + _decode(_env_template({"onEnter": _cmd("hi")}, extensions=[])) + + +class TestWrapActionsInvalid: + def test_all_or_nothing_missing_one(self): + actions = dict(_ALL_THREE) + del actions["onWrapEnvExit"] + with pytest.raises(DecodeValidationError, match="all of onWrapEnvEnter"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_only_one_wrap_hook(self): + actions = {"onWrapTaskRun": _cmd("only {{WrappedAction.Command}}")} + with pytest.raises(DecodeValidationError, match="all of onWrapEnvEnter"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrap_requires_extension(self): + with pytest.raises(DecodeValidationError, match="require the WRAP_ACTIONS extension"): + _decode(_env_template(dict(_ALL_THREE), extensions=["EXPR"])) + + def test_wrap_requires_expr(self): + with pytest.raises(DecodeValidationError, match="requires the EXPR extension"): + _decode(_env_template(dict(_ALL_THREE), extensions=["WRAP_ACTIONS"])) + + def test_empty_actions_rejected(self): + with pytest.raises(DecodeValidationError): + _decode(_env_template({}, extensions=["WRAP_ACTIONS", "EXPR"])) + + +# ── RFC 0008 single-wrap-layer rule (JobTemplate) ────────────────────── +# +# A session's environment stack is the job's jobEnvironments plus exactly one +# step's stepEnvironments, so at most one wrap-defining environment may be +# reachable in any single session. Mirrors openjd-rs +# validate_v2023_09/wrap_actions.rs. + + +def _wrap_env(name): + return {"name": name, "script": {"actions": dict(_ALL_THREE)}} + + +def _plain_step(name, *, step_environments=None): + step = {"name": name, "script": {"actions": {"onRun": _cmd("hi")}}} + if step_environments is not None: + step["stepEnvironments"] = step_environments + return step + + +def _job_template(*, job_environments=None, steps=None, extensions=("WRAP_ACTIONS", "EXPR")): + tmpl = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "steps": steps or [_plain_step("S")], + } + if job_environments is not None: + tmpl["jobEnvironments"] = job_environments + if extensions: + tmpl["extensions"] = list(extensions) + return tmpl + + +def _decode_job(template, extensions=_ALL_EXTS): + return decode_job_template(template=template, supported_extensions=extensions) + + +class TestSingleWrapLayerValid: + def test_single_job_env_wrap_ok(self): + _decode_job(_job_template(job_environments=[_wrap_env("W")])) + + def test_single_step_env_wrap_ok(self): + _decode_job(_job_template(steps=[_plain_step("S", step_environments=[_wrap_env("W")])])) + + def test_wrap_envs_in_different_steps_ok(self): + # Two steps, each with its own single wrap env, and no job-env wrap: + # each session has exactly one wrap layer, so this is valid. + _decode_job( + _job_template( + steps=[ + _plain_step("S1", step_environments=[_wrap_env("W1")]), + _plain_step("S2", step_environments=[_wrap_env("W2")]), + ] + ) + ) + + def test_non_wrap_job_envs_unaffected(self): + # Plenty of non-wrap job environments are fine. + envs = [ + {"name": "E1", "script": {"actions": {"onEnter": _cmd("a")}}}, + {"name": "E2", "script": {"actions": {"onEnter": _cmd("b")}}}, + ] + _decode_job(_job_template(job_environments=envs)) + + +class TestSingleWrapLayerInvalid: + def test_two_job_env_wraps_rejected(self): + with pytest.raises( + DecodeValidationError, match="only one environment in the session stack" + ): + _decode_job(_job_template(job_environments=[_wrap_env("W1"), _wrap_env("W2")])) + + def test_job_env_plus_step_env_wrap_rejected(self): + # One wrap env in jobEnvironments + one in a step's stepEnvironments = + # two wrap layers in that step's session. + with pytest.raises( + DecodeValidationError, match="only one environment in the session stack" + ): + _decode_job( + _job_template( + job_environments=[_wrap_env("W1")], + steps=[_plain_step("S", step_environments=[_wrap_env("W2")])], + ) + ) + + def test_two_wraps_in_one_step_rejected(self): + with pytest.raises( + DecodeValidationError, match="only one environment in the session stack" + ): + _decode_job( + _job_template( + steps=[_plain_step("S", step_environments=[_wrap_env("W1"), _wrap_env("W2")])] + ) + ) + + def test_single_layer_not_enforced_without_extension(self): + # Without WRAP_ACTIONS the wrap fields are rejected for a different + # reason (extension gating), so the template still fails — but the + # single-layer rule itself is only active under WRAP_ACTIONS. + with pytest.raises(DecodeValidationError): + _decode_job( + _job_template( + job_environments=[_wrap_env("W1"), _wrap_env("W2")], + extensions=(), + ) + ) + + +# ── RFC 0008 per-hook wrapped-variable scoping ───────────────────────── +# +# WrappedAction.* may be referenced only in the three wrap hooks; WrappedEnv.* +# only in onWrapEnvEnter/onWrapEnvExit; WrappedStep.* only in onWrapTaskRun; +# and none of them in the ordinary onEnter/onExit actions. + + +class TestWrappedVariableScopeValid: + def test_in_scope_references_ok(self): + # _ALL_THREE already uses WrappedEnv.Name in the env enter/exit hooks + # and WrappedAction.Command in the task-run hook — all in scope. + _decode(_env_template(dict(_ALL_THREE), extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrapped_step_in_task_run_ok(self): + actions = dict(_ALL_THREE) + actions["onWrapTaskRun"] = _cmd("run {{WrappedStep.Name}} {{WrappedAction.Command}}") + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + +class TestWrappedVariableScopeInvalid: + def test_wrapped_action_in_on_enter_rejected(self): + actions = dict(_ALL_THREE) + actions["onEnter"] = _cmd("setup {{WrappedAction.Command}}") + with pytest.raises(DecodeValidationError, match=r"WrappedAction\.\* variables may not"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrapped_env_in_on_exit_rejected(self): + actions = dict(_ALL_THREE) + actions["onExit"] = _cmd("teardown {{WrappedEnv.Name}}") + with pytest.raises(DecodeValidationError, match=r"WrappedEnv\.\* variables may not"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrapped_step_in_env_enter_rejected(self): + actions = dict(_ALL_THREE) + actions["onWrapEnvEnter"] = _cmd("{{WrappedStep.Name}}") + with pytest.raises(DecodeValidationError, match=r"WrappedStep\.\* variables may not"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrapped_env_in_task_run_rejected(self): + actions = dict(_ALL_THREE) + actions["onWrapTaskRun"] = _cmd("{{WrappedEnv.Name}}") + with pytest.raises(DecodeValidationError, match=r"WrappedEnv\.\* variables may not"): + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR"])) + + def test_wrapped_var_in_timeout_format_string_rejected(self): + # The scope check must inspect the timeout FormatString too (not only + # command/args). timeout is a FormatString under FEATURE_BUNDLE_1. + actions = dict(_ALL_THREE) + actions["onEnter"] = { + "command": "echo", + "args": ["hi"], + "timeout": "{{ WrappedAction.Timeout }}", + } + with pytest.raises(DecodeValidationError, match=r"WrappedAction\.\* variables may not"): + _decode( + _env_template(actions, extensions=["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"]), + extensions=_ALL_EXTS, + ) + + def test_wrapped_action_timeout_in_wrap_hook_ok(self): + # WrappedAction.Timeout is in scope inside the wrap hooks, including + # when referenced from the timeout FormatString field. + actions = dict(_ALL_THREE) + actions["onWrapTaskRun"] = { + "command": "echo", + "args": ["{{ WrappedAction.Command }}"], + "timeout": "{{ WrappedAction.Timeout }}", + } + _decode(_env_template(actions, extensions=["WRAP_ACTIONS", "EXPR", "FEATURE_BUNDLE_1"])) + + +# ── RFC 0008 create_job instantiation ────────────────────────────────── +# +# create_job re-validates the instantiated Job submodels without a parsing +# context. The EnvironmentActions extension gate must therefore only enforce +# the WRAP_ACTIONS/EXPR requirement at decode time (context present), not at +# instantiation time -- otherwise a template that decodes cleanly fails when a +# Job is generated from it. + + +class TestWrapActionsCreateJob: + def test_create_job_with_wrap_env_succeeds(self): + # A wrap-defining jobEnvironment decodes AND instantiates into a Job. + # Regression: instantiate_model used to re-fire the WRAP_ACTIONS + # extension gate (no context) and raise "require the WRAP_ACTIONS + # extension." + jt = _decode_job(_job_template(job_environments=[_wrap_env("W")])) + job = create_job(job_template=jt, job_parameter_values={}) + env_actions = job.jobEnvironments[0].script.actions + assert env_actions.onWrapEnvEnter is not None + assert env_actions.onWrapTaskRun is not None + assert env_actions.onWrapEnvExit is not None + + def test_create_job_with_step_env_wrap_succeeds(self): + jt = _decode_job( + _job_template(steps=[_plain_step("S", step_environments=[_wrap_env("W")])]) + ) + job = create_job(job_template=jt, job_parameter_values={}) + step_env_actions = job.steps[0].stepEnvironments[0].script.actions + assert step_env_actions.onWrapTaskRun is not None