diff --git a/.gitignore b/.gitignore index 1bf2c8f9..eec418ab 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ __pycache__/ /dist _version.py *.log + +# Local one-off helper scripts (not part of the project) +/tmp/ diff --git a/pyproject.toml b/pyproject.toml index e5277e8c..da8cf4ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,11 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ + # NOTE (version-skew guard): this branch requires openjd-model features that + # only exist as of openjd-model-for-python PR #318 (FormatString.resolve_value, + # FormatString.whole_field_expression, evaluate_let_bindings, Step.let). + # Before release, bump the version floor below to the first openjd-model + # release that includes PR #318's surface. "openjd-model >= 0.10,< 0.12", "pywin32 >= 307; platform_system == 'Windows'", "psutil >= 5.9,< 7.3; platform_system == 'Windows'", diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index b5444d9a..61fe86a4 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -209,11 +209,44 @@ def __init__( self._user = user def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None: + records = self.allocate_file_paths(files, symtab) + self.write_file_contents(records, symtab) + + def allocate_file_paths( + self, files: EmbeddedFilesListType, symtab: SymbolTable + ) -> list[_FileRecord]: + """Allocate the on-disk paths for the embedded files and define their + ``Env.File.*``/``Task.File.*`` symbols in ``symtab``, without writing + the file contents. + + Splitting allocation from :meth:`write_file_contents` lets the runner + evaluate EXPR ``let`` bindings between the two phases (RFC 0005): a + file's *path* never depends on ``let`` values (``filename`` is a plain + string), so the ``Env.File.*``/``Task.File.*`` symbols are available + to the bindings, while a file's ``data`` is written afterwards so it + can reference let-bound values. Mirrors the openjd-rs runners. + """ if self._scope == EmbeddedFilesScope.ENV: self._logger.info("Writing embedded files for Environment to disk.") else: self._logger.info("Writing embedded files for Task to disk.") + records = self.allocate_records(files) + self._define_symbols(records, symtab) + return records + + def allocate_records(self, files: EmbeddedFilesListType) -> list["_FileRecord"]: + """Allocate the on-disk paths for the embedded files, without logging + or defining any symbols. + + Used by the Session to pre-allocate a wrap environment's file records + once (RFC 0008); each wrap-hook invocation then defines the symbols + and logs through :meth:`register_file_paths` against its own symbol + table. + + Raises: + RuntimeError: If a file path could not be allocated. + """ try: records = list[_FileRecord]() # Generate the symbol table values and filenames @@ -221,18 +254,46 @@ def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None # Raises: OSError symbol, filename = self._get_symtab_entry(file) records.append(_FileRecord(symbol=symbol, filename=filename, file=file)) + return records + except (OSError, ValueError) as err: + raise RuntimeError(f"Could not write embedded file: {err}") - # Add symbols to the symbol table - for record in records: - symtab[record.symbol] = str(record.filename) - self._logger.info( - f"Mapping: {record.symbol} -> {record.filename}", - extra=LogExtraInfo( - openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO - ), - ) + def register_file_paths(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Define the ``Env.File.*``/``Task.File.*`` symbols in ``symtab`` + for already-allocated records, without allocating new paths. - # Write the files to disk. + Used to reuse a wrap environment's embedded-file records across + wrap-hook invocations (RFC 0008): the on-disk paths are allocated + once so the symbols stay stable and unnamed files do not accumulate, + while the contents are re-resolved and rewritten per invocation via + :meth:`write_file_contents`. + """ + if self._scope == EmbeddedFilesScope.ENV: + self._logger.info("Writing embedded files for Environment to disk.") + else: + self._logger.info("Writing embedded files for Task to disk.") + self._define_symbols(records, symtab) + + def _define_symbols(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + # Add symbols to the symbol table. For EXPR evaluation the + # Env.File.*/Task.File.* symbols are host-format path values + # (property access like `.parent` works), matching openjd-rs; + # the legacy (non-EXPR) interpolation path ignores the type and + # keeps the string form. + for record in records: + symtab[record.symbol] = str(record.filename) + symtab.expr_types[record.symbol] = "PATH" + self._logger.info( + f"Mapping: {record.symbol} -> {record.filename}", + extra=LogExtraInfo( + openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO + ), + ) + + def write_file_contents(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Resolve each allocated file's ``data`` against ``symtab`` and write + it to disk. See :meth:`allocate_file_paths`.""" + try: for record in records: # Raises: OSError self._materialize_file(record.filename, record.file, symtab) diff --git a/src/openjd/sessions/_path_mapping.py b/src/openjd/sessions/_path_mapping.py index b84178ff..65c00ef4 100644 --- a/src/openjd/sessions/_path_mapping.py +++ b/src/openjd/sessions/_path_mapping.py @@ -4,27 +4,43 @@ from enum import Enum from os import name as os_name from pathlib import PurePath, PurePosixPath, PureWindowsPath +from typing import Optional, Union class PathFormat(str, Enum): POSIX = "POSIX" WINDOWS = "WINDOWS" + # RFC 0006 §2.3.2 (EXPR extension): URI-form source paths + # (e.g. "s3://bucket/prefix") that map to local filesystem paths. + URI = "URI" @dataclass(frozen=True) class PathMappingRule: source_path_format: PathFormat - source_path: PurePath + # URI-format rules keep the source as the raw string: URIs are not + # filesystem paths, and PurePath normalization would corrupt the + # "scheme://" separator. + source_path: Union[PurePath, str] destination_path: PurePath def __init__( - self, *, source_path_format: PathFormat, source_path: PurePath, destination_path: PurePath + self, + *, + source_path_format: PathFormat, + source_path: Union[PurePath, str], + destination_path: PurePath, ): if source_path_format == PathFormat.POSIX: if not isinstance(source_path, PurePosixPath): raise ValueError( "Path mapping rule source_path_format does not match source_path type" ) + elif source_path_format == PathFormat.URI: + if not isinstance(source_path, str): + raise ValueError( + "Path mapping rule source_path must be a string for the URI source_path_format" + ) else: if not isinstance(source_path, PureWindowsPath): raise ValueError( @@ -49,9 +65,12 @@ def from_dict(rule: dict[str, str]) -> "PathMappingRule": raise ValueError(f"Path mapping rule requires the following fields: {field_names}") source_path_format = PathFormat(rule["source_path_format"].upper()) - source_path: PurePath + source_path: Union[PurePath, str] if source_path_format == PathFormat.POSIX: source_path = PurePosixPath(rule["source_path"]) + elif source_path_format == PathFormat.URI: + # Keep URIs verbatim; PurePath would collapse "scheme://". + source_path = rule["source_path"] else: source_path = PureWindowsPath(rule["source_path"]) destination_path = PurePath(rule["destination_path"]) @@ -76,6 +95,65 @@ def to_dict(self) -> dict[str, str]: "destination_path": str(self.destination_path), } + @staticmethod + def _uri_path_start(uri: str) -> Optional[int]: + """Byte offset where the path component begins in a URI (after + ``scheme://authority``), or None when there is no ``://`` separator. + Mirrors openjd-rs's ``uri_path_start``.""" + scheme_sep = uri.find("://") + if scheme_sep == -1: + return None + authority_start = scheme_sep + 3 + slash = uri.find("/", authority_start) + return len(uri) if slash == -1 else slash + + def _apply_uri(self, path: str) -> tuple[bool, str]: + """Apply a URI-format rule, mirroring openjd-rs's ``apply_uri``. + + Per RFC 3986 the scheme and authority match case-insensitively while + the path portion matches case-sensitively, on whole path components. + The result is a local path in the host's format. + """ + sep = "/" if os_name == "posix" else "\\" + source = str(self.source_path) + src_path_start = self._uri_path_start(source) + src_path_start = 0 if src_path_start is None else src_path_start + inp_path_start = self._uri_path_start(path) + inp_path_start = 0 if inp_path_start is None else inp_path_start + + # Scheme+authority must match case-insensitively. + if path[:inp_path_start].lower() != source[:src_path_start].lower(): + return False, path + # Path portion must match case-sensitively, on a component boundary. + src_path = source[src_path_start:] + inp_path = path[inp_path_start:] + if not inp_path.startswith(src_path): + return False, path + remainder = inp_path[len(src_path) :] + if remainder and not remainder.startswith("/"): + return False, path + + child_parts = remainder[1:].split("/") if remainder else [] + result = str(self.destination_path) + for part in child_parts: + result += sep + part + if path.endswith("/") and not result.endswith(sep): + result += sep + return True, result + + def source_path_component_count(self) -> int: + """Number of components in the source path, used to order rules from + most to least specific. For URI sources the ``scheme://authority`` + counts as one component plus one per path segment.""" + if isinstance(self.source_path, PurePath): + return len(self.source_path.parts) + source = str(self.source_path) + path_start = self._uri_path_start(source) + if path_start is None: + return 1 + path_portion = source[path_start:].strip("/") + return 1 + (len(path_portion.split("/")) if path_portion else 0) + def apply(self, *, path: str) -> tuple[bool, str]: """Applies the path mapping rule on the given path, if it matches the rule. Does not collapse ".." since symbolic paths could be used. @@ -83,18 +161,24 @@ def apply(self, *, path: str) -> tuple[bool, str]: Returns: tuple[bool, str] - indicating if the path matched the rule and the resulting mapped path. If it doesn't match, then it returns the original path unmodified. """ + if self.source_path_format == PathFormat.URI: + return self._apply_uri(path) + source_path = self.source_path + if not isinstance(source_path, PurePath): + # The constructor guarantees non-URI rules carry PurePath sources. + raise TypeError( + "Path mapping rule source_path must be a PurePath for filesystem source formats" + ) pure_path: PurePath if self.source_path_format == PathFormat.POSIX: pure_path = PurePosixPath(path) else: pure_path = PureWindowsPath(path) - if not pure_path.is_relative_to(self.source_path): + if not pure_path.is_relative_to(source_path): return False, path - remapped_parts = ( - self.destination_path.parts + pure_path.parts[len(self.source_path.parts) :] - ) + remapped_parts = self.destination_path.parts + pure_path.parts[len(source_path.parts) :] if os_name == "posix": result = str(PurePosixPath(*remapped_parts)) if self._has_trailing_slash(self.source_path_format, path): diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 27083937..5f66d236 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -2,6 +2,7 @@ import json import os +import re import stat import shlex from abc import ABC, abstractmethod @@ -11,14 +12,17 @@ from enum import Enum from pathlib import Path from threading import Lock, Timer -from typing import Callable, Optional, Sequence, Type, cast +from typing import Any, Callable, Optional, Sequence, Type, cast from types import TracebackType from tempfile import mkstemp from openjd.model import SymbolTable from openjd.model import FormatStringError +from openjd.model import evaluate_let_bindings from openjd.model.v2023_09 import Action as Action_2023_09 -from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user +from openjd.model.v2023_09 import CancelationMethodDeferred as CancelationMethodDeferred_2023_09 +from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import log_subsection_banner, LoggerAdapter, LogContent, LogExtraInfo from ._os_checker import is_posix from ._session_user import SessionUser @@ -32,9 +36,20 @@ "TerminateCancelMethod", "NotifyCancelMethod", "ScriptRunnerBase", + "apply_let_bindings", + "resolve_action_arg_values", + "resolve_effective_cancelation", + "resolve_optional_int_field", ) +_STRICT_INT_RE = re.compile(r"[+-]?[0-9]+") +"""The exact integer grammar accepted for dynamically resolved integer fields +(FEATURE_BUNDLE_1 ``timeout`` / ``notifyPeriodInSeconds`` format strings): +an optional sign followed by ASCII digits, exactly what Rust's ``str::parse`` +accepts. See resolve_optional_int_field.""" + + class ScriptRunnerState(str, Enum): """State of a ScriptRunner.""" @@ -90,6 +105,229 @@ class NotifyCancelMethod(CancelMethod): """Amount of time after a SIGTERM to wait to do the SIGKILL""" +def resolve_action_arg_values(args: Optional[Sequence], symtab: SymbolTable) -> list[str]: + """Resolve an action's ``args`` field into its flat list of argument + strings (excluding the command). + + RFC 0005 §1.3.2 argument semantics, mirroring openjd-rs's + resolve_action_args: a whole-field expression argument resolves typed — + a null result skips the argument, a list result flattens inline (one + argument per element, rendered with the engine's display coercion), and + a scalar becomes a single argument. Multi-segment format strings and + legacy (non-EXPR) expressions resolve to their string form. + + Shared by the enforcement path (:meth:`ScriptRunnerBase._run_action`) + and the RFC 0008 ``WrappedAction.Args`` injection, so a wrap hook sees + exactly the arguments the wrapped action would have run with unwrapped. + + Raises: + FormatStringError: If an argument's expression cannot be resolved. + """ + resolved: list[str] = [] + if args is not None: + for arg in args: + try: + value = arg.resolve_value(symtab=symtab) + except FormatStringError: + # Mirror openjd-rs's resolve_action_args: when typed + # resolution fails (e.g. a legacy-parsed expression meeting + # a typed symbol value), fall back to plain string + # resolution — which raises FormatStringError itself if the + # argument is genuinely unresolvable. + resolved.append(arg.resolve(symtab=symtab)) + continue + if isinstance(value, str): + resolved.append(value) + elif getattr(value, "is_null", False): + continue + elif str(getattr(value, "type", "")).startswith("list["): + resolved.extend(str(element) for element in value) + else: + resolved.append(str(value)) + return resolved + + +def resolve_optional_int_field( + value: Any, + symtab: SymbolTable, + *, + ge: Optional[int] = None, + le: Optional[int] = None, + description: str, +) -> Optional[int]: + """Resolve an optional int-or-format-string field (e.g. an action's + ``timeout`` or a cancelation's ``notifyPeriodInSeconds``) into an + optional integer. + + - ``None`` (field omitted) stays ``None``. + - A literal ``int`` passes through unchecked: literal values were + bounds-checked by the static validator at parse time. + - A FormatString (FEATURE_BUNDLE_1) is resolved against ``symtab`` + using typed resolution. A whole-field expression that resolves to a + typed null is treated as if the field were not provided (``None`` — + the caller applies any positional schema default). Any other result + — including a genuine empty string — must be an integer within the + given bounds; the bounds apply here because format-string values + could not be checked at parse time. This matches the openjd-rs + runtime (resolve_action_timeout / resolve_notify_period_seconds), + which only treats an ExprValue::Null result as "field omitted" and + errors on an empty string. + + Raises: + ValueError: If the resolved value is not an integer, or violates + the ``ge``/``le`` bounds. + FormatStringError: If expression resolution itself fails. + """ + if value is None: + return None + if isinstance(value, int): + return value + # Typed resolution: a whole-field EXPR expression yields the engine's + # typed value, so a null result is distinguishable from a genuine empty + # string. Multi-segment and legacy (non-EXPR) format strings fall back + # to plain string resolution — correct, since typed nulls only exist + # under EXPR whole-field semantics (Template Schemas 5.3). + resolved_value = value.resolve_value(symtab=symtab) + if getattr(resolved_value, "is_null", False): + return None + resolved = str(resolved_value) + if ge is not None and le is not None: + constraint = f"between {ge} and {le}" + elif ge == 1 and le is None: + constraint = "a positive integer" + elif ge is not None: + constraint = f"an integer >= {ge}" + else: + constraint = "an integer" + # Strict ASCII integer grammar, matching the Rust runtime's str::parse + # (openjd-rs resolve_action_timeout / resolve_notify_period_seconds). + # Python's int() is more lenient — it accepts surrounding whitespace, + # digit-group underscores ("1_0" == 10), and non-ASCII decimal digits — + # all of which Rust rejects, so accepting them here would be a + # spec-observable divergence. + if _STRICT_INT_RE.fullmatch(resolved) is None: + raise ValueError(f"{description} must be {constraint}, got '{resolved}'") + result = int(resolved) + if (ge is not None and result < ge) or (le is not None and result > le): + raise ValueError(f"{description} must be {constraint}, got '{result}'") + return result + + +def resolve_effective_cancelation( + cancelation: Any, symtab: SymbolTable +) -> tuple[Optional[str], Optional[int]]: + """Resolve an action's cancelation config against the symbol table into + an effective ``(mode, notify_period_seconds)`` pair. + + What is the problem this solves? + + Format strings are normally delay-processed: the parser stores "this is + a format string" and the value resolves right before the action runs. + But a cancelation's ``mode`` is the *schema selector* — the parser needs + it to know which object shape it is reading — while a forwarded value + like ``mode: "{{WrappedAction.Cancelation.Mode}}"`` (RFC 0008 + round-trip forwarding) only exists at run time. The model therefore + carries such a mode as :class:`CancelationMethodDeferred`, and *this* + function is where the deferred decision finally lands: with the + ``WrappedAction.*`` values in the symbol table, the mode expression + resolves to ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, or null — and + a null mode means the whole cancelation object is treated as never + declared (the runtime default applies). + + Returns: + (mode, notify_period_seconds) where ``mode`` is ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (no ```` + declared, or a deferred mode that resolved to null); and + ``notify_period_seconds`` is ``None`` when the field was omitted or + its whole-field expression resolved to null — the caller applies + the positional schema default. + + Raises: + ValueError: If a deferred mode resolves to anything other than the + two method names or null, if a resolved TERMINATE carries a + non-null notify period, or if a notify period does not resolve + to a positive integer. + FormatStringError: If expression resolution itself fails. + """ + + def resolve_period(period: Any) -> Optional[int]: + # Bounds mirror the static validator's on literal values (Template + # Schemas 5.3.2: 1..600); see resolve_optional_int_field. + return resolve_optional_int_field( + period, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) + + if cancelation is None: + return (None, None) + if isinstance(cancelation, CancelationMethodDeferred_2023_09): + # Typed resolution. Null semantics apply only to a whole-field + # expression ("{{ ... }}" with no surrounding text, target type + # string? — Template Schemas 5.3), and resolve_value only yields a + # typed null for a whole-field EXPR expression; every other format + # string resolves to its plain string form. A format string that + # happens to resolve to the empty string is NOT null; it falls + # through to the "must resolve to..." error below (matching the + # openjd-rs runtime, which errors on any non-null, non-mode-name + # result). + mode_value = cancelation.mode.resolve_value(symtab=symtab) + if getattr(mode_value, "is_null", False): + # Null mode drops the ENTIRE cancelation object: mode is the + # object's required discriminator, so an "omitted" mode cannot + # leave a partial object behind. The action behaves exactly as + # if no were declared. + return (None, None) + mode = str(mode_value) + if mode == CancelationMode_2023_09.TERMINATE.value: + # Post-resolution the object must validate against the resolved + # variant's shape: TERMINATE admits no notify period. + if resolve_period(cancelation.notifyPeriodInSeconds) is not None: + raise ValueError( + "cancelation mode resolved to TERMINATE, which does not " + "accept notifyPeriodInSeconds" + ) + return (CancelationMode_2023_09.TERMINATE.value, None) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(cancelation.notifyPeriodInSeconds), + ) + raise ValueError( + "cancelation mode must resolve to TERMINATE, NOTIFY_THEN_TERMINATE, " + f"or null; got '{mode}'" + ) + if cancelation.mode == CancelationMode_2023_09.TERMINATE: + return (CancelationMode_2023_09.TERMINATE.value, None) + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(getattr(cancelation, "notifyPeriodInSeconds", None)), + ) + + +def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: + """Evaluate EXPR ``let`` bindings (RFC 0005) and add them to ``symtab``. + + ``let_bindings`` is a script's ``let`` field: an ordered list of + ``"name = expression"`` strings. Each RHS is an EXPR expression evaluated + against the symbol table built so far (so later bindings can reference + earlier ones), and the engine's typed result is stored under the bound + name — a let-bound path stays a path for property access, and float + rendering fidelity is preserved — matching the Rust runtime's natively + typed symbol table. + + The runners evaluate bindings after embedded-file *path* allocation and + before file *contents* are written, so a binding may reference + ``Env.File.*``/``Task.File.*`` and a file's ``data`` may reference + let-bound values (mirroring openjd-rs's runner ordering). + + Raises: + ValueError (FormatStringError/ExpressionError): if a binding's + expression cannot be evaluated. + """ + # Single-sourced in openjd.model (parse-memoized; skips malformed + # bindings; raises ValueError naming the failing binding). + evaluate_let_bindings(symtab=symtab, let_bindings=let_bindings) + + class ScriptRunnerBase(ABC): """Base class for a runnable Environment or Step Script. Responsible for running a *single* Action, and optionally canceling it. @@ -177,6 +415,20 @@ class ScriptRunnerBase(ABC): e.g. We failed to write embedded files before even trying to run the action. """ + _resolved_cancel_method: Optional[CancelMethod] + """The running action's effective cancel method, resolved by + :meth:`_run_action` right before subprocess launch — against the same + final (let/embedded-file enriched) symbol table the command and args + resolved with — and consumed by the runners' :meth:`cancel`. + + Resolving at launch matches the openjd-rs runtime (``run_action`` + resolves ``cancel_method_for_action`` up front): a cancelation whose + deferred mode or notify period cannot be resolved fails the action at + start rather than surfacing only if a cancel later occurs, and the + resolution scope is the action's own (a mode referencing a script-level + ``let`` binding resolves correctly). ``None`` until an action launches. + """ + def __init__( self, *, @@ -225,6 +477,7 @@ def __init__( # Will run at most the run futures self._pool = ThreadPoolExecutor(max_workers=1) self._state_override = None + self._resolved_cancel_method = None self._print_section_banner = True # Context manager for use in our tests @@ -245,6 +498,21 @@ def shutdown(self) -> None: """ self._pool.shutdown() + def _fail_action(self, message: str) -> None: + """Fail the action through the normal failure path: surface the + failure reason to the customer via the action filter + (``openjd_fail``), set the FAILED state override, and invoke the + callback. The subprocess's future may not have been started yet, + but the Session still needs to know that the action is over. + """ + self._logger.info( + f"openjd_fail: {message}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + self._callback(ActionState.FAILED) + @abstractmethod def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False @@ -329,17 +597,7 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N args, self._user, self._os_env_vars, str(self._session_working_directory) ) except RuntimeError as e: - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(e)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(e)) return subprocess_args = [filename] if is_posix() else args @@ -398,10 +656,24 @@ def _materialize_files( files: EmbeddedFilesListType, dest_directory: Path, symtab: SymbolTable, + let_bindings: Optional[list[str]] = None, + preallocated_records: Optional[list[_FileRecord]] = None, ) -> None: """Helper for derived classes that wraps all of the logic around materializing embedded files to disk. + When ``let_bindings`` is given, they are evaluated between file-path + allocation and content writing (RFC 0005, mirroring the openjd-rs + runners): a file's *path* never depends on ``let`` values (filenames + are plain strings), so ``Env.File.*``/``Task.File.*`` are available to + the bindings, while a file's ``data`` is written afterwards so it can + reference let-bound values. + + When ``preallocated_records`` is given (RFC 0008: a wrap + environment's files, whose paths the Session allocates once and + reuses across wrap-hook invocations), path allocation is skipped; + the records' symbols are defined in ``symtab`` and the contents are + re-resolved and written as usual. """ file_writer = EmbeddedFiles( logger=self._logger, @@ -410,21 +682,30 @@ def _materialize_files( user=self._user, ) try: - file_writer.materialize(files, symtab) - except RuntimeError as exc: - # Had a problem writing at least one file to disk. - # Surface the error. - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + if preallocated_records is not None: + records = preallocated_records + file_writer.register_file_paths(records, symtab) + else: + records = file_writer.allocate_file_paths(files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + except (RuntimeError, ValueError) as exc: + # Had a problem writing at least one file to disk, or evaluating + # a `let` binding (FormatStringError/ExpressionError subclass + # ValueError). Surface the error. + self._fail_action(str(exc)) + + def _apply_let_bindings_or_fail(self, symtab: SymbolTable, let_bindings: list[str]) -> bool: + """Evaluate the script's EXPR ``let`` bindings into ``symtab``. On an + evaluation error the action is failed through the normal failure path + (openjd_fail log, FAILED state, callback). Returns True on success.""" + try: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + except ValueError as exc: + self._fail_action(str(exc)) + return False + return True def _run_action( self, @@ -432,6 +713,7 @@ def _run_action( symtab: SymbolTable, *, default_timeout: Optional[timedelta] = None, + default_notify_period_seconds: int = 30, ) -> None: """Helper for derived classes to run a specific Action. @@ -443,30 +725,82 @@ def _run_action( default_timeout (Optional[timedelta], optional): Default timeout duration for the action if no timeout is specified in the action. The default behaviour if None is passed will allow the action to run indefinitely until it completes. + default_notify_period_seconds (int): The Template Schemas 5.3.2 + positional default applied when a NOTIFY_THEN_TERMINATE + cancelation omits its notify period (120 for a task's onRun, + 30 for any other action). """ assert isinstance(action, Action_2023_09) try: command = [action.command.resolve(symtab=symtab)] - if action.args is not None: - command.extend(s.resolve(symtab=symtab) for s in action.args) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening) — see resolve_action_arg_values, shared with the + # RFC 0008 WrappedAction.Args injection. + command.extend(resolve_action_arg_values(action.args, symtab)) except FormatStringError as exc: # Extremely unlikely since a JobTemplate needs to have passed # validation before we could be running it, but just to be safe. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(exc)) else: time_limit: Optional[timedelta] = default_timeout - if action.timeout: - time_limit = timedelta(seconds=action.timeout) # type: ignore[arg-type] + # A FormatString timeout (FEATURE_BUNDLE_1) is resolved right + # before the action runs. A whole-field expression that + # resolves to a typed null — e.g. forwarding + # `timeout: "{{WrappedAction.Timeout}}"` (RFC 0008) when the + # wrapped action specified no timeout — is treated as if the + # field were not provided, so the positional default applies. + try: + seconds = resolve_optional_int_field( + action.timeout, symtab, ge=1, description="timeout" + ) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses + # ValueError, so this covers both a failed resolution and a + # non-positive-integer resolved value. + self._fail_action(str(exc)) + return + if seconds is not None: + time_limit = timedelta(seconds=seconds) + # Resolve the action's effective cancelation NOW, against the + # same final scope the command/args/timeout resolved with — a + # deferred (format-string) mode or FEATURE_BUNDLE_1 notify + # period may reference script-level `let` bindings or + # Env.File.*/Task.File.* symbols that only exist in this scope. + # This matches the openjd-rs runtime (run_action resolves + # cancel_method_for_action before launching): an unresolvable or + # invalid cancelation fails the action at start instead of + # surfacing only if a cancel later occurs. The runners' + # cancel() consumes the stored method. + try: + mode, period = resolve_effective_cancelation(action.cancelation, symtab) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses ValueError. + self._fail_action(str(exc)) + return + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + # Note: The default cancelation for a 2023-09 script is Terminate + self._resolved_cancel_method = TerminateCancelMethod() + else: + self._resolved_cancel_method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=(period if period is not None else default_notify_period_seconds) + ) + ) self._run(command, time_limit) + def _cancel_with_resolved_method( + self, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + """Shared implementation of the runners' :meth:`cancel`: cancel with + the effective cancel method that :meth:`_run_action` resolved at + launch time. No-op when no action was launched (e.g. setup failed + before the subprocess started — there is nothing to cancel). + """ + if self._resolved_cancel_method is None: + return + # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. + self._cancel(self._resolved_cancel_method, time_limit, mark_action_failed) + def _cancel( self, method: CancelMethod, diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 2ce1962c..4e96df80 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -6,25 +6,28 @@ from typing import Callable, Optional from openjd.model import SymbolTable -from openjd.model.v2023_09 import Action as Action_2023_09 -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 -from ._embedded_files import EmbeddedFilesScope +from ._embedded_files import EmbeddedFilesScope, _FileRecord from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, ) from ._session_user import SessionUser -from ._types import ActionModel, ActionState, EnvironmentScriptModel +from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ActionModel, + ActionState, + EnvironmentScriptModel, +) + +__all__ = ("EnvironmentScriptRunner", "WRAP_HOOK_ACTION_NAMES") -__all__ = ("EnvironmentScriptRunner",) + +WRAP_HOOK_ACTION_NAMES = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") +"""The three RFC 0008 wrap-hook action names an Environment's script may +define. Single-sourced here for the runner's hook dispatch and the Session's +wrap-environment lookup/validation.""" _ENV_EXIT_DEFAULT_TIMEOUT = timedelta(minutes=5) @@ -55,6 +58,13 @@ class EnvironmentScriptRunner(ScriptRunnerBase): """If defined, then this is the action that is currently running, or was last run. """ + _preallocated_file_records: Optional[list[_FileRecord]] + """RFC 0008: the script's embedded-file records with on-disk paths already + allocated by the Session (a wrap environment's files are allocated once + and reused across wrap-hook invocations). When set, the runner skips path + allocation and only re-resolves/rewrites the file contents. + """ + def __init__( self, *, @@ -72,6 +82,9 @@ def __init__( symtab: SymbolTable, # Directory within which files/attachments should be materialized session_files_directory: Path, + # RFC 0008: pre-allocated embedded-file records to reuse (see + # _preallocated_file_records) + preallocated_file_records: Optional[list[_FileRecord]] = None, ): """ Arguments (from base class): @@ -91,6 +104,11 @@ def __init__( Script's scope (exluding any symbols defined within the Step Script itself). session_files_directory (Path): The location in the filesystem where embedded files will be materialized. + preallocated_file_records (Optional[list[_FileRecord]]): RFC 0008: the script's + embedded-file records with on-disk paths already allocated by the Session + (a wrap environment's files are allocated once and reused across wrap-hook + invocations). When given, the runner skips path allocation and only + re-resolves/rewrites the file contents. """ super().__init__( logger=logger, @@ -104,6 +122,7 @@ def __init__( self._symtab = symtab self._session_files_directory = session_files_directory self._action = None + self._preallocated_file_records = preallocated_file_records if self._environment_script and not isinstance( self._environment_script, EnvironmentScript_2023_09 @@ -120,7 +139,13 @@ def _run_env_action( log_subsection_banner(self._logger, "Phase: Setup") - # Write any embedded files to disk + let_bindings = ( + self._environment_script.let if self._environment_script is not None else None + ) + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Env.File.*), and contents are written after (so `data` can reference + # let-bound values) — mirroring the openjd-rs runner. if ( self._environment_script is not None and self._environment_script.embeddedFiles is not None @@ -132,15 +157,26 @@ def _run_env_action( self._environment_script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, + preallocated_records=self._preallocated_file_records, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab # Construct the command by evalutating the format strings in the command self._action = action - self._run_action(self._action, symtab, default_timeout=default_timeout) + self._run_action( + self._action, + symtab, + default_timeout=default_timeout, + default_notify_period_seconds=ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def enter(self) -> None: """Run the Environment's onEnter action.""" @@ -181,6 +217,52 @@ def exit(self) -> None: default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT, ) + def wrap_task_run(self) -> None: + """Run the Environment's onWrapTaskRun action, wrapping a task's onRun.""" + self._run_wrap_hook("onWrapTaskRun") + + def wrap_env_enter(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvEnter`` action, + substituting it for an inner environment's ``onEnter``.""" + self._run_wrap_hook("onWrapEnvEnter") + + def wrap_env_exit(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvExit`` action, + substituting it for an inner environment's ``onExit``.""" + self._run_wrap_hook("onWrapEnvExit", default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT) + + def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = None) -> None: + """Common dispatch for the three RFC 0008 wrap hooks. ``hook`` is + one of ``onWrapEnvEnter``, ``onWrapTaskRun``, or ``onWrapEnvExit``.""" + if hook not in WRAP_HOOK_ACTION_NAMES: + # Guard the getattr below: without this, a typo'd hook name + # would silently become a SUCCESS no-op. + raise ValueError(f"Unknown wrap hook name: {hook}") + if self.state != ScriptRunnerState.READY: + raise RuntimeError("This cannot be used to run a second subprocess.") + + # For the type checker + if self._environment_script is not None: + assert isinstance(self._environment_script, EnvironmentScript_2023_09) + + action = ( + getattr(self._environment_script.actions, hook, None) + if self._environment_script is not None + else None + ) + if action is None: + self._state_override = ScriptRunnerState.SUCCESS + # Nothing to do, no wrap action defined. Call the callback + # to inform the caller that the run is complete, and then exit. + if self._callback is not None: + self._callback(ActionState.SUCCESS) + return + + if default_timeout is not None: + self._run_env_action(action, default_timeout=default_timeout) + else: + self._run_env_action(action) + def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: @@ -188,27 +270,8 @@ def cancel( # Nothing to do. return - # For the type checker - assert isinstance(self._action, Action_2023_09) - - method: CancelMethod - if ( - self._action.cancelation is None - or self._action.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): - # Note: Default cancelation for a 2023-09 Step Script is Terminate - method = TerminateCancelMethod() - else: - model_cancel_method = self._action.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 30s for a 2023-09 Environment Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=30)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] - ) - - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + # Cancel with the effective method resolved at launch time by + # _run_action, against the action's own final scope (the script's + # lets and Env.File.* symbols, plus WrappedAction.* for a wrap + # hook) — openjd-rs parity. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_runner_step_script.py b/src/openjd/sessions/_runner_step_script.py index a4d2e654..1d49736b 100644 --- a/src/openjd/sessions/_runner_step_script.py +++ b/src/openjd/sessions/_runner_step_script.py @@ -6,22 +6,15 @@ from typing import Callable, Optional from openjd.model import SymbolTable -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 from openjd.model.v2023_09 import StepScript as StepScript_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from ._embedded_files import EmbeddedFilesScope from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, ) from ._session_user import SessionUser -from ._types import ActionState, StepScriptModel +from ._types import TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, StepScriptModel __all__ = ("StepScriptRunner",) @@ -104,7 +97,11 @@ def run(self) -> None: # For the type checker. assert isinstance(self._script, StepScript_2023_09) - # Write any embedded files to disk + let_bindings = self._script.let + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Task.File.*), and contents are written after (so `data` can + # reference let-bound values) — mirroring the openjd-rs runner. if self._script.embeddedFiles is not None: symtab = SymbolTable(source=self._symtab) self._materialize_files( @@ -112,39 +109,28 @@ def run(self) -> None: self._script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab # Construct the command by evalutating the format strings in the command - self._run_action(self._script.actions.onRun, symtab) + self._run_action( + self._script.actions.onRun, + symtab, + default_notify_period_seconds=TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: - # For the type checker. - assert isinstance(self._script, StepScript_2023_09) - - method: CancelMethod - if ( - self._script.actions.onRun.cancelation is None - or self._script.actions.onRun.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): - # Note: Default cancelation for a 2023-09 Step Script is Terminate - method = TerminateCancelMethod() - else: - model_cancel_method = self._script.actions.onRun.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 120s for a 2023-09 Step Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=120)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] - ) - - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + # Cancel with the effective method resolved at launch time by + # _run_action, against the action's own final scope (its lets and + # Task.File.* symbols) — openjd-rs parity. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index dbb8af49..28bae1d7 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from openjd.model import ( + FormatStringError, JobParameterValues, ParameterValue, ParameterValueType, @@ -37,20 +38,29 @@ ValueReferenceConstants as ValueReferenceConstants_2023_09, ) from ._action_filter import ActionMessageKind, ActionMonitoringFilter -from ._embedded_files import write_file_for_user +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import LOG, log_section_banner, LoggerAdapter, LogExtraInfo, LogContent from ._os_checker import is_posix, is_windows from ._path_mapping import PathMappingRule -from ._runner_base import ScriptRunnerBase -from ._runner_env_script import EnvironmentScriptRunner +from ._runner_base import ( + ScriptRunnerBase, + apply_let_bindings, + resolve_action_arg_values, + resolve_effective_cancelation, + resolve_optional_int_field, +) +from ._runner_env_script import EnvironmentScriptRunner, WRAP_HOOK_ACTION_NAMES from ._runner_step_script import StepScriptRunner from ._session_user import SessionUser from ._subprocess import LoggingSubprocess from ._tempdir import TempDir, custom_gettempdir from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, EnvironmentIdentifier, EnvironmentModel, + EnvironmentScriptModel, StepScriptModel, ) from ._version import version @@ -135,6 +145,11 @@ class SimplifiedEnvironmentVariableChanges: """ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariableObject"]): + # Insertion-ordered map of every session-defined variable for this + # environment: the declarative `variables:` map seed plus any + # openjd_env stdout sets/unsets (None = unset). RFC 0008's + # ``WrappedAction.Environment`` surfaces all of these (openjd-rs + # #277); host-inherited variables never enter this map. self._to_set: dict[str, Optional[str]] if is_windows(): @@ -147,13 +162,21 @@ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariable def simplify_ordered_changes(self, changes: list[EnvironmentVariableChange]) -> None: """Apply a given list of sets and unsets to the current state in order""" for change in changes: + name = change.name.upper() if is_windows() else change.name if isinstance(change, EnvironmentVariableSetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = change.value + self._to_set[name] = change.value elif isinstance(change, EnvironmentVariableUnsetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = None + self._to_set[name] = None else: raise ValueError("Unknown type of environment variable change.") + def effective_items(self) -> dict[str, Optional[str]]: + """The effective session-defined variables for this environment, in + insertion order: the declarative ``variables:`` map seed plus any + openjd_env stdout sets/unsets applied so far (``None`` = unset). + Returns a copy; mutating it does not affect this object.""" + return dict(self._to_set) + def apply_to_environment(self, env_vars: dict[str, Optional[str]]) -> None: """Modify a given dictionary of environment variables to reflect the changes""" if is_windows(): @@ -256,6 +279,15 @@ class Session(object): """OS environment variables defined by Open Job Description Environments """ + _wrap_env_file_records: dict[EnvironmentIdentifier, list["_FileRecord"]] + """RFC 0008: per wrap environment, the embedded-file records whose on-disk + paths were allocated on the environment's first wrap-hook invocation and + are reused for every subsequent invocation — so the ``Env.File.*`` symbols + stay stable across tasks and unnamed files do not accumulate on disk. The + file *contents* are still re-resolved and rewritten per invocation by the + runner (a file's ``data`` may reference ``WrappedAction.*``). + """ + _log_filter: Filter """The handler that we've hooked to the LOG. Removed when the Session is deleted. """ @@ -323,6 +355,7 @@ def __init__( *, session_id: str, job_parameter_values: JobParameterValues, + job_name: Optional[str] = None, path_mapping_rules: Optional[list[PathMappingRule]] = None, retain_working_dir: bool = False, user: Optional[SessionUser] = None, @@ -339,6 +372,11 @@ def __init__( job_parameter_values (JobParameterValues): Values for any defined Job Parameters. This is a dictionary where the keys are parameter names, and the values are instances of ParameterValue (a dataclass containing the type and value of the parameter) + job_name (Optional[str]): The resolved name of the Job this Session belongs to. + When provided, it is seeded as the ``Job.Name`` template variable + (RFC 0005; Template Schemas §7.3.1, EXPR extension) for the session's actions — + mirroring the Job.Name symbol that openjd-rs carries in its + session symbol tables. Defaults to None (no Job.Name symbol). path_mapping_rules (Optional[list[PathMappingRule]]): A list of the path mapping rules to apply within all actions running within this session. Defaults to None. retain_working_dir (bool, optional): If set, then the Session's Working Directory @@ -382,21 +420,39 @@ def __init__( self._session_id = session_id self._ending_only = False self._environments = dict() + # Extra EXPR `let` bindings supplied when an environment was entered + # (e.g. the owning step's step-level bindings), re-applied when the + # environment exits so its onExit resolves in the same scope. + self._environment_extra_let_bindings: dict[EnvironmentIdentifier, list[str]] = dict() + # The owning step's name supplied when an environment was entered + # (seeds Step.Name, RFC 0005 EXPR), re-seeded when the environment + # exits so the re-applied extra `let` bindings resolve in the same + # scope. + self._environment_step_names: dict[EnvironmentIdentifier, str] = dict() self._environments_entered = list() self._runner = None self._running_environment_identifier = None self._process_env = dict(os_env_vars) if os_env_vars else dict() self._created_env_vars = dict() + self._wrap_env_file_records = dict() self._retain_working_dir = retain_working_dir self._user = user self._job_parameter_values = dict(job_parameter_values) if job_parameter_values else dict() + self._job_name = job_name self._cleanup_called = False self._callback = callback self._path_mapping_rules = path_mapping_rules[:] if path_mapping_rules else None if self._path_mapping_rules is not None: # Path mapping rules are applied in order of longest to shortest source path, # so sort them for when we apply them. - self._path_mapping_rules.sort(key=lambda rule: -len(rule.source_path.parts)) + self._path_mapping_rules.sort(key=lambda rule: -rule.source_path_component_count()) + # Engine (openjd.expr) form of the path mapping rules, seeded into the + # session's symbol tables so EXPR host-context functions such as + # apply_path_mapping() apply the session's rules at run time — + # mirroring openjd-rs's session-scope HostContext::WithRules. An empty + # list still constructs a host context (the session IS host scope); + # the rules are simply empty. + self._expr_host_rules = self._build_expr_host_rules() self._session_root_directory = session_root_directory if self._session_root_directory is not None: if not self._session_root_directory.is_dir(): @@ -603,12 +659,40 @@ def cancel_action( self._runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + def _make_env_script_runner( + self, + *, + environment_script: Optional[EnvironmentScriptModel], + os_env_vars: dict[str, Optional[str]], + symtab: SymbolTable, + preallocated_file_records: Optional[list[_FileRecord]] = None, + ) -> EnvironmentScriptRunner: + """Construct an :class:`EnvironmentScriptRunner` with this Session's + standard wiring (logger, user, working/files directories, and action + callback). Only the script, subprocess environment, symbol table, and + (for wrap hooks) pre-allocated embedded-file records vary between the + call sites.""" + return EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=os_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=environment_script, + symtab=symtab, + session_files_directory=self.files_directory, + preallocated_file_records=preallocated_file_records, + ) + def enter_environment( self, *, environment: EnvironmentModel, identifier: Optional[EnvironmentIdentifier] = None, os_env_vars: Optional[dict[str, str]] = None, + extra_let_bindings: Optional[list[str]] = None, + step_name: Optional[str] = None, ) -> EnvironmentIdentifier: """Enters an Open Job Description Environment within this Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -627,6 +711,20 @@ def enter_environment( by values defined in Environments. Key: Environment variable name Value: Value for the environment variable. + extra_let_bindings (Optional[list[str]]): Additional EXPR ``let`` + bindings (RFC 0005) evaluated into the symbol table before the + environment's variables and actions resolve. A step's + environments are entered with the step-level ``let`` bindings + (``Step.let`` on the instantiated Job) so both can reference + them — the v0 counterpart of the per-step resolved symbol + table that openjd-rs threads into enter_environment. + step_name (Optional[str]): The name of the step whose + stepEnvironments are being entered, if any. Seeds + ``Step.Name`` (RFC 0005 EXPR) into the symbol table before + the extra ``let`` bindings evaluate, so step-level bindings + and the environment's variables and actions can reference + it — openjd-rs threads a per-step resolved symbol table into + environment entry, and this is the v0 counterpart. Returns: EnvironmentIdentifier: An identifier by which the Environment is known by to this Session. @@ -638,6 +736,21 @@ def enter_environment( raise RuntimeError( f"Environment {identifier} has already been entered in this Session." ) + + # RFC 0008: at most one Environment in the session stack may + # define any wrap hook. Reject the new environment up front if it + # would create a second wrap layer. + if self._environment_defines_any_wrap_hook(environment): + for existing_id in self._environments_entered: + existing = self._environments[existing_id] + if self._environment_defines_any_wrap_hook(existing): + raise RuntimeError( + "RFC 0008: a session may have at most one Environment " + "defining wrap hooks (onWrapEnvEnter / onWrapTaskRun / " + f"onWrapEnvExit). Environment '{existing.name}' already " + f"defines wrap hooks; cannot also enter '{environment.name}'." + ) + log_section_banner(self._logger, f"Entering Environment: {environment.name}") self._reset_action_state() @@ -646,11 +759,50 @@ def enter_environment( identifier = f"{self._session_id}:{uuid.uuid4().hex}" self._environments[identifier] = environment + if extra_let_bindings: + self._environment_extra_let_bindings[identifier] = list(extra_let_bindings) + if step_name is not None: + self._environment_step_names[identifier] = step_name self._environments_entered.append(identifier) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) + # RFC 0005; Template Schemas §7.3.1 (EXPR): the owning step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. Seeded before the extra `let` bindings + # evaluate so a step-level binding may reference it. + if step_name is not None: + symtab["Step.Name"] = step_name + + # Step-level `let` bindings (RFC 0005) accompany a step's + # environments: evaluate them first so the environment's variables + # and actions can reference them. + if extra_let_bindings: + try: + apply_let_bindings(symtab=symtab, let_bindings=extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate (e.g. it referenced an + # undefined symbol). Fail the action through the normal + # failure path rather than raising out of the public API — + # the environment stays in the entered list, exactly as when + # a failing onEnter subprocess leaves it, so the caller's + # cleanup exits it as usual. + self._created_env_vars[identifier] = SimplifiedEnvironmentVariableChanges( + dict[str, str]() + ) + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return identifier + + # Note: the environment script's own EXPR `let` bindings (RFC 0005) + # are evaluated by the script runner, after embedded-file paths are + # allocated (so bindings can reference Env.File.*). The environment's + # `variables` resolve against the session symbol table without them, + # matching the openjd-rs runtime. + if environment.variables is not None: # We must process the current environment's variables # before we call _evaluate_current_session_env_vars() @@ -690,18 +842,65 @@ def enter_environment( # so it's important to set the action_state to RUNNING before calling enter(), rather # than after -- enter() itself may end up setting the action state to FAILED. - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # RFC 0008: an outer environment's onWrapEnvEnter intercepts an + # inner environment's onEnter. The outer environment's *own* + # onEnter is never wrapped — that's why the lookup excludes the + # environment we just appended (which is at the top of the stack). + on_enter_action = ( + environment.script.actions.onEnter if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvEnter") + if on_enter_action is not None + else None ) - self._runner.enter() + # The wrapping environment is whichever earlier-entered env defines + # onWrapEnvEnter — never the env we just entered. + if wrap_env is environment: + wrap_env = None + + if wrap_env is not None: + # The wrapped onEnter resolves against the INNER environment's + # own scope; the hook itself resolves against `symtab`, and its + # own script's lets/files are evaluated by the script runner + # from wrap_env.script (see _try_inject_wrapped_symbols). On + # failure the environment stays in the entered list, exactly as + # when enter() itself fails, so the caller's cleanup exits it + # as usual. + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( + symtab, environment, on_enter_action, inner_symtab=inner_symtab + ), + fail_message=( + f"Failed to resolve the wrapped onEnter action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvEnter" + ), + ): + return identifier + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return identifier + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=symtab, + preallocated_file_records=wrap_file_records, + ) + self._runner.wrap_env_enter() + else: + self._runner = self._make_env_script_runner( + environment_script=environment.script, + os_env_vars=action_env_vars, + symtab=symtab, + ) + self._runner.enter() return identifier @@ -758,14 +957,54 @@ def exit_environment( # Must be run _before_ we pop _environments_entered action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) + # RFC 0008: capture the openjd_env list for WrappedAction.Environment + # _before_ the exiting environment is removed from tracking, so the + # list includes that environment's own openjd_env variables — the + # real subprocess environment (action_env_vars, computed above) + # includes them, and the wrapped onExit runs with them in the + # unwrapped case too. + wrapped_session_env_list = self._collect_session_env_list() + # Remove the environment from our tracking since we're now exiting it. del self._environments[identifier] self._environments_entered.pop() + # RFC 0008: drop any embedded-file records reused across this (wrap) + # environment's hook invocations; the files themselves live in the + # session directory and are cleaned up with it. + self._wrap_env_file_records.pop(identifier, None) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) self._materialize_path_mapping(environment.revision, action_env_vars, symtab) + + # Re-seed the owning step's name (Step.Name, RFC 0005 EXPR) and + # re-apply the extra `let` bindings this environment was entered with + # (e.g. the owning step's step-level bindings, RFC 0005) so its onExit + # resolves in the same scope as its onEnter. + exit_step_name = self._environment_step_names.pop(identifier, None) + if exit_step_name is not None: + symtab["Step.Name"] = exit_step_name + exit_extra_let_bindings = self._environment_extra_let_bindings.pop(identifier, None) + if exit_extra_let_bindings: + try: + apply_let_bindings(symtab=symtab, let_bindings=exit_extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate. Fail the action through the + # normal failure path rather than raising out of the public + # API — the environment was already removed from tracking + # above, matching how a failing onExit subprocess leaves it. + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return + + # Note: the environment script's own EXPR `let` bindings (RFC 0005) + # are evaluated by the script runner (after embedded-file path + # allocation); the wrap-interception branch below evaluates them + # itself before resolving the wrapped onExit. + # Sets the subprocess running. # Returns immediately after it has started, or is running self._action_state = ActionState.RUNNING @@ -774,18 +1013,61 @@ def exit_environment( # so it's important to set the action_state to RUNNING before calling exit(), rather # than after -- exit() itself may end up setting the action state to FAILED. - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # RFC 0008: an outer environment's onWrapEnvExit intercepts an + # inner environment's onExit. The inner environment was already + # popped from ``_environments_entered`` above, so a stack search + # now only turns up genuinely-outer environments. + on_exit_action = ( + environment.script.actions.onExit if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvExit") + if on_exit_action is not None + else None ) - self._runner.exit() + + if wrap_env is not None: + # See the onWrapEnvEnter path (_try_inject_wrapped_symbols). On + # failure the environment was already removed from tracking + # above, matching how a failed exit() behaves. + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( + symtab, + environment, + on_exit_action, + session_env_list=wrapped_session_env_list, + inner_symtab=inner_symtab, + ), + fail_message=( + f"Failed to resolve the wrapped onExit action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvExit" + ), + ): + return + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=symtab, + preallocated_file_records=wrap_file_records, + ) + self._runner.wrap_env_exit() + else: + self._runner = self._make_env_script_runner( + environment_script=environment.script, + os_env_vars=action_env_vars, + symtab=symtab, + ) + self._runner.exit() def run_task( self, @@ -794,6 +1076,7 @@ def run_task( task_parameter_values: TaskParameterSet, os_env_vars: Optional[dict[str, str]] = None, log_task_banner: bool = True, + step_name: Optional[str] = None, ) -> None: """Run a Task within the Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -812,6 +1095,8 @@ def run_task( Value: Value for the environment variable. log_task_banner (bool): Whether to log a banner before running the Task. Default: True + step_name (Optional[str]): The name of the step whose task is being run. + Used by RFC 0008 to populate ``WrappedStep.Name`` in wrap hooks. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") @@ -832,27 +1117,88 @@ def run_task( self._reset_action_state() symtab = self._symbol_table(step_script.revision, task_parameter_values) + # RFC 0005; Template Schemas §7.3.1 (EXPR): the running step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. + if step_name is not None: + symtab["Step.Name"] = step_name action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) - self._runner = StepScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - script=step_script, - symtab=symtab, - session_files_directory=self.files_directory, - ) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling run(), rather - # than after -- run() itself may end up setting the action state to FAILED. - self._runner.run() + + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated + # by the script runner, after embedded-file paths are allocated (so + # bindings can reference Task.File.*). The wrap-interception branch + # below evaluates them itself before resolving the wrapped onRun. + + # Check if any active environment defines onWrapTaskRun. + # If so, inject WrappedAction.* into the symbol table and run the + # wrap action instead of the step script's onRun (RFC 0008). + wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") + if wrap_env is not None: + if step_name is None: + # RFC 0008: without a step name, {{WrappedStep.Name}} + # renders as the empty string in the wrap script. Callers + # predating the step_name kwarg won't pass it; make the + # gap visible rather than silently rendering empty. + self._logger.warning( + "A wrap environment is active but run_task() was not given a " + "step_name; WrappedStep.Name will render as an empty string." + ) + # The wrapped onRun resolves against the STEP's own scope; the + # hook resolves against `symtab`, and the wrap environment's own + # lets/files are evaluated by the script runner from + # wrap_env.script (see _try_inject_wrapped_symbols). + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.STEP, + inner_script=step_script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_task_symbols( + symtab, step_script, step_name or "", inner_symtab=inner_symtab + ), + fail_message=( + f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " + "onWrapTaskRun" + ), + ): + return + + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=symtab, + preallocated_file_records=wrap_file_records, + ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.wrap_task_run() + else: + # Original path: run the step script directly. + self._runner = StepScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + script=step_script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + # Sets the subprocess running. + # Returns immediately after it has started, or is running + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), + # so it's important to set the action_state to RUNNING before calling run(), rather + # than after -- run() itself may end up setting the action state to FAILED. + self._runner.run() def _run_task_without_session_env( self, @@ -891,6 +1237,10 @@ def _run_task_without_session_env( action_env_vars.update(**os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) + + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated + # by the script runner, after embedded-file paths are allocated. + self._runner = StepScriptRunner( logger=self._logger, user=self._user, @@ -1059,39 +1409,420 @@ def _symbol_table( ) -> SymbolTable: """Construct a SymbolTable, with fully qualified value names, suitable for running a Script.""" - def processed_parameter_value(param: ParameterValue) -> str: - if param.type == ParameterValueType.PATH and self._path_mapping_rules is not None: + def apply_mapping(path: str) -> str: + if self._path_mapping_rules is not None: # Apply path mapping rules in the order given until one does a replacement for rule in self._path_mapping_rules: - changed, result = rule.apply(path=param.value) + changed, result = rule.apply(path=path) if changed: return result + return path + + def processed_parameter_value(param: ParameterValue) -> Any: + if param.type == ParameterValueType.PATH: + return apply_mapping(param.value) + if param.type == ParameterValueType.LIST_PATH and isinstance(param.value, list): + # openjd-rs maps each element of a LIST[PATH] parameter at + # session scope; mirror that element-wise mapping. + return [apply_mapping(p) for p in param.value] return param.value + def record_expr_types( + expr_types: dict[str, str], key: str, raw_key: str, ptype: ParameterValueType + ) -> None: + """Record the EXPR types for a parameter's ``Param.*``/``RawParam.*`` + (or ``Task.Param.*``/``Task.RawParam.*``) symbols, mirroring the + openjd-rs session-scope symbol table: + + - ``Param.*`` carries the declared type (a PATH is a host-format + path value with path mapping applied). + - ``RawParam.*`` carries the raw string form: PATH stays a plain + string and LIST[PATH] is a LIST[STRING]. + - CHUNK[INT] range strings stay strings (engine inference). + + The types only affect EXPR-extension expression evaluation; the + legacy (non-EXPR) interpolation path ignores them. + """ + if ptype == ParameterValueType.CHUNK_INT: + return + expr_types[key] = ptype.value + if ptype == ParameterValueType.PATH: + pass # raw form stays an (inferred) plain string + elif ptype == ParameterValueType.LIST_PATH: + expr_types[raw_key] = ParameterValueType.LIST_STRING.value + else: + expr_types[raw_key] = ptype.value + if version == SpecificationRevision.v2023_09: symtab = SymbolTable() - symtab[ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value] = str( - self.working_directory - ) + # The session is host scope: enable EXPR host-context functions + # (e.g. apply_path_mapping) with this session's rules. + symtab.expr_host_rules = self._expr_host_rules + working_dir_key = ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value + symtab[working_dir_key] = str(self.working_directory) + # Session.WorkingDirectory is a host-format path value in openjd-rs. + symtab.expr_types[working_dir_key] = ParameterValueType.PATH.value + # RFC 0005; Template Schemas §7.3.1 (EXPR): the job's resolved name. Only templates + # declaring EXPR pass validation referencing Job.Name, so seeding + # it whenever known does not change non-EXPR behavior. + if self._job_name is not None: + symtab["Job.Name"] = self._job_name for param_name, param_props in self._job_parameter_values.items(): - symtab[ + raw_key = ( f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + ) + key = f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) if task_parameter_values: for param_name, param_props in task_parameter_values.items(): - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + raw_key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" + key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) return symtab else: raise NotImplementedError(f"Schema version {str(version.value)} is not supported.") + def _build_expr_host_rules(self) -> Optional[list[Any]]: + """Convert this session's path mapping rules to their engine + (``openjd.expr.PathMappingRule``) form for EXPR host-context + evaluation. Returns an empty list when the session has no rules (the + session is still host scope), or ``None`` when the engine bindings + are unavailable (pre-EXPR openjd-model).""" + try: + from openjd.expr import PathFormat as ExprPathFormat + from openjd.expr import PathMappingRule as ExprPathMappingRule + except ImportError: + return None + rules = [] + for rule in self._path_mapping_rules or []: + rules.append( + ExprPathMappingRule( + source_path_format=getattr(ExprPathFormat, rule.source_path_format.name), + source_path=str(rule.source_path), + destination_path=str(rule.destination_path), + ) + ) + return rules + + # ------------------------------------------------------------------ + # RFC 0008 wrap-action helpers + # ------------------------------------------------------------------ + + _WRAP_HOOK_NAMES = WRAP_HOOK_ACTION_NAMES + + def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: + """Walk the environment stack (innermost first) and return the + active wrapping environment for ``hook`` (one of ``onWrapEnvEnter``, + ``onWrapTaskRun``, or ``onWrapEnvExit``). + + Per RFC 0008 the session is only valid with at most one + wrap-defining environment in the stack, so the first match is + always the one that applies.""" + if hook not in self._WRAP_HOOK_NAMES: + raise ValueError(f"Unknown wrap hook name: {hook}") + for env_id in reversed(self._environments_entered): + env = self._environments[env_id] + if ( + env.script is not None + and hasattr(env.script.actions, hook) + and getattr(env.script.actions, hook) is not None + ): + return env + return None + + def _get_wrap_env_file_records(self, wrap_env: EnvironmentModel) -> Optional[list[_FileRecord]]: + """Return the wrap environment's embedded-file records, allocating + their on-disk paths on first use and reusing them for every + subsequent wrap-hook invocation (see ``_wrap_env_file_records``). + Returns ``None`` when the wrap environment has no embedded files. + + The allocation only reserves the paths (defining the symbols into a + throwaway table); the per-invocation symbol definitions and content + writes happen in the runner against that invocation's symbol table. + + Raises: + RuntimeError: if a file path could not be allocated. + """ + if wrap_env.script is None or wrap_env.script.embeddedFiles is None: + return None + # The wrap env's identifier: it is in the entered stack (that's how + # _find_wrap_environment found it). + identifier = next( + ( + env_id + for env_id in self._environments_entered + if self._environments[env_id] is wrap_env + ), + None, + ) + if identifier is None: # pragma: no cover - guarded by _find_wrap_environment + raise RuntimeError( + f"Wrap environment '{wrap_env.name}' is not in this Session's entered stack." + ) + records = self._wrap_env_file_records.get(identifier) + if records is None: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=EmbeddedFilesScope.ENV, + session_files_directory=self.files_directory, + user=self._user, + ) + # Paths only: symbols are defined (and logged) per wrap-hook + # invocation via register_file_paths, against that invocation's + # own symbol table. + records = file_writer.allocate_records(wrap_env.script.embeddedFiles) + self._wrap_env_file_records[identifier] = records + return records + + def _environment_defines_any_wrap_hook(self, env: EnvironmentModel) -> bool: + """``True`` iff ``env``'s script declares any of the three wrap + hooks. Used by the single-wrap-layer validation in + :meth:`enter_environment`.""" + if env.script is None: + return False + return any( + hasattr(env.script.actions, name) and getattr(env.script.actions, name) is not None + for name in self._WRAP_HOOK_NAMES + ) + + def _build_wrapped_inner_scope( + self, + scope: EmbeddedFilesScope, + let_bindings: Optional[list[str]], + embedded_files: Optional[Any], + base: SymbolTable, + ) -> SymbolTable: + """Build the scope a wrapped action would have resolved against had + it run unwrapped: a copy of ``base`` (the session-scope table) plus + the inner entity's embedded files and script-level ``let`` bindings, + in the same order the runners use (allocate file paths → evaluate + lets → write file contents, so a file's ``data`` can reference + let-bound values and a binding can reference ``*.File.*``). + + ``WrappedAction.*`` values are resolved against this table so that + names defined by the WRAPPING environment (its ``let`` bindings) can + never leak into the wrapped action's resolved command/args — and, + symmetrically, the inner entity's lets never apply to the hook's own + resolution scope. Mirrors openjd-rs's ``build_wrapped_inner_scope``. + + Raises: + ValueError (FormatStringError/ExpressionError): a binding or file + reference did not resolve. + RuntimeError: an embedded file could not be written to disk. + """ + symtab = SymbolTable(source=base) + if embedded_files: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=scope, + session_files_directory=self.files_directory, + user=self._user, + ) + records = file_writer.allocate_file_paths(embedded_files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + elif let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + return symtab + + def _try_inject_wrapped_symbols( + self, + *, + scope: EmbeddedFilesScope, + inner_script: Any, + symtab: SymbolTable, + inject: Callable[[SymbolTable], None], + fail_message: str, + ) -> bool: + """Common wrap-interception step for the three RFC 0008 hooks: build + the wrapped (inner) entity's own resolution scope — its ``let`` + bindings and embedded files on a COPY of the session table, so the + hook's own scope never sees them (openjd-rs #277) — and call + ``inject`` with it to populate the ``WrappedAction.*`` symbols in + ``symtab``, the hook's scope. + + On failure (e.g. a binding or embedded file did not resolve or + write, or a FEATURE_BUNDLE_1 timeout/notifyPeriod format string did + not resolve to an integer) the action fails through the normal + failure path (:meth:`_fail_action_before_start` with + ``fail_message``) rather than raising out of the public API, and + ``False`` is returned so the caller can bail out.""" + try: + inner_symtab = self._build_wrapped_inner_scope( + scope, + inner_script.let if inner_script is not None else None, + inner_script.embeddedFiles if inner_script is not None else None, + symtab, + ) + inject(inner_symtab) + except (FormatStringError, ValueError, RuntimeError) as e: + self._fail_action_before_start(f"{fail_message}: {e}") + return False + return True + + def _collect_session_env_list(self) -> list[str]: + """Collect all session-defined variables across the active + environment stack as ``["KEY=value", ...]`` for + ``WrappedAction.Environment``. + + Session-defined variables are ``openjd_env`` stdout definitions + *and* entered environments' declarative ``variables:`` maps + (openjd-rs #277); host-inherited variables remain intentionally + excluded per RFC 0008. + + The per-environment changes are flattened cumulatively, in + environment-entry order, so the list reflects the *effective* + state exactly like the real subprocess environment does: a later + set overrides an earlier value for the same name (one entry, not + two), and a later unset removes the name entirely. This mirrors + the Rust runtime's single cumulative ``env_vars`` map.""" + effective: dict[str, Optional[str]] = {} + for env_id in self._environments_entered: + if env_id in self._created_env_vars: + changes = self._created_env_vars[env_id] + # effective_items() is insertion-ordered, holding both the + # environment's declarative `variables:` seed and any + # openjd_env sets/unsets, so the list order is deterministic. + for key, value in changes.effective_items().items(): + effective[key] = value + return [f"{key}={value}" for key, value in effective.items() if value is not None] + + def _resolve_action_timeout(self, action: Any, symtab: SymbolTable) -> Optional[int]: + """Return the wrapped action's timeout as an int (seconds), or + ``None`` if none was specified — ``WrappedAction.Timeout`` is typed + ``int?`` (RFC 0008), following the EXPR semantics for optional + data, so whole-field forwarding + (``timeout: "{{WrappedAction.Timeout}}"``) drops the field when the + wrapped action has no timeout. + + A resolved format-string value must be a positive integer, matching + the openjd-rs runtime and the enforcement path in + :meth:`ScriptRunnerBase._run_action` (a non-positive value raises + ``ValueError``, failing the action through the caller's normal + failure path).""" + return resolve_optional_int_field(action.timeout, symtab, ge=1, description="timeout") + + def _inject_wrapped_cancelation_symbols( + self, symtab: SymbolTable, action: Any, *, is_task_run: bool, inner_symtab: SymbolTable + ) -> None: + """Populate ``WrappedAction.Cancelation.Mode`` and + ``WrappedAction.Cancelation.NotifyPeriodInSeconds`` from the wrapped + action's ```` (RFC 0008 follow-up, + openjd-specifications#148). + + ``Mode`` is typed ``string?``: ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (rendering as + ``null``/empty in format strings) when the wrapped action defines + no ```` — the null case is deliberately distinct from + an explicit ``TERMINATE`` so wrap scripts can tell "author declared + TERMINATE" apart from "author declared nothing". + + ``NotifyPeriodInSeconds`` is typed ``int?``: the effective grace + period when the mode is ``NOTIFY_THEN_TERMINATE``, applying the + Template Schemas 5.3.2 defaults (120 seconds for a task's ``onRun``, + 30 otherwise) when the wrapped action omits the field — i.e. the + value the runtime would have enforced in the unwrapped case. It is + ``None`` (rendering as ``null``/empty in format strings) when the + mode is ``TERMINATE`` or no ```` is defined, so "not + applicable" is not conflated with a zero-length notify period. + """ + # Resolution goes through the same helper the enforcement path uses + # (resolve_effective_cancelation) — including a wrapped action + # whose own mode is deferred (a format string) — so the value a + # wrap script sees is always the value the runtime would enforce. + cancelation = getattr(action, "cancelation", None) + mode, notify_period = resolve_effective_cancelation(cancelation, inner_symtab) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value and notify_period is None: + notify_period = ( + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS + if is_task_run + else ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS + ) + symtab["WrappedAction.Cancelation.Mode"] = mode + symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] = notify_period + + def _inject_wrapped_env_symbols( + self, + symtab: SymbolTable, + environment: EnvironmentModel, + inner_action: Any, + session_env_list: Optional[list[str]] = None, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedEnv.Name`` for + ``onWrapEnvEnter`` / ``onWrapEnvExit`` scripts (RFC 0008). + + The wrapped action's command/args/timeout/cancelation resolve + against ``inner_symtab`` — the scope the action would have used had + it run unwrapped (the inner environment's own lets and embedded + files) — while the results are written into ``symtab``, the hook's + own scope. Keeping the two apart means a wrapper-defined name can + never leak into the wrapped action's resolved values, and vice + versa (openjd-rs #277). + + ``session_env_list`` overrides the collected session env list when + the caller must capture it at a different point in time — the + ``onWrapEnvExit`` path collects it before the exiting environment + is removed from tracking, so the wrapped environment's own + variables are included.""" + command = inner_action.command.resolve(symtab=inner_symtab) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path + # (ScriptRunnerBase._run_action) so the hook sees exactly the + # arguments the wrapped action would have run with unwrapped — + # mirroring openjd-rs's seed_wrapped_action_symbols, which resolves + # via the same resolve_action_args as the runner. + args = resolve_action_arg_values(inner_action.args, inner_symtab) + symtab["WrappedAction.Command"] = command + symtab["WrappedAction.Args"] = args + symtab["WrappedAction.Environment"] = ( + session_env_list if session_env_list is not None else self._collect_session_env_list() + ) + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(inner_action, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, inner_action, is_task_run=False, inner_symtab=inner_symtab + ) + symtab["WrappedEnv.Name"] = environment.name + + def _inject_wrapped_task_symbols( + self, + symtab: SymbolTable, + step_script: StepScriptModel, + step_name: str, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedStep.Name`` for + ``onWrapTaskRun`` (RFC 0008). + + Resolves the step script's ``onRun`` command and args format + strings against ``inner_symtab`` — the step's own scope (its lets + and embedded files), see ``_inject_wrapped_env_symbols`` — producing + concrete values the wrap action can safely shell-quote with + ``repr_sh()``. ``WrappedAction.Args`` is stored as a native + ``list[str]`` so that ``repr_sh(WrappedAction.Args)`` quotes each + element individually.""" + assert isinstance(step_script, StepScript_2023_09) + on_run = step_script.actions.onRun + + symtab["WrappedAction.Command"] = on_run.command.resolve(symtab=inner_symtab) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path — see + # _inject_wrapped_env_symbols. + symtab["WrappedAction.Args"] = resolve_action_arg_values(on_run.args, inner_symtab) + symtab["WrappedAction.Environment"] = self._collect_session_env_list() + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(on_run, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, on_run, is_task_run=True, inner_symtab=inner_symtab + ) + symtab["WrappedStep.Name"] = step_name + def _openjd_session_root_dir(self) -> Path: """ Returns (and creates if necessary) the top-level directory where Open Job Description step session @@ -1173,11 +1904,21 @@ def _materialize_path_mapping( else: rules_dict = dict() symtab[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = "false" + # RFC 0005; Template Schemas §7.3: for EXPR evaluation Session.HasPathMappingRules is a + # boolean and Session.PathMappingRulesFile is a path, matching + # openjd-rs's typed session symbols. The legacy (non-EXPR) + # interpolation path ignores these types and keeps the string forms. + symtab.expr_types[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = ( + ParameterValueType.BOOL.value + ) rules_json = json.dumps(rules_dict) file_handle, filename = mkstemp(dir=self.working_directory, suffix=".json", text=True) os.close(file_handle) write_file_for_user(Path(filename), rules_json, self._user) symtab[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = str(filename) + symtab.expr_types[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = ( + ParameterValueType.PATH.value + ) def _resolve_env_variable_format_strings( self, symtab: SymbolTable, variables: "EnvironmentVariableObject" @@ -1270,6 +2011,26 @@ def _action_log_filter_callback( assert action_status is not None self._callback(self._session_id, action_status) + def _fail_action_before_start(self, message: str) -> None: + """Mark the pending action as FAILED before any runner/subprocess + exists (RFC 0008: e.g. when resolving the wrapped action's format + strings for ``WrappedAction.*`` injection fails). + + Mirrors the failure branch of :meth:`_action_callback` — the + session transitions to READY_ENDING so the caller can exit the + entered environments — but does not require ``self._runner``. + """ + self._logger.error(message) + self._action_fail_message = message + self._action_exit_code = None + self._action_state = ActionState.FAILED + self._state = SessionState.READY_ENDING + if self._callback: + action_status = self.action_status + # for the type checker + assert action_status is not None + self._callback(self._session_id, action_status) + def _action_callback(self, state: ActionState) -> None: """This callback is invoked: 1. When the Action process is successfully started, by the same thread that is running the diff --git a/src/openjd/sessions/_types.py b/src/openjd/sessions/_types.py index db62db9d..310a766e 100644 --- a/src/openjd/sessions/_types.py +++ b/src/openjd/sessions/_types.py @@ -20,6 +20,14 @@ EnvironmentModel = Environment_2023_09 EnvironmentScriptModel = EnvironmentScript_2023_09 +# Default notifyPeriodInSeconds for a NOTIFY_THEN_TERMINATE cancelation +# when the action omits the field (2023-09 Template Schemas 5.3.2). +TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS = 120 +"""Default notify period for a Step Script's onRun action.""" +ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS = 30 +"""Default notify period for any other action (e.g. an Environment's +onEnter/onExit).""" + class ActionState(str, Enum): RUNNING = "running" diff --git a/test/openjd/sessions_v0/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py index 586f08e6..0c046019 100644 --- a/test/openjd/sessions_v0/test_runner_env_script.py +++ b/test/openjd/sessions_v0/test_runner_env_script.py @@ -35,13 +35,13 @@ DataString as DataString_2023_09, ) from openjd.sessions import ActionState -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_env_script import ( +from openjd.sessions._runner_base import ( CancelMethod, - EnvironmentScriptRunner, NotifyCancelMethod, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_env_script import EnvironmentScriptRunner from .conftest import build_logger, collect_queue_messages @@ -387,7 +387,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(EnvironmentScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(EnvironmentScriptRunner, "_run"): with patch.object(EnvironmentScriptRunner, "_cancel") as mock_cancel: # GIVEN script = EnvironmentScript_2023_09( @@ -468,6 +470,7 @@ def test_run_env_action_passes_default_timeout( action, symtab, default_timeout=default_timeout, + default_notify_period_seconds=30, ) def test_exit_uses_default_timeout( diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index 2b258adf..6e4c780b 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -35,13 +35,13 @@ from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from openjd.sessions import WindowsSessionUser -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_step_script import ( +from openjd.sessions._runner_base import ( CancelMethod, NotifyCancelMethod, - StepScriptRunner, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_step_script import StepScriptRunner from openjd.sessions._tempdir import TempDir from openjd.sessions._os_checker import is_posix, is_windows @@ -252,7 +252,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(StepScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(StepScriptRunner, "_run"): with patch.object(StepScriptRunner, "_cancel") as mock_cancel: # GIVEN script = StepScript_2023_09( diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py new file mode 100644 index 00000000..40575053 --- /dev/null +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -0,0 +1,345 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for the Session-level handling of EXPR ``let`` +bindings (RFC 0005): + +- a failing ``extra_let_bindings`` entry on ``enter_environment`` / + ``exit_environment`` fails the action through the normal callback path + instead of raising out of the public API; +- ``enter_environment(step_name=...)`` seeds ``Step.Name`` so step-level + bindings and the environment's actions can reference it (on both the + enter and exit sides); +- binding-RHS parsing is memoized across applications; +- the unified optional int-or-format-string field resolver + (``resolve_optional_int_field``) enforces consistent bounds. +""" + +from __future__ import annotations + +import time +import uuid +from typing import Any + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + ModelParsingContext as ModelParsingContext_2023_09, +) +from openjd.model._let_bindings import _parse_rhs +from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.model.v2023_09 import StepScript as StepScript_2023_09 +from openjd.sessions._runner_base import ( + apply_let_bindings, + resolve_action_arg_values, + resolve_optional_int_field, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _format_string_field(raw: str) -> Any: + """Build a FormatString-typed ``timeout`` field value (FEATURE_BUNDLE_1).""" + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate({"command": "echo", "timeout": raw}, context=context) + return action.timeout + + +# --------------------------------------------------------------------------- +# A failing extra `let` binding must FAIL the action via the callback path, +# never raise out of enter_environment()/exit_environment(). +# --------------------------------------------------------------------------- + + +class TestExtraLetBindingFailure: + def test_enter_environment_failing_binding_fails_action_cleanly(self) -> None: + # GIVEN: an extra binding referencing an undefined symbol. + callback_events: list[ActionStatus] = [] + + def callback(session_id: str, status: ActionStatus) -> None: + callback_events.append(status) + + env = _env("Env", onEnter=_action("true"), onExit=_action("true")) + with Session( + session_id=uuid.uuid4().hex, job_parameter_values={}, callback=callback + ) as session: + # WHEN: entering must not raise. + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = NoSuchSymbol"], + ) + _run_until_ready(session) + + # THEN: the action failed cleanly, the callback fired, and the + # environment remains entered-but-failed (exactly as a failing + # onEnter subprocess leaves it) so cleanup can exit it. + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "let" in status.fail_message + assert callback_events and callback_events[-1].state == ActionState.FAILED + assert identifier in session.environments_entered + + # WHEN: exiting the failed environment re-applies the failing + # bindings — the exit action must also fail cleanly, not raise. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert identifier not in session.environments_entered + + +# --------------------------------------------------------------------------- +# enter_environment(step_name=...) seeds Step.Name (RFC 0005 EXPR), for both +# the enter side and the re-applied bindings on the exit side. +# --------------------------------------------------------------------------- + + +class TestEnterEnvironmentStepName: + def test_step_name_resolvable_in_bindings_and_actions( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step-level binding referencing Step.Name, echoed by both + # the environment's onEnter and onExit actions. + env = _env( + "StepEnv", + onEnter=_action("echo", "enter:{{ msg }}"), + onExit=_action("echo", "exit:{{ msg }}"), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = 'step is ' + Step.Name"], + step_name="MyStep", + ) + _run_until_ready(session) + + # THEN: the enter action ran with the binding resolved. + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("enter:step is MyStep" in m for m in caplog.messages) + + # WHEN: the exit re-applies the bindings — Step.Name must be + # re-seeded so onExit resolves in the same scope as onEnter. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("exit:step is MyStep" in m for m in caplog.messages) + + +# --------------------------------------------------------------------------- +# Binding-RHS parsing is memoized: re-applying the same bindings (per task, +# per env enter/exit) must not re-parse through the engine each time. +# --------------------------------------------------------------------------- + + +class TestLetBindingParseMemoization: + def test_parse_is_cached_across_applications(self) -> None: + # GIVEN: apply_let_bindings delegates to the model's single-sourced + # evaluator, whose RHS parse (_parse_rhs) is memoized. + _parse_rhs.cache_clear() + bindings = ["a = 1 + 2", "b = a * 10"] + + # WHEN: the same bindings apply against several symbol tables. + for _ in range(5): + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=bindings) + # THEN: per-application evaluation is unchanged. (Values are the + # engine's typed results; compare via their string rendering.) + assert str(symtab["a"]) == "3" + assert str(symtab["b"]) == "30" + + # THEN: each unique RHS was parsed exactly once. + info = _parse_rhs.cache_info() + assert info.misses == len(bindings) + assert info.hits == 4 * len(bindings) + + def test_cached_expression_evaluates_against_per_call_symtab(self) -> None: + # The memoized expression object must hold no symbol-table state. + _parse_rhs.cache_clear() + first = SymbolTable() + first["X"] = 1 + second = SymbolTable() + second["X"] = 41 + apply_let_bindings(symtab=first, let_bindings=["y = X + 1"]) + apply_let_bindings(symtab=second, let_bindings=["y = X + 1"]) + assert str(first["y"]) == "2" + assert str(second["y"]) == "42" + + +# --------------------------------------------------------------------------- +# A let-bound LIST keeps its engine type through the symbol table and +# flattens through whole-field argument resolution (RFC 0005 §1.3.2) — +# regression for the typed round trip of non-string binding values. +# --------------------------------------------------------------------------- + + +class TestLetBoundListInArgs: + def test_let_bound_list_flattens_into_args(self) -> None: + # GIVEN: a step script whose `let` binds a list and whose onRun + # consumes it as a whole-field argument expression. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "let": ["files = ['alpha beta', 'gamma']"], + "actions": { + "onRun": { + "command": "echo", + "args": ["front", "{{ files }}", "back"], + } + }, + }, + context=context, + ) + + # WHEN: the bindings are applied the way the runner applies them, + # and the args resolve through the shared enforcement helper. + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=script.let or []) + resolved = resolve_action_arg_values(script.actions.onRun.args, symtab) + + # THEN: the stored value survived the engine symbol-table build as a + # typed list — flattened inline, one argument per element, embedded + # whitespace preserved — not rendered as a single stringified list. + assert resolved == ["front", "alpha beta", "gamma", "back"] + + +# --------------------------------------------------------------------------- +# resolve_optional_int_field: one implementation for the three +# "optional int-or-format-string" call sites, with consistent bounds. +# --------------------------------------------------------------------------- + + +class TestResolveOptionalIntField: + def test_none_field_stays_none(self) -> None: + assert resolve_optional_int_field(None, SymbolTable(), ge=1, description="timeout") is None + + def test_literal_int_passes_through(self) -> None: + # Literal values were bounds-checked by the static validator at + # parse time; they pass through unchecked at run time. + assert resolve_optional_int_field(30, SymbolTable(), ge=1, description="timeout") == 30 + + def test_whole_field_null_resolves_to_none(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = None + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") is None + + def test_whole_field_empty_string_rejected(self) -> None: + # A genuine empty STRING is not null (openjd-rs parity: only an + # ExprValue::Null result means "field omitted"; an empty string + # falls through to the integer parse and errors). + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "" + with pytest.raises(ValueError, match="timeout must be a positive integer, got ''"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_resolved_value_below_ge_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 0 + with pytest.raises(ValueError, match="timeout must be a positive integer, got '0'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_resolved_value_above_le_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 700 + with pytest.raises( + ValueError, match="notifyPeriodInSeconds must be between 1 and 600, got '700'" + ): + resolve_optional_int_field( + field, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) + + @pytest.mark.parametrize( + "value", + [ + pytest.param(" 45 ", id="surrounding whitespace"), + pytest.param("1_0", id="digit-group underscore"), + pytest.param("\u0661\u0662\u0663", id="non-ascii decimal digits"), + ], + ) + def test_lenient_int_spellings_rejected(self, value: str) -> None: + # Strict ASCII integer grammar (openjd-rs parity): Python's int() + # accepts all of these spellings, but Rust's str::parse rejects + # them — accepting them here would be a spec-observable divergence + # for dynamically resolved timeout/notifyPeriodInSeconds values. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = value + with pytest.raises(ValueError, match="timeout must be a positive integer"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_leading_plus_sign_accepted(self) -> None: + # Rust's u64/i64 from_str accepts a leading '+'; so do we. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "+45" + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") == 45 + + def test_non_integer_resolved_value_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "abc" + with pytest.raises(ValueError, match="timeout must be a positive integer, got 'abc'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_session_resolve_action_timeout_rejects_non_positive(self) -> None: + # Regression: Session._resolve_action_timeout previously accepted + # any int() — a resolved non-positive timeout must now be rejected, + # matching the openjd-rs runtime and the enforcement path in + # ScriptRunnerBase._run_action. + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate( + {"command": "echo", "timeout": "{{ X }}"}, context=context + ) + symtab = SymbolTable() + symtab["X"] = 0 + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + with pytest.raises(ValueError, match="timeout must be a positive integer"): + session._resolve_action_timeout(action, symtab) diff --git a/test/openjd/sessions_v0/test_wrap_actions.py b/test/openjd/sessions_v0/test_wrap_actions.py new file mode 100644 index 00000000..f01bca7b --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_actions.py @@ -0,0 +1,685 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""End-to-end tests for the three RFC 0008 wrap hooks +(``onWrapEnvEnter``, ``onWrapTaskRun``, ``onWrapEnvExit``) and the +single-wrap-layer validation. + +The tests use the marker-file pattern from the RFC: each action +appends a tagged line to a shared file in the session working +directory, and the final contents prove which actions ran via which +path. No containers required — ``echo`` and ``cat`` only. +""" + +from __future__ import annotations + +import time +import uuid +from pathlib import Path + +import pytest + +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + """Build an environment whose script defines whichever hooks the + caller passes (e.g. ``onEnter=…``, ``onWrapTaskRun=…``).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _step(command: str, *args: str) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=_action(command, *args))) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _trace_path(session: Session) -> Path: + return Path(str(session.working_directory)) / "trace.log" + + +_NOOP = _action("true") + + +# --------------------------------------------------------------------------- +# Single-wrap-layer enforcement (RFC 0008: at most one wrap-defining +# environment in the session stack). +# --------------------------------------------------------------------------- + + +class TestSingleWrapLayer: + def test_two_wrap_envs_rejected_at_enter(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + inner = _env( + "inner", + onWrapEnvEnter=_action("sh", "-c", "echo inner-wrap"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + + with pytest.raises(RuntimeError, match=r"RFC 0008"): + session.enter_environment(environment=inner) + + def test_two_envs_with_only_one_wrap_layer_ok(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + # The inner env defines no wrap hooks — fine to enter. + inner = _env("inner", onEnter=_action("sh", "-c", "echo inner-onEnter")) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + assert session.state == SessionState.READY + + +# --------------------------------------------------------------------------- +# onWrapEnvEnter intercepts inner onEnter (RFC Test 1). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEnter: + def test_wrap_env_enter_intercepts_inner_on_enter(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onEnter' >> {trace} && " + f"echo 'inner-onEnter body' >> {trace}", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), + ) + inner = _env( + "inner", + onEnter=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + ) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + assert trace.exists() + content = trace.read_text() + # The outer env's own onEnter ran on host. + assert "outer-onEnter" in content + # The wrap script ran in place of the inner onEnter. + assert "[WRAPPED] inner-onEnter" in content + assert "inner-onEnter body" in content + # The inner env's host-side onEnter did NOT run. + assert "should not run on host" not in content + + def test_wrap_env_enter_receives_wrapped_symbols(self) -> None: + """``WrappedEnv.Name`` and ``WrappedAction.Command`` resolve to the + inner env's identity inside the wrap script.""" + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + "echo " + "'name={{WrappedEnv.Name}} cmd={{WrappedAction.Command}}' " + f">> {trace}", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = _env("inner-env-name", onEnter=_action("echo", "hello")) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + content = trace.read_text() + assert "name=inner-env-name" in content + assert "cmd=echo" in content + + +# --------------------------------------------------------------------------- +# onWrapEnvExit intercepts inner onExit (RFC Test 2). +# --------------------------------------------------------------------------- + + +class TestWrapEnvExit: + def test_wrap_env_exit_intercepts_inner_on_exit(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onExit' >> {trace}", + ), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + ) + inner = _env( + "inner", + onExit=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + ) + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + content = trace.read_text() + assert "[WRAPPED] inner-onExit" in content + assert "should not run on host" not in content + + # The outer env's own onExit must still run on host when we + # eventually exit it. + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + content = trace.read_text() + assert "outer-onExit" in content + + +# --------------------------------------------------------------------------- +# Visible end-to-end ordering across all three hooks (RFC Test 5). +# --------------------------------------------------------------------------- + + +class TestVisibleOrdering: + def test_full_ordering_across_three_hooks(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), + onWrapEnvEnter=_action("sh", "-c", f"echo '[WRAPPED] inner-onEnter' >> {trace}"), + onWrapTaskRun=_action("sh", "-c", f"echo '[WRAPPED] task-onRun' >> {trace}"), + onWrapEnvExit=_action("sh", "-c", f"echo '[WRAPPED] inner-onExit' >> {trace}"), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + ) + wrapped_inner = _env( + "wrapped-inner", + onEnter=_action("sh", "-c", f"echo 'inner-onEnter body' >> {trace}"), + onExit=_action("sh", "-c", f"echo 'inner-onExit body' >> {trace}"), + ) + step = _step("echo", "task-onRun-body") + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + wrapped_id = session.enter_environment(environment=wrapped_inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + session.exit_environment(identifier=wrapped_id) + _run_until_ready(session) + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + + lines = [line for line in trace.read_text().splitlines() if line.strip()] + # The exact order matters here — RFC 0008's pass criterion. + assert lines[0] == "outer-onEnter" + assert lines[1] == "[WRAPPED] inner-onEnter" + assert lines[2] == "[WRAPPED] task-onRun" + assert lines[3] == "[WRAPPED] inner-onExit" + assert lines[4] == "outer-onExit" + + +# --------------------------------------------------------------------------- +# WrappedAction.* injection: inner embedded files, and failure handling. +# +# openjd-rs #277: the wrapped entity's embedded files ARE materialized on +# the wrap path (into the inner scope only), so a wrapped action +# referencing {{Task.File.*}} / {{Env.File.*}} resolves to a real on-disk +# path in WrappedAction.Command. Injection failures that remain possible +# (e.g. an undefined symbol) must still FAIL the action through the normal +# callback path instead of raising out of the public API, and the session +# transitions to READY_ENDING (never stuck in RUNNING). +# --------------------------------------------------------------------------- + + +def _embedded_file_step() -> StepScript_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.Foo }}")) + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Foo", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho hello\n"), + ) + ], + ) + + +def _embedded_file_env(name: str) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{ Env.File.Setup }}")), + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Setup", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho setup\n"), + ) + ], + ), + ) + + +class TestWrapInjectionFailure: + def _wrap_env(self) -> Environment_2023_09: + return _env( + "Wrapper", + onWrapEnvEnter=_action("echo", "{{WrappedAction.Command}}"), + onWrapTaskRun=_action("echo", "{{WrappedAction.Command}}"), + onWrapEnvExit=_NOOP, + ) + + def test_run_task_with_embedded_file_command_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onRun referencing {{Task.File.*}} now + # resolves — the step's embedded files are materialized into the + # inner scope, and WrappedAction.Command carries the on-disk path. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task(step_script=_embedded_file_step(), task_parameter_values={}) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + # The wrap hook echoed WrappedAction.Command: the materialized + # file path inside the session's files directory. + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + + def test_enter_environment_with_embedded_file_on_enter_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onEnter referencing {{Env.File.*}} now + # resolves against the inner environment's own materialized files. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=_embedded_file_env("Inner")) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_run_task_with_unresolvable_wrapped_symbol_fails_action(self) -> None: + # The graceful-failure contract still holds for injection failures: + # a wrapped onRun referencing an undefined symbol must FAIL the + # action via the callback path, not raise out of run_task(). + bad_step = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.DoesNotExist }}")) + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task(step_script=bad_step, task_parameter_values={}) + _run_until_ready(session) + + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "onWrapTaskRun" in status.fail_message + + +class TestWrapScopeSeparation: + """openjd-rs #277: the wrapper's and the wrapped entity's scopes must be + kept strictly apart. WrappedAction.* resolves against the INNER scope + only; the hook script resolves against the WRAP environment's own scope + (including its script-level ``let`` bindings) only.""" + + def test_same_let_name_each_side_sees_own_value(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: the wrapper and the step both bind the same `let` name + # `who` to different values. The step's onRun args reference + # {{who}}; the hook also references {{who}} directly. + wrapper = Environment_2023_09( + name="Wrapper", + script=EnvironmentScript_2023_09( + let=["who = 'wrapper-scope'"], + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action( + "sh", + "-c", + "echo HOOK-WHO={{who}} WRAPPED-ARGS={{WrappedAction.Args}}", + ), + onWrapEnvExit=_NOOP, + ), + ), + ) + step = StepScript_2023_09( + let=["who = 'step-scope'"], + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{who}}")], + ) + ), + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=wrapper) + _run_until_ready(session) + + # WHEN: the task runs through the wrap hook. + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the hook saw the WRAPPER's binding, and the wrapped + # action's args resolved with the STEP's binding. + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "HOOK-WHO=wrapper-scope" in messages, messages + # WrappedAction.Args is a list; its default rendering brackets the + # single resolved element. + assert "WRAPPED-ARGS=[step-scope]" in messages, messages + + def test_inner_env_let_does_not_leak_into_hook_scope(self) -> None: + # An inner environment's `let` binding must resolve in + # WrappedAction.* but must NOT be visible to the hook script: a + # hook referencing it directly fails the action. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo LEAKED={{inner_only}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["inner_only = 'secret'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{inner_only}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + # The hook referenced {{inner_only}}, which is not in its scope. + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_wrapped_env_enter_command_resolves_inner_let( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # The inner environment's `let` binding must be visible to + # WrappedAction.* on the onWrapEnvEnter path. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo INNERCMD={{WrappedAction.Command}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["tool = 'inner-tool'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{tool}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + messages = "\n".join(caplog.messages) + assert "INNERCMD=inner-tool" in messages, messages + + +class TestWrappedActionEnvironmentContents: + def test_variables_map_included_in_wrapped_environment(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions AND entered + # environments' declarative variables: maps. Host-inherited + # variables remain excluded. + from openjd.model.v2023_09 import ( + EnvironmentVariableValueString as EnvironmentVariableValueString_2023_09, + ) + + declaring = Environment_2023_09( + name="Declaring", + variables={ + "DECLARED_VAR": EnvironmentVariableValueString_2023_09("from-variables-map") + }, + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + session.enter_environment(environment=declaring) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert any(entry == "DECLARED_VAR=from-variables-map" for entry in env_list), env_list + + def test_later_set_overrides_earlier_value_without_duplicate(self) -> None: + # Cumulative flattening: when a later environment's openjd_env sets + # a name an earlier environment already set, the list carries only + # the effective (later) value — matching the real subprocess env + # and the Rust runtime's single cumulative map. + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + EnvironmentVariableUnsetChange, + SimplifiedEnvironmentVariableChanges, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + scenarios: list[ + tuple[str, list[EnvironmentVariableSetChange | EnvironmentVariableUnsetChange]] + ] = [ + ("env-a", [EnvironmentVariableSetChange(name="FOO", value="1")]), + ("env-b", [EnvironmentVariableSetChange(name="FOO", value="2")]), + ] + for env_id, changes in scenarios: + session._environments_entered.append(env_id) + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes(changes) + session._created_env_vars[env_id] = tracked + + assert session._collect_session_env_list() == ["FOO=2"] + + # And a later unset removes the name entirely. + session._environments_entered.append("env-c") + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes([EnvironmentVariableUnsetChange(name="FOO")]) + session._created_env_vars["env-c"] = tracked + + assert session._collect_session_env_list() == [] + + # Clear the fake stack so Session.cleanup() doesn't try to + # unwind environments that were never really entered. + session._environments_entered.clear() + session._created_env_vars.clear() + + def test_exit_path_includes_exiting_envs_openjd_env_vars(self) -> None: + # On the onWrapEnvExit path, WrappedAction.Environment must include + # the exiting environment's own openjd_env variables: the real + # subprocess env (and the unwrapped onExit) includes them, so the + # list must match. The wrapper's onWrapEnvEnter emits the + # openjd_env message, which is attributed to the inner (wrapped) + # environment being entered. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo 'openjd_env: INNER_VAR=inner-value'"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo \"ENVLIST=<{{{{WrappedAction.Environment}}}}>\" >> '{trace}'", + ), + ) + inner = _env("Inner", onEnter=_NOOP, onExit=_NOOP) + + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + contents = trace.read_text() + assert "INNER_VAR=inner-value" in contents, contents + + def test_openjd_env_vars_included_in_wrapped_environment(self) -> None: + # openjd_env-emitted variables must still be surfaced. The wrapper's + # own onEnter is never wrapped, so it can emit the message directly. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onEnter=_action("sh", "-c", "echo 'openjd_env: EMITTED_VAR=from-openjd-env'"), + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert "EMITTED_VAR=from-openjd-env" in env_list + + +class TestRunWrapHookGuard: + def test_unknown_hook_name_raises(self) -> None: + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + import logging + + from openjd.sessions._logging import LoggerAdapter + from openjd.model import SymbolTable + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + runner = EnvironmentScriptRunner( + logger=LoggerAdapter(logging.getLogger(__name__), extra={}), + session_working_directory=Path(str(session.working_directory)), + environment_script=None, + symtab=SymbolTable(), + session_files_directory=Path(str(session.files_directory)), + ) + with pytest.raises(ValueError, match="Unknown wrap hook name"): + runner._run_wrap_hook("onWrapTypo") diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py new file mode 100644 index 00000000..55327ab8 --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -0,0 +1,477 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for ``WrappedAction.Cancelation.*`` injection (RFC 0008 follow-up, +openjd-specifications#148). + +``WrappedAction.Cancelation.Mode`` is ``string?`` and carries the wrapped +action's cancelation method — ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, +or ``None`` when the wrapped action defines no ````. The null +case is deliberately distinct from an explicit ``TERMINATE``. + +``WrappedAction.Cancelation.NotifyPeriodInSeconds`` is ``int?``: the +effective grace period when the mode is ``NOTIFY_THEN_TERMINATE`` (with the +Template Schemas 5.3.2 defaults applied — 120 for a task's ``onRun``, 30 +otherwise), and ``None`` when a notify period does not apply. + +Mirrors the Rust integration coverage in openjd-rs +``tests/integration/test_wrap_actions.rs`` and the conformance fixtures +``conformance-tests/2023-09/WRAP_ACTIONS/jobs/wrap-cancelation-*``. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CancelationMethodNotifyThenTerminate as NotifyThenTerminate_2023_09, + CancelationMethodTerminate as Terminate_2023_09, + CancelationMode as CancelationMode_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + +_TERMINATE = Terminate_2023_09(mode=CancelationMode_2023_09.TERMINATE) + + +def _notify(period: int | None) -> NotifyThenTerminate_2023_09: + return NotifyThenTerminate_2023_09( + mode=CancelationMode_2023_09.NOTIFY_THEN_TERMINATE, + notifyPeriodInSeconds=period, + ) + + +def _step_script_with_cancelation(cancelation) -> StepScript_2023_09: + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("placeholder")], + cancelation=cancelation, + ) + ) + ) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _inner_env_action(cancelation) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09("inner-enter-cmd"), + args=[ArgString_2023_09("--flag")], + cancelation=cancelation, + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests — task-run injection path +# --------------------------------------------------------------------------- + + +class TestInjectTaskCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_task_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script_with_cancelation(cancelation) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + return symtab + finally: + session.cleanup() + + def test_mode_none_and_period_none_when_no_cancelation(self) -> None: + # No declared: Mode is None (NOT "TERMINATE") and the + # notify period is None — the string?/int? null values. + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_mode_terminate_and_period_none(self) -> None: + symtab = self._inject(_TERMINATE) + assert symtab["WrappedAction.Cancelation.Mode"] == "TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_notify_then_terminate_with_explicit_period(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_notify_then_terminate_defaults_to_120_for_task_on_run(self) -> None: + # Template Schemas 5.3.2: the default notify period for a task's + # onRun is 120 seconds. The runtime supplies the value it would have + # enforced in the unwrapped case. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 120 + + +# --------------------------------------------------------------------------- +# Unit tests — env enter/exit injection path +# --------------------------------------------------------------------------- + + +class TestInjectEnvCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_env_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + inner_env = Environment_2023_09( + name="InnerEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=_inner_env_action(cancelation), + ), + ), + ) + session._inject_wrapped_env_symbols( + symtab, inner_env, _inner_env_action(cancelation), inner_symtab=symtab + ) + return symtab + finally: + session.cleanup() + + def test_notify_then_terminate_defaults_to_30_for_env_action(self) -> None: + # Template Schemas 5.3.2: for anything other than a task's onRun — + # including an inner environment's onEnter — the default is 30. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 30 + + def test_explicit_period_forwards_verbatim_for_env_action(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_mode_none_when_env_action_has_no_cancelation(self) -> None: + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + +# --------------------------------------------------------------------------- +# Integration tests — real subprocess, format-string interpolation +# --------------------------------------------------------------------------- + + +class TestWrapCancelationExecution: + """End-to-end: the wrap action interpolates the Cancelation variables in + a format string. The ``<...>`` sentinel wrap makes the null-renders-empty + behavior observable (``NP=<>``), matching the conformance fixtures.""" + + _PROBE = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + 'echo "MODE=<{{WrappedAction.Cancelation.Mode}}>";' + ' echo "NP=<{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}>"' + ), + ], + ) + + def _run(self, cancelation, caplog: pytest.LogCaptureFixture) -> str: + session_id = uuid.uuid4().hex + env = _wrap_env("wrap_env", self._PROBE) + step = _step_script_with_cancelation(cancelation) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + return "\n".join(caplog.messages) + + def test_terminate_mode_renders_null_period_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_TERMINATE, caplog) + assert "MODE=" in messages + # int? null interpolates as the empty string (RFC 0005), so the + # sentinel wrap renders as NP=<> — distinct from the old 0 sentinel, + # which would have rendered NP=<0>. + assert "NP=<>" in messages + + def test_notify_then_terminate_default_renders_120( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_notify(None), caplog) + assert "MODE=" in messages + assert "NP=<120>" in messages + + def test_no_cancelation_renders_null_mode_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # No declared: Mode is null (string?), which + # interpolates as the empty string (RFC 0005) — MODE=<>, matching + # the wrap-cancelation-mode-null-when-no-cancelation conformance + # fixture. The nullness itself (vs. an empty string value) is + # asserted by the unit tests above and observable via EXPR + # null-coalescing in the conformance fixture. + messages = self._run(None, caplog) + assert "MODE=<>" in messages + assert "NP=<>" in messages + + +# --------------------------------------------------------------------------- +# Unit tests — deferred-mode resolution (resolve_effective_cancelation) +# --------------------------------------------------------------------------- + + +class TestResolveEffectiveCancelation: + """Unit tests for the shared deferred-cancelation resolution helper. + + A CancelationMethodDeferred carries a format-string mode whose + TERMINATE-vs-NOTIFY_THEN_TERMINATE decision is made at run time + against the live symbol table (see resolve_effective_cancelation's + docstring for the full story). + """ + + def _deferred(self, mode: str, period: str | None = None): + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + # A deferred mode is an RFC 0008 forwarding construct, and + # WRAP_ACTIONS requires the EXPR extension — so the format strings + # here parse as EXPR expressions, giving the typed null semantics + # the runtime relies on (a whole-field null drops the cancelation + # object; an empty STRING is an error, matching openjd-rs). + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + return CancelationMethodDeferred( + mode=FormatString(mode, context=ctx), + notifyPeriodInSeconds=( + FormatString(period, context=ctx) if period is not None else None + ), + ) + + def _symtab(self, **values) -> SymbolTable: + symtab = SymbolTable() + for key, value in values.items(): + symtab[key] = value + return symtab + + def test_mode_resolving_null_drops_whole_object(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X=None, P=None)) + assert result == (None, None) + + def test_mode_resolving_terminate(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE")) + assert result == ("TERMINATE", None) + + def test_mode_resolving_terminate_rejects_non_null_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="does not accept"): + resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE", P=45)) + + def test_mode_resolving_notify_then_terminate_with_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=45) + ) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_whole_field_mode_resolving_empty_string_raises(self) -> None: + # A genuine empty STRING is not null, even for a whole-field + # expression (openjd-rs parity: only an ExprValue::Null result + # drops the cancelation object; an empty string is an invalid + # mode). E.g. a STRING parameter whose value is "". + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to .* got ''"): + resolve_effective_cancelation(cancelation, self._symtab(X="")) + + def test_mode_resolving_garbage_raises(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X="SOMETHING_ELSE")) + + def test_partial_interpolation_mode_resolves_normally(self) -> None: + # Normal format string behavior (Template Schemas 5.3): partial + # interpolation is permitted; the resolved value is checked + # against the two mode names. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}_THEN_TERMINATE", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="NOTIFY", P=45)) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_partial_interpolation_mode_resolving_empty_raises(self) -> None: + # Null semantics (dropping the cancelation object) apply only to a + # whole-field expression. A normal format string that resolves to + # the empty string is not null — it is an invalid mode. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}{{Y}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X=None, Y=None)) + + def test_resolved_period_exceeding_cap_raises(self) -> None: + # The static validator caps literal periods at 600 (Template + # Schemas 5.3.2); format-string values could not be checked at + # parse time, so the resolved value is bounds-checked at run time. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="between 1 and 600"): + resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=9999) + ) + + +# --------------------------------------------------------------------------- +# Launch-time cancelation resolution (openjd-rs run_action parity): +# the effective cancel method is resolved by _run_action against the SAME +# final scope the command/args resolved with (script lets, *.File.*, +# WrappedAction.*) and stored on the runner; cancel() consumes it. An +# unresolvable or invalid cancelation fails the action at start. +# --------------------------------------------------------------------------- + + +class TestLaunchTimeCancelationResolution: + def _env_with_let_bound_cancelation(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["EXPR", "WRAP_ACTIONS", "FEATURE_BUNDLE_1"]) + return Environment_2023_09.model_validate( + { + "name": "Wrapper", + "script": { + "let": [ + "hookMode = 'NOTIFY_THEN_TERMINATE'", + "hookPeriod = 9", + ], + "actions": { + "onEnter": {"command": "true"}, + "onWrapEnvEnter": {"command": "true"}, + "onWrapEnvExit": {"command": "true"}, + "onWrapTaskRun": { + "command": "sleep", + "args": ["20"], + "cancelation": { + "mode": "{{hookMode}}", + "notifyPeriodInSeconds": "{{hookPeriod}}", + }, + }, + }, + }, + }, + context=ctx, + ) + + def test_let_bound_cancelation_resolved_against_final_scope(self) -> None: + # Regression: cancel() used to re-resolve the cancelation against + # the runner's BASE symtab — which lacks the script's `let` + # bindings — so a let-referencing mode fell back to Terminate with + # a warning. It must resolve at launch, in the hook's final scope. + from datetime import timedelta + from unittest.mock import MagicMock, patch + + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + from openjd.sessions._runner_base import NotifyCancelMethod + + env = self._env_with_let_bound_cancelation() + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + with patch.object(EnvironmentScriptRunner, "_run"): + runner = EnvironmentScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + environment_script=env.script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + runner.wrap_task_run() + assert runner._resolved_cancel_method == NotifyCancelMethod( + terminate_delay=timedelta(seconds=9) + ) + + def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: + # Eager validation (openjd-rs parity): a cancelation whose deferred + # mode resolves to something other than the two method names or + # null must FAIL the action at start — not launch successfully and + # only surface if a cancel later occurs. + session_id = uuid.uuid4().hex + probe = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo should-not-run")], + ) + env = _wrap_env("wrap_env", probe) + step = _step_script_with_cancelation(None) + # Forward an invalid mode through the wrap round trip: the wrap + # action's own cancelation defers to a symbol that resolves to + # garbage at run time. + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + object.__setattr__( + env.script.actions.onWrapTaskRun, + "cancelation", + CancelationMethodDeferred( + mode=FormatString("{{WrappedStep.Name}}", context=ctx), + notifyPeriodInSeconds=None, + ), + ) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}, step_name="NotAMode") + _run_until_ready(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py new file mode 100644 index 00000000..9126f36d --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -0,0 +1,558 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the ``onWrapTaskRun`` environment action (RFC 0008). + +These tests exercise the end-to-end behaviour that a job template's ``onRun`` is +intercepted and the active environment's ``onWrapTaskRun`` is executed in its +place, with ``WrappedAction.Command``, ``WrappedAction.Args``, +``WrappedAction.Environment``, ``WrappedAction.Timeout``, and +``WrappedStep.Name`` injected into the wrap action's symbol table. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _step_script(command: str, args: list[str]) -> StepScript_2023_09: + """Build a minimal StepScript that runs ``command`` with ``args``.""" + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + ) + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + """Block until the session transitions back to READY or the timeout elapses.""" + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests on the pure helpers — no subprocess needed +# --------------------------------------------------------------------------- + + +class TestInjectWrappedTaskSymbols: + """Unit tests for the ``_inject_wrapped_task_symbols`` helper.""" + + def test_injects_wrapped_command_and_args_as_list(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("python3", ["-c", "print('hi')"]) + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "python3" + assert symtab["WrappedAction.Args"] == ["-c", "print('hi')"] + assert isinstance(symtab["WrappedAction.Args"], list) + assert symtab["WrappedStep.Name"] == "MyStep" + finally: + session.cleanup() + + def test_injects_empty_args_when_step_has_no_args(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("whoami", []) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "whoami" + assert symtab["WrappedAction.Args"] == [] + finally: + session.cleanup() + + def test_injects_typed_args_null_skip_and_list_flatten(self) -> None: + # RFC 0005 §1.3.2 typed argument semantics (openjd-rs parity: the + # wrapped path in seed_wrapped_action_symbols resolves through the + # same resolve_action_args as the runner): a whole-field list + # expression flattens inline (one argument per element), a + # whole-field null is skipped, and the hook sees exactly the argv + # the wrapped action would have run with unwrapped. + from openjd.model.v2023_09 import ModelParsingContext + from openjd.sessions._runner_base import resolve_action_arg_values + + context = ModelParsingContext(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "actions": { + "onRun": { + "command": "echo", + "args": ["front", '{{ ["a", "b c"] }}', "{{ null }}", "back"], + } + } + }, + context=context, + ) + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + assert symtab["WrappedAction.Args"] == ["front", "a", "b c", "back"] + # The unwrapped enforcement path (_run_action) resolves the same + # action's args via the same shared helper — wrapped and + # unwrapped runs of this action use identical argv. + assert resolve_action_arg_values(script.actions.onRun.args, symtab) == [ + "front", + "a", + "b c", + "back", + ] + finally: + session.cleanup() + + def test_injects_wrapped_environment_as_key_value_list(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions (applied via + # simplify_ordered_changes) AND the declarative variables:-map seed. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + SimplifiedEnvironmentVariableChanges, + ) + + fake_id = "env-1" + session._environments_entered.append(fake_id) + changes = SimplifiedEnvironmentVariableChanges({"DECLARED": "from-variables-map"}) + changes.simplify_ordered_changes( + [ + EnvironmentVariableSetChange(name="FOO", value="bar"), + EnvironmentVariableSetChange(name="BAZ", value="qux"), + ] + ) + session._created_env_vars[fake_id] = changes + + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + task_env = symtab["WrappedAction.Environment"] + assert isinstance(task_env, list) + normalized = {item.upper() for item in task_env} + # Both openjd_env-set variables and the variables:-map seed are + # surfaced (openjd-rs #277). + assert normalized == { + "DECLARED=from-variables-map".upper(), + "FOO=bar".upper(), + "BAZ=qux".upper(), + } + finally: + session.cleanup() + + def test_injects_timeout_none_when_no_timeout(self) -> None: + # WrappedAction.Timeout is int? (RFC 0008): None (null) when the + # wrapped action specifies no timeout, so whole-field forwarding + # (timeout: "{{WrappedAction.Timeout}}") drops the field. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Timeout"] is None + finally: + session.cleanup() + + def test_find_wrap_environment_returns_none_when_no_env_has_wrap(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + assert session._find_wrap_environment(hook="onWrapTaskRun") is None + finally: + session.cleanup() + + +# --------------------------------------------------------------------------- +# Integration tests — actually run a subprocess wrap action +# --------------------------------------------------------------------------- + + +class TestWrapTaskRunExecution: + """Integration tests that enter an environment with ``onWrapTaskRun`` and + run a task through it, verifying the wrap action is what actually ran.""" + + def test_no_wrap_runs_original_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: a session with no environment defining onWrapTaskRun. + session_id = uuid.uuid4().hex + step = _step_script("echo", ["original-step"]) + + # WHEN: we run the task. + with Session(session_id=session_id, job_parameter_values={}) as session: + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the original step ran, not a wrap action. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + assert any("original-step" in msg for msg in caplog.messages) + + def test_wrap_action_intercepts_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: an environment that defines onWrapTaskRun. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo WRAPPED-RAN")], + ) + env = _wrap_env("container_env", wrap) + + step = _step_script("echo", ["original-step"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + # Enter the wrap env (no onEnter, nothing runs but the env becomes active). + session.enter_environment(environment=env) + _run_until_ready(session) + + # WHEN: we run a task. + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the wrap action ran, not the original step. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "WRAPPED-RAN" in messages + assert "original-step" not in messages + + def test_wrap_action_receives_wrapped_command_symbol( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a wrap action that echoes WrappedAction.Command via a format string. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo CMD={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("container_env", wrap) + step = _step_script("maya-batch", ["-render", "scene.ma"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "CMD=maya-batch" in messages + + def test_innermost_environment_wins(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: two environments — only the inner one defines onWrapTaskRun. + session_id = uuid.uuid4().hex + outer = Environment_2023_09( + name="outer", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo outer-enter"), + ], + ) + ) + ), + ) + inner = _wrap_env( + "inner", + Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo INNER-WRAP")], + ), + ) + step = _step_script("echo", ["original"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "INNER-WRAP" in messages + assert "original" not in messages + + def test_wrap_action_runs_multiple_tasks(self, caplog: pytest.LogCaptureFixture) -> None: + # Regression: symbols are re-injected cleanly per task. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo RAN={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("env", wrap) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=_step_script("cmd-one", []), task_parameter_values={}) + _run_until_ready(session) + session.run_task(step_script=_step_script("cmd-two", []), task_parameter_values={}) + _run_until_ready(session) + + messages = "\n".join(caplog.messages) + assert "RAN=cmd-one" in messages + assert "RAN=cmd-two" in messages + + +# --------------------------------------------------------------------------- +# Security and execution constraints — RFC 0008 test matrix +# --------------------------------------------------------------------------- +# +# These tests cover the 8 scenarios the RFC calls out under "Recommended test +# cases for implementation". They verify that the symbol injection layer +# preserves WrappedAction.Command and WrappedAction.Args byte-for-byte — no +# shell expansion, no quoting transformation, no truncation. + + +class TestSecurityAndExecutionConstraints: + """RFC 0008 §"Recommended test cases for implementation" matrix.""" + + def _inject(self, command: str, args: list[str]) -> tuple[str, list[str]]: + """Helper: run the injection and return the resolved WrappedAction.Command/Args.""" + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script(command, args) + session._inject_wrapped_task_symbols(symtab, script, "TestStep", inner_symtab=symtab) + return symtab["WrappedAction.Command"], symtab["WrappedAction.Args"] + finally: + session.cleanup() + + # --- 1. Nested quoting ------------------------------------------------ + + def test_nested_quoting_preserved(self) -> None: + # echo "O'Reilly's Guide" — both " and ' must survive injection. + cmd, args = self._inject("echo", ["O'Reilly's Guide"]) + assert cmd == "echo" + assert args == ["O'Reilly's Guide"] + + def test_double_quotes_preserved(self) -> None: + cmd, args = self._inject("python3", ["-c", 'print("hello world")']) + assert cmd == "python3" + assert args == ["-c", 'print("hello world")'] + + # --- 2. Shell metacharacters ------------------------------------------ + + def test_shell_metacharacters_preserved(self) -> None: + # `, $, |, &&, ;, >, <, *, ?, (, ), \, all in one arg. + nasty = "$(cat /etc/passwd); `id`; rm -rf / || echo pwned" + cmd, args = self._inject("echo", [nasty, "a|b", "c&&d", "e;f"]) + assert cmd == "echo" + assert args == [nasty, "a|b", "c&&d", "e;f"] + + def test_backticks_preserved(self) -> None: + cmd, args = self._inject("echo", ["`whoami`", "$USER"]) + assert args == ["`whoami`", "$USER"] + # The $ must NOT have been expanded by the runtime. + assert "$USER" in args + + # --- 3. Path traversal ------------------------------------------------ + + def test_path_traversal_preserved_literally(self) -> None: + # The runtime must not resolve or reject `../` paths — the wrap + # script + container boundary is what prevents escape. + cmd, args = self._inject("cat", ["../../../etc/passwd", "/tmp/../etc/shadow"]) + assert cmd == "cat" + assert args == ["../../../etc/passwd", "/tmp/../etc/shadow"] + + # --- 4. Shell globbing ------------------------------------------------ + + def test_glob_characters_preserved(self) -> None: + # ls *.txt must be passed literally — no expansion at injection time. + cmd, args = self._inject("ls", ["*.txt", "?.log", "[abc]*"]) + assert cmd == "ls" + assert args == ["*.txt", "?.log", "[abc]*"] + + # --- 5. Unicode paths ------------------------------------------------- + + def test_unicode_cjk_preserved(self) -> None: + path = "/projects/映画/シーン01/レンダー.exr" + cmd, args = self._inject("render", ["--scene", path]) + assert cmd == "render" + assert args == ["--scene", path] + assert len(path.encode("utf-8")) > len(path) + + def test_unicode_emoji_preserved(self) -> None: + cmd, args = self._inject("echo", ["🎬 render 🎥", "🔥"]) + assert args == ["🎬 render 🎥", "🔥"] + + def test_unicode_mixed_scripts_preserved(self) -> None: + cmd, args = self._inject( + "tool", + ["--русский", "--中文", "--日本語", "--한국어", "--العربية"], + ) + assert args == ["--русский", "--中文", "--日本語", "--한국어", "--العربية"] + + # --- 6. Empty and whitespace-only arguments --------------------------- + + def test_empty_string_arg_preserved(self) -> None: + cmd, args = self._inject("echo", ["before", "", "after"]) + assert cmd == "echo" + assert args == ["before", "", "after"] + assert len(args) == 3 + + def test_whitespace_only_arg_preserved(self) -> None: + cmd, args = self._inject("echo", [" ", " ", " "]) + assert args == [" ", " ", " "] + + # --- 7. Newlines in arguments ----------------------------------------- + + def test_newline_in_arg_preserved(self) -> None: + # The EXPR extension relaxes the ArgString regex to accept + # control characters; the runtime must preserve them literally. + cmd, args = self._inject("echo", ["line1\nline2"]) + assert cmd == "echo" + assert args == ["line1\nline2"] + + # --- 8. Near-limit command length ------------------------------------- + + def test_large_argument_preserved(self) -> None: + big = "x" * (100 * 1024) + cmd, args = self._inject("cat", ["--data", big]) + assert cmd == "cat" + assert args == ["--data", big] + assert len(args[1]) == 100 * 1024 + + def test_many_arguments_preserved(self) -> None: + many = [f"arg{i}" for i in range(1000)] + cmd, args = self._inject("tool", many) + assert cmd == "tool" + assert args == many + assert len(args) == 1000 + + +# --------------------------------------------------------------------------- +# Wrap-environment embedded-file reuse across tasks. +# +# The wrap environment's embedded-file PATHS are allocated once and reused +# for every task run through the wrap (so Env.File.* symbols stay stable and +# unnamed files do not accumulate on disk), while the file CONTENTS are +# re-resolved and rewritten per task (data may reference WrappedAction.*). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEmbeddedFileReuse: + def _wrap_env_with_unnamed_file(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + # The embedded file is UNNAMED (no `filename`), so its on-disk path + # is mkstemp-allocated; its data references WrappedAction.Command, + # so contents must be rewritten per task. + return Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=Action_2023_09( + command=CommandString_2023_09("cat"), + args=[ArgString_2023_09("{{ Env.File.WrapData }}")], + ), + onWrapEnvExit=_NOOP, + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="WrapData", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("wrapped-command={{WrappedAction.Command}}\n"), + ) + ], + ), + ) + + def test_unnamed_wrap_file_path_reused_across_tasks(self) -> None: + # GIVEN: a wrap environment with an unnamed embedded file whose data + # references WrappedAction.Command, and three tasks with distinct + # wrapped commands (never executed; the wrap action runs instead). + env = self._wrap_env_with_unnamed_file() + commands = ("cmd-one", "cmd-two", "cmd-three") + file_paths = [] + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment(environment=env) + _run_until_ready(session) + assert session.state == SessionState.READY + + for command in commands: + # WHEN: a task runs through the wrap. + session.run_task( + step_script=_step_script(command, []), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + # THEN: the task succeeded, and the files directory contains + # exactly ONE file for the record (not one per task) ... + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + files = [p for p in session.files_directory.iterdir() if p.is_file()] + assert len(files) == 1 + # ... whose contents reflect THIS task's wrapped command. + assert files[0].read_text() == f"wrapped-command={command}\n" + file_paths.append(files[0]) + + # THEN: the file path is identical across all three tasks. + assert file_paths[0] == file_paths[1] == file_paths[2] + + session.exit_environment(identifier=identifier) + _run_until_ready(session)