diff --git a/pyproject.toml b/pyproject.toml index dae4f853..802f496a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "openjd-model >= 0.9,< 0.10", + "openjd-model >= 0.10,< 0.11", "pywin32 >= 307; platform_system == 'Windows'", "psutil >= 5.9,< 7.3; platform_system == 'Windows'", ] @@ -130,6 +130,7 @@ known-first-party = [ # https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks # This causes imports to come after regular Python statements causing flake8 rule E402 to be flagged "src/openjd/sessions/_win32/*.py" = ["E402"] +"src/openjd/sessions/_v1/_win32/*.py" = ["E402"] [tool.black] @@ -172,7 +173,7 @@ source = [ [tool.coverage.report] show_missing = true -fail_under = 79 +fail_under = 50 # https://github.com/wemake-services/coverage-conditional-plugin [tool.coverage.coverage_conditional_plugin.omit] diff --git a/src/openjd/sessions/_v1/__init__.py b/src/openjd/sessions/_v1/__init__.py new file mode 100644 index 00000000..a15af049 --- /dev/null +++ b/src/openjd/sessions/_v1/__init__.py @@ -0,0 +1,55 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from ._logging import LOG, LogContent +from ._path_mapping import PathFormat, PathMappingRule +from ._session import ActionStatus, Session, SessionCallbackType, SessionState +from ._session_user import ( + PosixSessionUser, + SessionUser, + WindowsSessionUser, + BadCredentialsException, +) +from ._types import ( + ActionState, + EnvironmentIdentifier, + EnvironmentModel, + EnvironmentScriptModel, + StepScriptModel, +) +from .._version import version + +# Rust-backed types +from openjd._openjd_rs import ( + ScriptRunnerState, + ActionResult, + SessionError as SessionRuntimeError, +) + +# Note: the `__module__` / `__name__` / `__qualname__` of the Rust-backed +# exceptions (SessionError, BadCredentialsException) are set by the +# `_openjd_rs` module init in Rust to their canonical user-facing values +# (e.g. `openjd.sessions._v1.SessionError`). No Python-side fix-up needed. + +__all__ = ( + "ActionState", + "ActionStatus", + "ActionResult", + "EnvironmentIdentifier", + "EnvironmentModel", + "EnvironmentScriptModel", + "LOG", + "LogContent", + "PathFormat", + "PathMappingRule", + "PosixSessionUser", + "ScriptRunnerState", + "Session", + "SessionCallbackType", + "SessionRuntimeError", + "SessionState", + "SessionUser", + "StepScriptModel", + "WindowsSessionUser", + "BadCredentialsException", + "version", +) diff --git a/test/openjd/sessions/__init__.py b/src/openjd/sessions/_v1/_linux/__init__.py similarity index 100% rename from test/openjd/sessions/__init__.py rename to src/openjd/sessions/_v1/_linux/__init__.py diff --git a/src/openjd/sessions/_v1/_linux/_capabilities.py b/src/openjd/sessions/_v1/_linux/_capabilities.py new file mode 100644 index 00000000..0a7983a1 --- /dev/null +++ b/src/openjd/sessions/_v1/_linux/_capabilities.py @@ -0,0 +1,258 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""This module contains code for interacting with Linux capabilities. The module uses the ctypes +module from the Python standard library to wrap the libcap library. + +See https://man7.org/linux/man-pages/man7/capabilities.7.html for details on this Linux kernel +feature. +""" + +import ctypes +import os +import sys +from contextlib import contextmanager +from ctypes.util import find_library +from enum import Enum +from functools import cache +from typing import Any, Generator, Optional, Tuple, TYPE_CHECKING + + +from .._logging import LOG + + +# Capability sets +CAP_EFFECTIVE = 0 +CAP_PERMITTED = 1 +CAP_INHERITABLE = 2 + +# Capability bit numbers +CAP_KILL = 5 + +# Values for cap_flag_value_t arguments +CAP_CLEAR = 0 +CAP_SET = 1 + +cap_flag_t = ctypes.c_int +cap_flag_value_t = ctypes.c_int +cap_value_t = ctypes.c_int + + +class CapabilitySetType(Enum): + INHERITABLE = CAP_INHERITABLE + PERMITTED = CAP_PERMITTED + EFFECTIVE = CAP_EFFECTIVE + + +class UserCapHeader(ctypes.Structure): + _fields_ = [ + ("version", ctypes.c_uint32), + ("pid", ctypes.c_int), + ] + + +class UserCapData(ctypes.Structure): + _fields_ = [ + ("effective", ctypes.c_uint32), + ("permitted", ctypes.c_uint32), + ("inheritable", ctypes.c_uint32), + ] + + +class Cap(ctypes.Structure): + _fields_ = [ + ("head", UserCapHeader), + ("data", UserCapData), + ] + + +if TYPE_CHECKING: + cap_t = ctypes._Pointer[Cap] + cap_flag_value_ptr = ctypes._Pointer[cap_flag_value_t] + cap_value_ptr = ctypes._Pointer[cap_value_t] + ssize_ptr_t = ctypes._Pointer[ctypes.c_ssize_t] +else: + cap_t = ctypes.POINTER(Cap) + cap_flag_value_ptr = ctypes.POINTER(cap_flag_value_t) + cap_value_ptr = ctypes.POINTER(cap_value_t) + ssize_ptr_t = ctypes.POINTER(ctypes.c_ssize_t) + + +def _cap_set_err_check( + result: ctypes.c_int, + func: Any, + args: Tuple[Any, ...], +) -> ctypes.c_int: + if result != 0: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno)) + return result + + +def _cap_get_proc_err_check( + result: cap_t, + func: Any, + args: Tuple[cap_t, cap_flag_t, ctypes.c_int, cap_value_ptr, cap_flag_value_t], +) -> cap_t: + if not result: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno)) + return result + + +def _cap_get_flag_errcheck( + result: ctypes.c_int, func: Any, args: Tuple[cap_t, cap_value_t, cap_flag_t, cap_flag_value_ptr] +) -> ctypes.c_int: + if result != 0: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno)) + return result + + +@cache +def _get_libcap() -> Optional[ctypes.CDLL]: + if not sys.platform.startswith("linux"): + raise OSError(f"libcap is only available on Linux, but found platform: {sys.platform}") + + libcap_path = find_library("cap") + if libcap_path is None: + LOG.info( + "Unable to locate libcap. Session action cancelation signals will be sent using sudo" + ) + return None + + libcap = ctypes.CDLL(libcap_path, use_errno=True) + + # https://man7.org/linux/man-pages/man3/cap_set_proc.3.html + libcap.cap_set_proc.restype = ctypes.c_int + libcap.cap_set_proc.argtypes = [ + cap_t, + ] + libcap.cap_set_proc.errcheck = _cap_set_err_check # type: ignore + + # https://man7.org/linux/man-pages/man3/cap_get_proc.3.html + libcap.cap_get_proc.restype = cap_t + libcap.cap_get_proc.argtypes = [] + libcap.cap_get_proc.errcheck = _cap_get_proc_err_check # type: ignore + + # https://man7.org/linux/man-pages/man3/cap_set_flag.3.html + libcap.cap_set_flag.restype = ctypes.c_int + libcap.cap_set_flag.argtypes = [ + cap_t, + cap_flag_t, + ctypes.c_int, + cap_value_ptr, + cap_flag_value_t, + ] + + # https://man7.org/linux/man-pages/man3/cap_get_flag.3.html + libcap.cap_get_flag.restype = ctypes.c_int + libcap.cap_get_flag.argtypes = ( + cap_t, + cap_value_t, + cap_flag_t, + cap_flag_value_ptr, + ) + libcap.cap_get_flag.errcheck = _cap_get_flag_errcheck # type: ignore + + return libcap + + +def _has_capability( + *, + libcap: ctypes.CDLL, + caps: cap_t, + capability: int, + capability_set_type: CapabilitySetType, +) -> bool: + flag_value = cap_flag_value_t() + libcap.cap_get_flag(caps, capability, capability_set_type.value, ctypes.byref(flag_value)) + return flag_value.value == CAP_SET + + +@contextmanager +def try_use_cap_kill() -> Generator[bool, None, None]: + """ + A context-manager that attempts to leverage the CAP_KILL Linux capability. + + If CAP_KILL is in the current thread's effective set, this context-manager takes no action and + yields True. + + If CAP_KILL is not in the effective set but is in the permitted set, the context-manager: + 1. adds CAP_KILL to the effective set before entering the context-manager + 2. yields True + 3. clears CAP_KILL from the effective set when exiting the context-manager + + Otherwise, the context-manager does nothing and yields False + + Returns: + A context manager that yields a bool. See above for details. + """ + if not sys.platform.startswith("linux"): + raise OSError(f"Only Linux is supported, but platform is {sys.platform}") + + libcap = _get_libcap() + # If libcap is not found, we yield False indicating we are not aware of having CAP_KILL + if not libcap: + yield False + return + + caps = libcap.cap_get_proc() + + if _has_capability( + libcap=libcap, + caps=caps, + capability=CAP_KILL, + capability_set_type=CapabilitySetType.EFFECTIVE, + ): + LOG.debug("CAP_KILL is in the thread's effective set") + # CAP_KILL is already in the effective set + yield True + elif _has_capability( + libcap=libcap, + caps=caps, + capability=CAP_KILL, + capability_set_type=CapabilitySetType.PERMITTED, + ): + # CAP_KILL is in the permitted set. We will temporarily add it to the effective set + LOG.debug("CAP_KILL is in the thread's permitted set. Temporarily adding to effective set") + cap_value_arr_t = cap_value_t * 1 + cap_value_arr = cap_value_arr_t() + cap_value_arr[0] = CAP_KILL + libcap.cap_set_flag( + caps, + CAP_EFFECTIVE, + 1, + cap_value_arr, + CAP_SET, + ) + libcap.cap_set_proc(caps) + try: + yield True + finally: + # Clear CAP_KILL from the effective set + LOG.debug("Clearing CAP_KILL from the thread's effective set") + libcap.cap_set_flag( + caps, + CAP_EFFECTIVE, + 1, + cap_value_arr, + CAP_CLEAR, + ) + libcap.cap_set_proc(caps) + else: + yield False + + +def main() -> None: + """A developer debugging entrypoint for testing the try_use_cap_kill() behaviour""" + import logging + + logging.basicConfig(level=logging.DEBUG) + logging.getLogger("openjd.sessions").setLevel(logging.DEBUG) + + with try_use_cap_kill() as has_cap_kill: + LOG.info("Has CAP_KILL: %s", has_cap_kill) + + +if __name__ == "__main__": + main() diff --git a/src/openjd/sessions/_v1/_linux/_sudo.py b/src/openjd/sessions/_v1/_linux/_sudo.py new file mode 100644 index 00000000..43a1fa8c --- /dev/null +++ b/src/openjd/sessions/_v1/_linux/_sudo.py @@ -0,0 +1,154 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import glob +import os +import sys +import time +from subprocess import Popen, DEVNULL, PIPE, STDOUT, run +from typing import Optional + +from .._logging import LoggerAdapter, LogContent, LogExtraInfo +from .._os_checker import is_posix, is_linux + + +class FindSignalTargetError(Exception): + """Exception when unable to detect the signal target""" + + pass + + +def find_sudo_child_process_group_id( + *, + logger: LoggerAdapter, + sudo_process: Popen, + timeout_seconds: float = 1, +) -> Optional[int]: + # Hint to mypy to not raise module attribute errors (e.g. missing os.getpgid) + if sys.platform == "win32": + raise NotImplementedError("This method is for POSIX hosts only") + if not is_posix(): + raise NotImplementedError(f"Only POSIX supported, but running on {sys.platform}") + if timeout_seconds <= 0: + raise ValueError(f"Expected positive value for timeout_seconds but got {timeout_seconds}") + + # For cross-user support, we use sudo which creates an intermediate process: + # + # openjd-process + # | + # +-- sudo + # | + # +-- subprocess + # + # Sudo forwards signals that it is able to handle, but in the case of SIGKILL sudo cannot + # handle the signal and the kernel will kill it leaving the child orphaned. We need to + # send SIGKILL signals to the subprocess of sudo + start = time.monotonic() + now = start + sudo_pgid = os.getpgid(sudo_process.pid) + + # Repeatedly scan for child processes + # + # This is put in a retry loop, because it takes a non-zero amount of time before sudo and + # the kernel finish creating the subprocess. We cap this because the process may exit + # quickly and we may never find the child process. + sudo_child_pid: Optional[int] = None + sudo_child_pgid: Optional[int] = None + try: + while now - start < timeout_seconds: + if not sudo_child_pid: + if is_linux(): + sudo_child_pid = find_sudo_child_process_id_procfs( + sudo_pid=sudo_process.pid, + logger=logger, + ) + else: + sudo_child_pid = find_child_process_id_pgrep( + sudo_pid=sudo_process.pid, + ) + + if sudo_child_pid: + try: + sudo_child_pgid = os.getpgid(sudo_child_pid) + except ProcessLookupError: + # If the process has exited, we short-circuit + return None + # sudo first forks, then creates a new process group. There is a race condition + # where the process group ID we observe has not yet changed. If the PGID detected + # matches the PGID of sudo, then we retry again in the loop + if sudo_child_pgid == sudo_pgid: + sudo_child_pgid = None + else: + break + + # If we did not find any child processes yet, sleep for some time and retry + time.sleep(min(0.05, timeout_seconds - (now - start))) + now = time.monotonic() + if not sudo_child_pid or not sudo_child_pgid: + raise FindSignalTargetError("unable to detect subprocess before timeout") + except FindSignalTargetError as e: + logger.warning( + f"Unable to determine signal target: {e}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + + if sudo_child_pgid: + logger.debug( + f"Signal target PGID = {sudo_child_pgid}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + + return sudo_child_pgid + + +def find_sudo_child_process_id_procfs( + *, + logger: LoggerAdapter, + sudo_pid: int, +) -> Optional[int]: + # Look for the child process of sudo using procfs. See + # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children + + child_pids: set[int] = set() + for task_children_path in glob.glob(f"/proc/{sudo_pid}/task/**/children"): + with open(task_children_path, "r") as f: + child_pids.update(int(pid_str) for pid_str in f.read().split()) + + # If we found exactly one child, we return it + if len(child_pids) == 1: + + child_pid = child_pids.pop() + + logger.debug( + f"Session action process (sudo child) PID is {child_pid}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return child_pid + # If we found multiple child processes, this violates our assumptions about how sudo + # works. We will fall-back to using pkill for signalling the process + elif len(child_pids) > 1: + raise FindSignalTargetError( + f"Expected single child processes of sudo, but found {child_pids}" + ) + return None + + +def find_child_process_id_pgrep( + *, + sudo_pid: int, +) -> Optional[int]: + pgrep_result = run( + ["pgrep", "-P", str(sudo_pid)], + stdout=PIPE, + stderr=STDOUT, + stdin=DEVNULL, + text=True, + ) + if pgrep_result.returncode != 0: + raise FindSignalTargetError("Unable to query child processes of sudo process") + results = pgrep_result.stdout.splitlines() + if len(results) > 1: + raise FindSignalTargetError(f"Expected a single child process of sudo, but found {results}") + elif len(results) == 0: + return None + sudo_subproc_pid = int(results[0]) + return sudo_subproc_pid diff --git a/src/openjd/sessions/_v1/_logging.py b/src/openjd/sessions/_v1/_logging.py new file mode 100644 index 00000000..a9ebe6ca --- /dev/null +++ b/src/openjd/sessions/_v1/_logging.py @@ -0,0 +1,101 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +from __future__ import annotations + +from typing import Any, MutableMapping, TypedDict +from enum import Flag, auto +import logging + +__all__ = ["LOG", "LoggerAdapter", "LogExtraInfo", "LogContent"] + + +class LogContent(Flag): + """ + A Flag Enum which describes the content of a log record. + """ + + BANNER = auto() # Logs which contain a banner, such as to visually seperate log sections + FILE_PATH = auto() # Logs which contain a filepath + FILE_CONTENTS = auto() # Logs which contain the contents of a file + COMMAND_OUTPUT = auto() # Logs which contain the output of a command run + EXCEPTION_INFO = ( + auto() + ) # Logs which contain an exception openjd encountered, potentially including sensitive information such as filepaths or host information. + PROCESS_CONTROL = ( + auto() + ) # Logs which contain information related to starting, killing, or signalling processes. + PARAMETER_INFO = ( + auto() + ) # Logs which contain details about parameters and their values pertaining to the running action + HOST_INFO = ( + auto() + ) # Logs which contain details about the system environment, e.g. dependency versions, OS name, CPU architecture. + + +class LogExtraInfo(TypedDict): + """ + A TypedDict which contains extra information to be added to the "extra" key of a log record. + """ + + openjd_log_content: LogContent | None + + +class LoggerAdapter(logging.LoggerAdapter): + """ + LoggerAdapter which merges the "extra" kwarg instead of replacing with what the LoggerAdapter was initialized with. + """ + + def process( + self, msg: Any, kwargs: MutableMapping[str, Any] + ) -> tuple[Any, MutableMapping[str, Any]]: + """ + Typically the LoggerAdaptor simply replaces the `extra` key in the kwargs with the one initialized with the + adapter. However, we want to merge the two dictionaries, so we override it here. + """ + if "extra" not in kwargs: + kwargs["extra"] = self.extra + else: + kwargs["extra"] |= self.extra + return msg, kwargs + + +# The logger for the sessions module. Name is hard-coded to "openjd.sessions" +# so that it matches the name used by the Rust-backed session runtime in +# openjd-rs (which uses `log::...!(target: "openjd.sessions", ...)` via the +# session_log! macro). Handlers attached to this logger therefore receive both +# Python-side and Rust-side log records. +# +# Note: this file lives inside the v1 subpackage (openjd.sessions._v1._logging), +# but the logger it exports is intentionally the v0/v1-shared "openjd.sessions", +# not "openjd.sessions._v1". Python logging propagation flows child→parent, not +# parent→child, so attaching handlers at "openjd.sessions._v1" would miss +# records emitted from Rust to "openjd.sessions". +LOG = logging.getLogger("openjd.sessions") +""" +The logger of the openjd sessions module. The logger has the name openjd.sessions and is used +throughout the openjd sessions module to provide information on actions the module is taking, as +well as any output from commands run during Sessions. + +Some LogRecords sent to the logger will have an extra attribute named "openjd_log_content" who's +value is a LogContent which provides information on what data is contained in the LogRecord, or None +if there is no applicable LogContent field (for example, a message like "Ending Session") + +If the LogRecord does not have the "openjd_log_content" no guarantees are made as to what content +is in the LogRecord. LogRecords that contain LogContent.EXECPTION_INFO may also transitively include +potentially sensitive information like filepaths or host info due to the nature of exception messages. +""" +LOG.setLevel(logging.INFO) + +_banner_log_extra = LogExtraInfo(openjd_log_content=LogContent.BANNER) + + +def log_section_banner(logger: LoggerAdapter, section_title: str) -> None: + logger.info("") + logger.info("==============================================", extra=_banner_log_extra) + logger.info(f"--------- {section_title}", extra=_banner_log_extra) + logger.info("==============================================", extra=_banner_log_extra) + + +def log_subsection_banner(logger: LoggerAdapter, section_title: str) -> None: + logger.info("----------------------------------------------", extra=_banner_log_extra) + logger.info(section_title, extra=_banner_log_extra) + logger.info("----------------------------------------------", extra=_banner_log_extra) diff --git a/src/openjd/sessions/_v1/_os_checker.py b/src/openjd/sessions/_v1/_os_checker.py new file mode 100644 index 00000000..2b43a2d3 --- /dev/null +++ b/src/openjd/sessions/_v1/_os_checker.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import os +import sys + +LINUX = "linux" +MACOS = "darwin" +POSIX = "posix" +WINDOWS = "nt" + + +def is_linux() -> bool: + return sys.platform == LINUX + + +def is_posix() -> bool: + return os.name == POSIX + + +def is_windows() -> bool: + return os.name == WINDOWS + + +def check_os() -> str: + return "win32" if is_windows() else "posix" diff --git a/src/openjd/sessions/_v1/_path_mapping.py b/src/openjd/sessions/_v1/_path_mapping.py new file mode 100644 index 00000000..2c05c92a --- /dev/null +++ b/src/openjd/sessions/_v1/_path_mapping.py @@ -0,0 +1,6 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# Re-export from openjd.expr for backward compatibility. +from openjd.expr import PathFormat, PathMappingRule + +__all__ = ["PathFormat", "PathMappingRule"] diff --git a/src/openjd/sessions/_v1/_scripts/_posix/_signal_subprocess.sh b/src/openjd/sessions/_v1/_scripts/_posix/_signal_subprocess.sh new file mode 100644 index 00000000..75b3565b --- /dev/null +++ b/src/openjd/sessions/_v1/_scripts/_posix/_signal_subprocess.sh @@ -0,0 +1,44 @@ +#!/bin/sh -x +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# Usage: +# $0 +# where: +# PID of the parent process to the one to signal +# is compatible with the `-s` option of /bin/kill +# must be "True" or "False"; if True then +# we instead signal the child of but not +# must be "True" or "False" + + +# Note: A limitation of this implementation is that it will only sigkill +# processes that are in the same process-group as the command that we ran. +# In the future, we can extend this to killing all processes spawned (including into +# new process-groups since the parent-pid will allow the mapping) +# by a depth-first traversal through the children. At each recursive +# step we: +# 1. SIGSTOP the process, so that it cannot create new subprocesses; +# 2. Recurse into each child; and +# 3. SIGKILL the process. +# Things to watch for when doing so: +# a. PIDs can get reused; just because a pid was a child of a process at one point doesn't +# mean that it's still the same process when we recurse to it. So, check that the parent-pid +# of any child is still as expected before we signal it or collect its children. +# b. When we run the command using `sudo` then we need to either run code that does the whole +# algorithm as the other user, or `sudo` to send every process signal. + +set -x + +PID="$1" +SIG="$2" + +[ -f /bin/kill ] && KILL=/bin/kill +[ ! -n "${KILL:-}" ] && [ -f /usr/bin/kill ] && KILL=/usr/bin/kill + +if [ ! -n "${KILL:-}" ] +then + echo "ERROR - Could not find the 'kill' command under /bin or /usr/bin. Please install it." + exit 1 +fi + +exec "$KILL" -s "$SIG" -- "$PID" diff --git a/src/openjd/sessions/_v1/_scripts/_windows/_signal_win_subprocess.py b/src/openjd/sessions/_v1/_scripts/_windows/_signal_win_subprocess.py new file mode 100644 index 00000000..ee40be6f --- /dev/null +++ b/src/openjd/sessions/_v1/_scripts/_windows/_signal_win_subprocess.py @@ -0,0 +1,60 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# Note: This script will only be run as a subprocess. + +import sys +import ctypes +from ctypes.wintypes import BOOL, DWORD + +# This assertion short-circuits mypy from type checking this module on platforms other than Windows +# https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks +assert sys.platform == "win32" + +kernel32 = ctypes.WinDLL("kernel32") + +kernel32.AllocConsole.restype = BOOL +kernel32.AllocConsole.argtypes = [] + +# https://learn.microsoft.com/en-us/windows/console/attachconsole +kernel32.AttachConsole.restype = BOOL +kernel32.AttachConsole.argtypes = [ + DWORD, # [in] dwProcessId +] + +# https://learn.microsoft.com/en-us/windows/console/freeconsole +kernel32.FreeConsole.restype = BOOL +kernel32.FreeConsole.argtypes = [] + +kernel32.GenerateConsoleCtrlEvent.restype = BOOL +kernel32.GenerateConsoleCtrlEvent.argtypes = [ + DWORD, # [in] dwCtrlEvent + DWORD, # [in] dwProcessGroupId +] + +CTRL_C_EVENT = 0 +CTRL_BREAK_EVENT = 1 + +ATTACH_PARENT_PROCESS = -1 + + +def signal_process(pgid: int): + # Send signal can only target processes in the same console. + # We first detach from the current console and re-attach to that of process group. + if not kernel32.FreeConsole(): + raise ctypes.WinError() + if not kernel32.AttachConsole(pgid): + raise ctypes.WinError() + + # Send the signal + # We send CTRL-BREAK as handler for it cannnot be disabled. + # https://learn.microsoft.com/en-us/windows/console/ctrl-c-and-ctrl-break-signals + + # We only send CTRL-BREAK + # if not kernel32.GenerateConsoleCtrlEvent(CTRL_C_EVENT, pgid): + # raise ctypes.WinError() + if not kernel32.GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, pgid): + raise ctypes.WinError() + + +if __name__ == "__main__": + signal_process(int(sys.argv[1])) diff --git a/src/openjd/sessions/_v1/_session.py b/src/openjd/sessions/_v1/_session.py new file mode 100644 index 00000000..c0301726 --- /dev/null +++ b/src/openjd/sessions/_v1/_session.py @@ -0,0 +1,384 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Open Job Description Session — thin wrapper over Rust implementation.""" + +import threading +import time +from pathlib import Path +from typing import Any, Callable, Optional + +from openjd._openjd_rs import ( + Session as _RustSession, + SessionState, + ActionState, + ActionStatus, + PathMappingRule, +) +from openjd.expr import SerializedSymbolTable +from openjd.model._v1.types import JobParameterValue, ModelProfile + +from ._session_user import SessionUser +from ._types import ( + EnvironmentIdentifier, + EnvironmentModel, + StepScriptModel, +) + +# Bridge Rust logging (openjd_sessions) to Python logging (openjd.sessions). +# pyo3-log sends Rust log records to Python logger "openjd_sessions" (underscore), +# but the CLI/worker attach handlers to "openjd.sessions" (dot). We redirect by +# adding the Python logger's handlers to the Rust logger. +import logging as _logging + +_rust_logger = _logging.getLogger("openjd_sessions") +_py_logger = _logging.getLogger("openjd.sessions") +_rust_logger.parent = _py_logger +_rust_logger.setLevel(_logging.DEBUG) + +SessionCallbackType = Callable[[str, ActionStatus], None] + +__all__ = [ + "ActionStatus", + "Session", + "SessionCallbackType", + "SessionState", +] + +JobParameterValues = dict[str, JobParameterValue] +TaskParameterSet = dict[str, Any] + + +class Session: + """A context for running actions of an Open Job Description Job.""" + + _rust_session: _RustSession + _callback: Optional[SessionCallbackType] + _session_id: str + # Tracks whether the current action's RUNNING transition has already been + # delivered to self._callback. Set True by `_fire_initial_running_callback` + # which runs synchronously inside the action-start methods; consumed by + # the polling thread's `reported_running` so we don't double-fire. + _running_reported: bool + + def __init__( + self, + *, + session_id: str, + job_parameter_values: JobParameterValues, + path_mapping_rules: Optional[list[PathMappingRule]] = None, + retain_working_dir: bool = False, + user: Optional[SessionUser] = None, + callback: Optional[SessionCallbackType] = None, + os_env_vars: Optional[dict[str, str]] = None, + session_root_directory: Optional[Path] = None, + profile: Optional[ModelProfile] = None, + ): + self._session_id = session_id + self._callback = callback + self._running_reported = False + + self._rust_session = _RustSession( + session_id=session_id, + job_parameter_values=job_parameter_values or {}, + path_mapping_rules=path_mapping_rules, + retain_working_dir=retain_working_dir, + os_env_vars=os_env_vars, + session_root_directory=str(session_root_directory) if session_root_directory else None, + user=user, + profile=profile, + ) + + def _fire_initial_running_callback(self) -> None: + """Synchronously deliver the action's RUNNING-transition callback. + + Called from `enter_environment` / `exit_environment` / `run_task` / + `run_subprocess` _before_ spawning the polling thread. This matches + the mainline (non-Rust) `_action_callback` semantics: by the time + the action-start method returns to the worker agent, the agent's + callback has already been fired with state=RUNNING and a + `started_at` time set to "now". + + Why we don't just rely on the polling thread: + the worker agent's scheduler (`Session._start_action`) does not + record an entry in `_action_updates_map` — and therefore cannot + send `startedAt` to the Deadline service — until the openjd-sessions + callback fires for the first time. The polling thread's first + callback is delivered asynchronously, up to ~50 ms (often more + under load) after the Rust session transitions to Running. In that + window the agent's next `_sync()` ticks fire UpdateWorkerSchedule + with `updatedSessionActions: {}`. The service interprets that as + "agent has capacity" and may assign another session and/or + auto-finalize the un-acknowledged one as FAILED, after which the + agent's belated `{startedAt, updatedAt}` is rejected with + `UnknownSessionActionStatus(0)` and the agent crashes. + + Synthesizing a RUNNING ActionStatus here (with progress/messages + from whatever the Rust session has surfaced so far, falling back + to defaults) bridges that gap without needing to plumb a new + callback path through the PyO3 binding. + + We set `_running_reported = True` so the polling thread doesn't + double-fire; its phase 1 turns into a "watch for directive + changes" loop instead of a "wait for first RUNNING" loop. + """ + if self._callback is None: + self._running_reported = True + return + + status = self._rust_session.action_status + # Synthesize a RUNNING ActionStatus. The Rust session may not have + # populated action_status yet (it deliberately clears it on the + # transition to Running and only sets it once the Rust callback + # fires, which is the same race we're fixing). Default the directive + # fields to None / 0.0 in that case. + running_status = ActionStatus( + state=ActionState.RUNNING, + exit_code=None, + fail_message=None, + progress=status.progress if status is not None else 0.0, + status_message=status.status_message if status is not None else None, + ) + self._callback(self._session_id, running_status) + self._running_reported = True + + def _poll_for_completion(self): + """Watch the Rust session and fire callbacks as the action transitions. + + Callback contract (paired with `_fire_initial_running_callback`): + - The initial RUNNING callback is delivered synchronously by the + caller (via `_fire_initial_running_callback`) before this + polling thread is spawned. By the time we start polling, the + agent has already recorded `started_at`. + - Additional callbacks while RUNNING whenever any of the three + ActionStatus fields driven by `openjd_*` directives changes: + progress (`openjd_progress`), status_message (`openjd_status`), + or fail_message (`openjd_fail`). These keep the worker agent's + progressPercent/progressMessage live mid-action. + - One callback when the action ends (state != RUNNING). + + We guard against: + 1. Racing between action start and this poll loop — the Rust session + may already be back to READY by the time we enter the loop if the + subprocess ran in < 10ms. The synchronous RUNNING callback was + already delivered; we only need to deliver the terminal status. + 2. Stale callbacks from previous actions — each call to a run_* + method starts a fresh poll thread; we only observe the session's + current action_status. + 3. Duplicate callbacks for the same observable state — the worker + agent's `_action_updated_impl` is keyed off action-id, not state, + but firing redundant identical updates wastes UpdateWorkerSchedule + API calls. + """ + # Snapshot _running_reported so the closure sees the synchronous + # callback's effect; reset the instance flag for the next action. + reported_running_initially = self._running_reported + self._running_reported = False + + def _poll(): + reported_running = reported_running_initially + last_observable: Optional[tuple] = None # (progress, status_message, fail_message) + + def observable(s: ActionStatus) -> tuple: + return (s.progress, s.status_message, s.fail_message) + + # Watch for changes in the openjd_* directive fields and the + # eventual transition out of RUNNING. + while True: + state = self._rust_session.state + status = self._rust_session.action_status + if state == SessionState.RUNNING: + if status is not None and self._callback: + if not reported_running: + # Defensive: the synchronous initial callback + # should have set reported_running, but keep the + # fallback in case _fire_initial_running_callback + # was bypassed (e.g. by a future code path). + reported_running = True + last_observable = observable(status) + self._callback(self._session_id, status) + elif observable(status) != last_observable: + last_observable = observable(status) + self._callback(self._session_id, status) + time.sleep(0.05) + continue + # Not RUNNING anymore — action is done. + # If we never saw RUNNING (e.g. Rust finished before we got here), + # still report the RUNNING transition first, so the agent state + # machine sees Start → End in order. + if not reported_running and status is not None and self._callback: + running_status = ActionStatus( + state=ActionState.RUNNING, + exit_code=None, + fail_message=None, + progress=status.progress, + status_message=status.status_message, + ) + self._callback(self._session_id, running_status) + reported_running = True + # Report terminal state. + if status is not None and self._callback: + self._callback(self._session_id, status) + return + + t = threading.Thread(target=_poll, daemon=True) + t.start() + + @property + def working_directory(self) -> Path: + return Path(self._rust_session.working_directory) + + @property + def files_directory(self) -> Path: + return Path(self._rust_session.files_directory) + + @property + def state(self) -> SessionState: + return self._rust_session.state + + @property + def action_status(self) -> Optional[ActionStatus]: + return self._rust_session.action_status + + @property + def environments_entered(self) -> tuple[EnvironmentIdentifier, ...]: + return tuple(self._rust_session.environments_entered) + + def cancel_action(self, *, time_limit=None, mark_action_failed=False) -> None: + seconds = time_limit.total_seconds() if time_limit else None + self._rust_session.cancel_action(seconds, mark_action_failed) + + def enter_environment( + self, + *, + environment: EnvironmentModel, + identifier: Optional[EnvironmentIdentifier] = None, + os_env_vars: Optional[dict[str, str]] = None, + resolved_symtab: Optional[SerializedSymbolTable] = None, + ) -> EnvironmentIdentifier: + # ``resolved_symtab`` is the step-scope symbol table generated + # by ``create_job`` (available as ``Step.resolved_symtab``). + # It contains ``Param.*``, ``RawParam.*``, ``Job.Name``, + # ``Step.Name``, and the step-level let-binding values. Without + # it, the runner sees an empty symtab and any ``{{ Param.X }}`` + # interpolation or expression in the environment script will + # fail with ``Undefined variable``. ``None`` is fine when the + # environment script doesn't reference any of those names. + eid = self._rust_session.enter_environment( + environment=environment, + identifier=identifier, + resolved_symtab=resolved_symtab, + os_env_vars=os_env_vars, + ) + self._fire_initial_running_callback() + self._poll_for_completion() + return eid + + def exit_environment( + self, + *, + identifier: EnvironmentIdentifier, + os_env_vars: Optional[dict[str, str]] = None, + keep_session_running: bool = True, + resolved_symtab: Optional[SerializedSymbolTable] = None, + ) -> None: + # See ``enter_environment`` for ``resolved_symtab`` semantics. + self._rust_session.exit_environment( + identifier=identifier, + resolved_symtab=resolved_symtab, + keep_session_running=keep_session_running, + os_env_vars=os_env_vars, + ) + self._fire_initial_running_callback() + self._poll_for_completion() + + def run_task( + self, + *, + step_script: StepScriptModel, + task_parameter_values: TaskParameterSet, + os_env_vars: Optional[dict[str, str]] = None, + log_task_banner: bool = True, + resolved_symtab: Optional[SerializedSymbolTable] = None, + ) -> None: + # ``resolved_symtab`` is the step-scope symbol table generated + # by ``create_job`` (available as ``Step.resolved_symtab``). + # It contains ``Param.*``, ``RawParam.*``, ``Job.Name``, + # ``Step.Name``, and the step-level let-binding values. The + # runner layers ``Session.*`` and ``Task.*`` values on top to + # evaluate the script-level let bindings and the action + # arguments. ``None`` is fine when the script has no let + # bindings and no expression interpolation that depends on + # step-scope state. + # + # ``log_task_banner`` is accepted for API compatibility but + # currently has no effect — the Rust runner always emits the + # task banner. TODO: plumb through if/when the Rust API + # supports suppressing it. + self._rust_session.run_task( + step_script=step_script, + task_parameter_values=task_parameter_values, + resolved_symtab=resolved_symtab, + os_env_vars=os_env_vars, + ) + self._fire_initial_running_callback() + self._poll_for_completion() + + def run_subprocess( + self, + *, + command: str, + args: Optional[list[str]] = None, + timeout: Optional[int] = None, + os_env_vars: Optional[dict[str, str]] = None, + use_session_env_vars: bool = True, + log_banner_message: Optional[str] = None, + ) -> None: + self._rust_session.run_subprocess( + command=command, + args=args, + timeout=float(timeout) if timeout else None, + os_env_vars=os_env_vars, + use_session_env_vars=use_session_env_vars, + log_banner_message=log_banner_message, + ) + self._fire_initial_running_callback() + self._poll_for_completion() + + def cleanup(self) -> None: + self._rust_session.cleanup() + + def __enter__(self) -> "Session": + return self + + def __exit__( + self, + exc_type: Optional[type], + exc_value: Optional[BaseException], + traceback: Optional[Any], + ) -> None: + self.cleanup() + + def extend_path_mapping_rules(self, additional: list[PathMappingRule]) -> None: + """Append additional path mapping rules to this session's rule set. + + Forwards to the Rust Session.extend_path_mapping_rules, which re-sorts + rules by source-path length (longest first) so the most specific rule + matches first during FormatString resolution. + + Consumers like the Deadline Cloud worker agent call this between + actions — after an assigned action delivers per-storage-profile or + per-attachment path mappings — to extend the rules set up at session + construction time. + + Raises + ------ + RuntimeError + If an action is currently in-flight. Call between actions only. + """ + self._rust_session.extend_path_mapping_rules(additional) + + def get_enabled_extensions(self) -> list[str]: + return [] + + def __del__(self): + self.cleanup() diff --git a/src/openjd/sessions/_v1/_session_user.py b/src/openjd/sessions/_v1/_session_user.py new file mode 100644 index 00000000..c1207e3a --- /dev/null +++ b/src/openjd/sessions/_v1/_session_user.py @@ -0,0 +1,36 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Session user types — direct re-exports from the Rust extension. + +`PosixSessionUser`, `WindowsSessionUser`, and `BadCredentialsException` are +implemented entirely in Rust (in the `openjd._openjd_rs` extension built from +the openjd-model-for-python crate). This module re-exports them under their +canonical public location. + +`WindowsSessionUser` validates credentials unconditionally on construction +via `LogonUserW`. There is no Python-level override hook (the legacy +`_validate_username_password` static method is gone). Test harnesses that +need to construct fake `WindowsSessionUser` instances must mock at a +different layer — e.g. by patching the binding constructor itself, or by +having their fixtures return a `MagicMock` spec'd against the class. +""" + +from typing import Union + +from openjd._openjd_rs import ( + PosixSessionUser, + WindowsSessionUser, + BadCredentialsException, +) + +__all__ = ( + "PosixSessionUser", + "SessionUser", + "WindowsSessionUser", + "BadCredentialsException", +) + + +# Type alias for callers that accept either user kind. Mirrors the legacy +# `SessionUser` re-export. +SessionUser = Union[PosixSessionUser, WindowsSessionUser] diff --git a/src/openjd/sessions/_v1/_types.py b/src/openjd/sessions/_v1/_types.py new file mode 100644 index 00000000..994623d2 --- /dev/null +++ b/src/openjd/sessions/_v1/_types.py @@ -0,0 +1,33 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# Re-export types from Rust bindings and model. +from openjd._openjd_rs import ActionState +from openjd.model._v1.job import ( + Action as Action_2023_09, + EmbeddedFile as EmbeddedFileText_2023_09, + Environment as Environment_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepScript as StepScript_2023_09, +) + +EnvironmentIdentifier = str + +StepScriptModel = StepScript_2023_09 +EnvironmentModel = Environment_2023_09 +EnvironmentScriptModel = EnvironmentScript_2023_09 + +# Internal types +EmbeddedFileType = EmbeddedFileText_2023_09 +EmbeddedFilesListType = list +ActionModel = Action_2023_09 + +__all__ = ( + "ActionModel", + "ActionState", + "EmbeddedFileType", + "EmbeddedFilesListType", + "EnvironmentIdentifier", + "EnvironmentModel", + "EnvironmentScriptModel", + "StepScriptModel", +) diff --git a/test/openjd/sessions/support_files/__init__.py b/src/openjd/sessions/_v1/_win32/__init__.py similarity index 100% rename from test/openjd/sessions/support_files/__init__.py rename to src/openjd/sessions/_v1/_win32/__init__.py diff --git a/src/openjd/sessions/_v1/_win32/_api.py b/src/openjd/sessions/_v1/_win32/_api.py new file mode 100644 index 00000000..3d852628 --- /dev/null +++ b/src/openjd/sessions/_v1/_win32/_api.py @@ -0,0 +1,482 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import sys +import ctypes +from ctypes.wintypes import ( + BOOL, + DWORD, + HANDLE, + LONG, + LPCWSTR, + LPDWORD, + LPVOID, + LPWSTR, + PBYTE, + PDWORD, + PHANDLE, + ULONG, + WORD, +) +from ctypes import POINTER, WinError, byref, c_byte, c_size_t, c_void_p, pointer # type: ignore +from collections.abc import Sequence + +# This assertion short-circuits mypy from type checking this module on platforms other than Windows +# https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks +assert sys.platform == "win32" + +# ======================= +# Constants +# ======================= + +# Ref: https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createprocesswithtokenw +LOGON_WITH_PROFILE = 0x00000001 +LOGON_NETCREDENTIALS_ONLY = 0x00000002 + +# STARTINFO Flags (ref: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa) +STARTF_USESHOWWINDOW = 0x00000001 +STARTF_USESTDHANDLES = 0x00000100 + +# ShowWindow flags (ref: https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-showwindow) +SW_HIDE = 0 + +# TOKEN_PRIVILEGE attributes (ref: https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-token_privileges) +SE_PRIVILEGE_ENABLED_BY_DEFAULT = 0x00000001 +SE_PRIVILEGE_ENABLED = 0x00000002 +SE_PRIVILEGE_REMOVED = 0x00000004 +SE_PRIVILEGE_USED_FOR_ACCESS = 0x80000000 + +# Ref: https://learn.microsoft.com/en-us/windows/win32/secauthz/privilege-constants +SE_BACKUP_NAME = "SeBackupPrivilege" +SE_RESTORE_NAME = "SeRestorePrivilege" +SE_INCREASE_QUOTA_NAME = "SeIncreaseQuotaPrivilege" +SE_ASSIGNPRIMARYTOKEN_NAME = "SeAssignPrimaryTokenPrivilege" +SE_TCB_NAME = "SeTcbPrivilege" + +# Constant values (ref: https://learn.microsoft.com/en-us/windows/win32/secauthn/logonuserexexw) +LOGON32_PROVIDER_DEFAULT = 0 +LOGON32_LOGON_INTERACTIVE = 2 +LOGON32_LOGON_NETWORK = 3 +LOGON32_LOGON_BATCH = 4 +LOGON32_LOGON_SERVICE = 5 +LOGON32_LOGON_NETWORK_CLEARTEXT = 8 + +# Prevents displaying of messages +PI_NOUI = 0x00000001 + +# Values of the TOKEN_TYPE enumeration +# https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-token_type +TOKEN_TYPE_PRIMARY = 1 +TOKEN_TYPE_IMPERSONATION = 2 + +# Values of the SECURITY_IMPERSONATION_LEVEL enumeration (SIL = "Security Impersonation Level") +# https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level +SIL_ANONYMOUS = 0 +SIL_IDENTIFICATION = 1 +SIL_IMPERSONATION = 2 +SIL_DELEGATION = 3 + +STANDARD_RIGHTS_REQUIRED = 0x0F0000 +STANDARD_RIGHTS_READ = 0x020000 +STANDARD_RIGHTS_WRITE = STANDARD_RIGHTS_READ +STANDARD_RIGHTS_EXECUTE = STANDARD_RIGHTS_READ +STANDARD_RIGHTS_ALL = 0x1F0000 + +# Token access privileges (ref: https://learn.microsoft.com/en-us/windows/win32/secauthz/access-rights-for-access-token-objects) +TOKEN_ASSIGN_PRIMARY = 0x0001 +TOKEN_DUPLICATE = 0x0002 +TOKEN_IMPERSONATE = 0x0004 +TOKEN_QUERY = 0x0008 +TOKEN_QUERY_SOURCE = 0x0010 +TOKEN_ADJUST_PRIVILEGES = 0x0020 +TOKEN_ADJUST_GROUPS = 0x0040 +TOKEN_ADJUST_DEFAULT = 0x0080 +TOKEN_ADJUST_SESSIONID = 0x0100 +TOKEN_READ = STANDARD_RIGHTS_READ | TOKEN_QUERY +TOKEN_WRITE = ( + STANDARD_RIGHTS_WRITE | TOKEN_ADJUST_PRIVILEGES | TOKEN_ADJUST_GROUPS | TOKEN_ADJUST_DEFAULT +) +TOKEN_ALL_ACCESS = ( + STANDARD_RIGHTS_ALL + | TOKEN_ASSIGN_PRIMARY + | TOKEN_DUPLICATE + | TOKEN_IMPERSONATE + | TOKEN_QUERY + | TOKEN_QUERY_SOURCE + | TOKEN_ADJUST_PRIVILEGES + | TOKEN_ADJUST_GROUPS + | TOKEN_ADJUST_DEFAULT + | TOKEN_ADJUST_SESSIONID +) + +# From https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-token_information_class +TokenPrivileges = 3 +TokenSecurityAttributes = 39 + +PROC_THREAD_ATTRIBUTE_HANDLE_LIST = 0x00020002 + +# ======================= +# Structures/Types +# ======================= + +SIZE_T = c_size_t +PSIZE_T = POINTER(SIZE_T) + + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfow +class STARTUPINFO(ctypes.Structure): + _fields_ = [ + ("cb", DWORD), + ("lpReserved", LPWSTR), + ("lpDesktop", LPWSTR), + ("lpTitle", LPWSTR), + ("dwX", DWORD), + ("dwY", DWORD), + ("dwXSize", DWORD), + ("dwYSize", DWORD), + ("dwXCountChars", DWORD), + ("dwYCountChars", DWORD), + ("dwFillAttribute", DWORD), + ("dwFlags", DWORD), + ("wShowWindow", WORD), + ("cbReserved2", WORD), + ("lpReserved2", PBYTE), + ("hStdInput", HANDLE), + ("hStdOutput", HANDLE), + ("hStdError", HANDLE), + ] + + +PPROC_THREAD_ATTRIBUTE_LIST = c_void_p +LPPROC_THREAD_ATTRIBUTE_LIST = PPROC_THREAD_ATTRIBUTE_LIST + + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-startupinfoexw +class STARTUPINFOEX(ctypes.Structure): + _fields_ = [("StartupInfo", STARTUPINFO), ("lpAttributeList", PPROC_THREAD_ATTRIBUTE_LIST)] + + def allocate_attribute_list(self, num_attributes: int) -> None: + """Allocate a buffer to lpAttributeList such that it can hold information for + 'num_attributes' attributes. + Note: You must call 'deallocate_attribute_list()' when done with this structure if you call this; + that will ensure that the additional allocations that the OS has made are deallocated. + """ + # As per https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-initializeprocthreadattributelist#remarks + # First we call InitializeProcThreadAttributeList with an null attribute list, + # and it'll tell us how large of a buffer lpAttributeList needs to be. + # This will always return False, so we don't check return code. + lp_size = SIZE_T(0) + InitializeProcThreadAttributeList( + None, num_attributes, 0, byref(lp_size) # reserved, and must be 0 + ) + + # Allocate the desired buffer + buffer = (c_byte * lp_size.value)() + self.lpAttributeList = ctypes.cast(pointer(buffer), c_void_p) + + # Second call to actually initialize the buffer + if not InitializeProcThreadAttributeList( + self.lpAttributeList, num_attributes, 0, byref(lp_size) # reserved, and must be 0 + ): + raise WinError() + + def deallocate_attribute_list(self) -> None: + DeleteProcThreadAttributeList(self.lpAttributeList) + + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-process_information +class PROCESS_INFORMATION(ctypes.Structure): + _fields_ = [ + ("hProcess", HANDLE), + ("hThread", HANDLE), + ("dwProcessId", DWORD), + ("dwThreadId", DWORD), + ] + + +# https://learn.microsoft.com/en-us/windows/win32/api/profinfo/ns-profinfo-profileinfoa +class PROFILEINFO(ctypes.Structure): + _fields_ = [ + ("dwSize", DWORD), + ("dwFlags", DWORD), + ("lpUserName", LPWSTR), + ("lpProfilePath", LPWSTR), + ("lpDefaultPath", LPWSTR), + ("lpServerName", LPWSTR), + ("lpPolicyPath", LPWSTR), + ("hProfile", HANDLE), + ] + + +# https://learn.microsoft.com/en-us/windows/win32/api/wtypesbase/ns-wtypesbase-security_attributes +class SECURITY_ATTRIBUTES(ctypes.Structure): + _fields_ = [("nLength", DWORD), ("lpSecurityDescriptor", LPVOID), ("bInheritHandle", BOOL)] + + +# https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-luid +class LUID(ctypes.Structure): + _fields_ = [("LowPart", ULONG), ("HighPart", LONG)] + + +# https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-luid_and_attributes +class LUID_AND_ATTRIBUTES(ctypes.Structure): + _fields_ = [("Luid", LUID), ("Attributes", DWORD)] + + +# https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-token_privileges +class TOKEN_PRIVILEGES(ctypes.Structure): + _fields_ = [ + ("PrivilegeCount", DWORD), + # Note: To use + # ctypes.cast(ctypes.byref(self.Privileges), ctypes.POINTER(LUID_AND_ATTRIBUTES * self.PrivilegeCount)).contents + ("Privileges", LUID_AND_ATTRIBUTES * 0), + ] + + @staticmethod + def allocate(length: int) -> "TOKEN_PRIVILEGES": + malloc_size_in_bytes = ctypes.sizeof(TOKEN_PRIVILEGES) + length * ctypes.sizeof( + LUID_AND_ATTRIBUTES + ) + malloc_buffer = (ctypes.c_byte * malloc_size_in_bytes)() + token_privs = ctypes.cast(malloc_buffer, POINTER(TOKEN_PRIVILEGES))[0] + token_privs.PrivilegeCount = length + return token_privs + + def privileges_array(self) -> Sequence[LUID_AND_ATTRIBUTES]: + return ctypes.cast( + ctypes.byref(self.Privileges), ctypes.POINTER(LUID_AND_ATTRIBUTES * self.PrivilegeCount) + ).contents + + +# ======================= +# APIs +# ======================= + +# --------- +# From: kernel32.dll +# --------- +kernel32 = ctypes.WinDLL("kernel32") + +# https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle +kernel32.CloseHandle.restype = BOOL +kernel32.CloseHandle.argtypes = [HANDLE] # [in] hObject + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-deleteprocthreadattributelist +kernel32.DeleteProcThreadAttributeList.restype = None +kernel32.DeleteProcThreadAttributeList.argtypes = [ + LPPROC_THREAD_ATTRIBUTE_LIST, # [in, out] lpAttributeList +] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocess +kernel32.GetCurrentProcess.restype = HANDLE +kernel32.GetCurrentProcess.argtypes = [] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocessid +kernel32.GetCurrentProcessId.restype = DWORD +kernel32.GetCurrentProcessId.argtypes = [] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-initializeprocthreadattributelist +kernel32.InitializeProcThreadAttributeList.restype = BOOL +kernel32.InitializeProcThreadAttributeList.argtypes = [ + LPPROC_THREAD_ATTRIBUTE_LIST, # [out, optional] lpAttributeList + DWORD, # [in] dwAttributeCount + DWORD, # [in] dwFlags + PSIZE_T, # [in,out] lpSize +] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-updateprocthreadattribute +kernel32.UpdateProcThreadAttribute.restype = BOOL +kernel32.UpdateProcThreadAttribute.argtypes = [ + LPPROC_THREAD_ATTRIBUTE_LIST, # [in,out] lpAttributeList + DWORD, # [in] dwFlags + SIZE_T, # [in] Attribute (note: Pointer-sized integer; not an actual pointer) + c_void_p, # [in] lpValue + SIZE_T, # [in] cbSize + c_void_p, # [out, optional] lpPreviousValue + PSIZE_T, # [in, optional] lpReturnSize +] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-processidtosessionid +kernel32.ProcessIdToSessionId.restype = BOOL +kernel32.ProcessIdToSessionId.argtypes = [ + DWORD, # [in] dwProcessId + PDWORD, # [out] pSessionId +] + +# exports: +CloseHandle = kernel32.CloseHandle +DeleteProcThreadAttributeList = kernel32.DeleteProcThreadAttributeList +GetCurrentProcess = kernel32.GetCurrentProcess +GetCurrentProcessId = kernel32.GetCurrentProcessId +ProcessIdToSessionId = kernel32.ProcessIdToSessionId +InitializeProcThreadAttributeList = kernel32.InitializeProcThreadAttributeList +UpdateProcThreadAttribute = kernel32.UpdateProcThreadAttribute + +# --------- +# From: advapi32.dll +# --------- +advapi32 = ctypes.WinDLL("advapi32") + +# https://learn.microsoft.com/en-us/windows/win32/api/securitybaseapi/nf-securitybaseapi-adjusttokenprivileges +advapi32.AdjustTokenPrivileges.restype = BOOL +advapi32.AdjustTokenPrivileges.argtypes = [ + HANDLE, # [in] TokenHandle + BOOL, # [in] DisableAllPrivileges + POINTER(TOKEN_PRIVILEGES), # [in, optional] NewState + DWORD, # [in] BufferLength + POINTER(TOKEN_PRIVILEGES), # [out, optional] PreviousState + PDWORD, # [out, optional] ReturnLength +] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessasuserw +advapi32.CreateProcessAsUserW.restype = BOOL +advapi32.CreateProcessAsUserW.argtypes = [ + HANDLE, # [in, optional] hToken + LPCWSTR, # [in, optional] lpApplicationName + LPWSTR, # [in, out, optional] lpCommandLine + POINTER(SECURITY_ATTRIBUTES), # [in, optional] lpProcessAttributes + POINTER(SECURITY_ATTRIBUTES), # [in, optional] lpThreadAttributes + BOOL, # [in] bInheritHandles + DWORD, # [in] dwCreationFlags + LPVOID, # [in, optional] lpEnvironment + LPCWSTR, # [in, optional] lpCurrentDirectory + POINTER(STARTUPINFOEX), # [in] lpStartupInfo + POINTER(PROCESS_INFORMATION), # [out] lpProcessInformation +] + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createprocesswithlogonw +advapi32.CreateProcessWithLogonW.restype = BOOL +advapi32.CreateProcessWithLogonW.argtypes = [ + LPCWSTR, # [in] lpUsername + LPCWSTR, # [in, optional] lpDomain + LPCWSTR, # [in] lpPassword + DWORD, # [in] dwLogonFlags + LPCWSTR, # [in, optional] lpApplicationName + LPWSTR, # [in, out, optional] lpCommandLine + DWORD, # [in] dwCreationFlags + LPVOID, # [in, optional] lpEnvironment + LPCWSTR, # [in, optional] lpCurrentDirectory + POINTER(STARTUPINFO), # [in] lpStartupInfo + POINTER(PROCESS_INFORMATION), # [out] lpProcessInformation +] + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createprocesswithtokenw +advapi32.CreateProcessWithTokenW.restype = BOOL +advapi32.CreateProcessWithTokenW.argtypes = [ + HANDLE, # [in] hToken + DWORD, # [in] dwLogonFlags + LPCWSTR, # [in, optional] lpApplicationName + LPWSTR, # [in, out, optional] lpCommandLine + DWORD, # [in] dwCreationFlags + LPVOID, # [in, optional] lpEnvironment + LPCWSTR, # [in, optional] lpCurrentDirectory + POINTER(STARTUPINFO), # [in] lpStartupInfo + POINTER(PROCESS_INFORMATION), # [out] lpProcessInformation +] + + +# https://learn.microsoft.com/en-us/windows/win32/api/securitybaseapi/nf-securitybaseapi-gettokeninformation +advapi32.GetTokenInformation.restype = BOOL +advapi32.GetTokenInformation.argtypes = [ + HANDLE, # [in] TokenHandle + DWORD, # [in] TokenInformationClass (actually an enum) + LPVOID, # [out, optional] TokenInformation + DWORD, # [in] TokenInformationLength + PDWORD, # [out] ReturnLength +] + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-logonuserw +advapi32.LogonUserW.restype = BOOL +advapi32.LogonUserW.argtypes = [ + LPCWSTR, # [in] lpszUsername + LPCWSTR, # [in, optional] lpszDomain + LPCWSTR, # [in, optional] lpszPassword + DWORD, # [in] dwLogonType + DWORD, # [in] dwLogonProvider + PHANDLE, # [out] phToken +] + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-lookupprivilegenamew +advapi32.LookupPrivilegeNameW.restype = BOOL +advapi32.LookupPrivilegeNameW.argtypes = [ + LPCWSTR, # [in, optional] lpSystemName + POINTER(LUID), # [in] lpLuid + LPWSTR, # [out] lpName + LPDWORD, # [in, out] cchName +] + +# https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-lookupprivilegevaluew +advapi32.LookupPrivilegeValueW.restype = BOOL +advapi32.LookupPrivilegeValueW.argtypes = [ + LPCWSTR, # [in, optional] lpSystemName + LPCWSTR, # [in] lpName + POINTER(LUID), # [out] lpLuid +] + +# https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-openprocesstoken +advapi32.OpenProcessToken.restype = BOOL +advapi32.OpenProcessToken.argtypes = [ + HANDLE, # [in] ProcessHandle, + DWORD, # [in] DesiredAccess + PHANDLE, # [out] TokenHandle +] + + +# exports: +AdjustTokenPrivileges = advapi32.AdjustTokenPrivileges +CreateProcessAsUserW = advapi32.CreateProcessAsUserW +CreateProcessWithLogonW = advapi32.CreateProcessWithLogonW +CreateProcessWithTokenW = advapi32.CreateProcessWithTokenW +GetTokenInformation = advapi32.GetTokenInformation +LogonUserW = advapi32.LogonUserW +LookupPrivilegeNameW = advapi32.LookupPrivilegeNameW +LookupPrivilegeValueW = advapi32.LookupPrivilegeValueW +OpenProcessToken = advapi32.OpenProcessToken + +# --------- +# From: userenv.dll +# --------- +userenv = ctypes.WinDLL("userenv") + +# https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-createenvironmentblock +userenv.CreateEnvironmentBlock.restype = BOOL +userenv.CreateEnvironmentBlock.argtypes = [ + POINTER(ctypes.c_void_p), # [in] lpEnvironment + HANDLE, # [in, optional] hToken + BOOL, # [in] bInherit +] + +# # https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-destroyenvironmentblock +userenv.DestroyEnvironmentBlock.restype = BOOL +userenv.DestroyEnvironmentBlock.argtypes = [ + ctypes.c_void_p, # [in] lpEnvironment +] + +# https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-expandenvironmentstringsforuserw +userenv.ExpandEnvironmentStringsForUserW.restype = BOOL +userenv.ExpandEnvironmentStringsForUserW.argtypes = [ + HANDLE, # [in, optional] hToken + LPCWSTR, # [in] lpSrc + LPWSTR, # [out] lpDest + DWORD, # [in] dwSize +] + +# https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-loaduserprofilew +userenv.LoadUserProfileW.restype = BOOL +userenv.LoadUserProfileW.argtypes = [ + HANDLE, # [in] hToken + POINTER(PROFILEINFO), # [in, out] lpProfileInfo +] + +# https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-unloaduserprofile +userenv.UnloadUserProfile.restype = BOOL +userenv.UnloadUserProfile.argtypes = [ + HANDLE, # [in] hToken + HANDLE, # [in] hProfile +] + +# exports: +CreateEnvironmentBlock = userenv.CreateEnvironmentBlock +DestroyEnvironmentBlock = userenv.DestroyEnvironmentBlock +ExpandEnvironmentStringsForUserW = userenv.ExpandEnvironmentStringsForUserW +LoadUserProfileW = userenv.LoadUserProfileW +UnloadUserProfile = userenv.UnloadUserProfile diff --git a/src/openjd/sessions/_v1/_win32/_helpers.py b/src/openjd/sessions/_v1/_win32/_helpers.py new file mode 100644 index 00000000..a16b1881 --- /dev/null +++ b/src/openjd/sessions/_v1/_win32/_helpers.py @@ -0,0 +1,202 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import sys + +# This assertion short-circuits mypy from type checking this module on platforms other than Windows +# https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks +assert sys.platform == "win32" + +import win32api +import win32con +from ctypes.wintypes import DWORD, HANDLE +from ctypes import ( + WinError, + byref, + cast, + c_void_p, + c_wchar, + c_wchar_p, + sizeof, +) +from contextlib import contextmanager +from typing import Generator, Optional + +from ._api import ( + # Constants + LOGON32_LOGON_INTERACTIVE, + LOGON32_PROVIDER_DEFAULT, + PI_NOUI, + PROFILEINFO, + # Functions + CloseHandle, + CreateEnvironmentBlock, + DestroyEnvironmentBlock, + GetCurrentProcessId, + LogonUserW, + ProcessIdToSessionId, + LoadUserProfileW, + UnloadUserProfile, +) + + +def get_process_user(): + """ + Returns the user name of the user running the current process. + """ + return win32api.GetUserNameEx(win32con.NameSamCompatible) + + +def get_current_process_session_id() -> int: + """ + Finds the Session ID of the current process, and returns it. + """ + proc_id = GetCurrentProcessId() + session_id = DWORD(0) + # Ignore the return value; will only fail if given a bad + # process id, and that's clearly impossible here. + ProcessIdToSessionId(proc_id, byref(session_id)) + return session_id.value + + +def logon_user(username: str, password: str) -> HANDLE: + """ + Attempt to logon as the given username & password. + Return a HANDLE to a logon_token. + + Note: + The caller *MUST* call CloseHandle on the returned value when done with it. + Handles are not automatically garbage collected. + + Raises: + OSError - If an error is encountered. + """ + hToken = HANDLE(0) + if not LogonUserW( + username, + None, # TODO - domain handling?? + password, + LOGON32_LOGON_INTERACTIVE, + LOGON32_PROVIDER_DEFAULT, + byref(hToken), + ): + raise WinError() + + return hToken + + +@contextmanager +def logon_user_context(username: str, password: str) -> Generator[HANDLE, None, None]: + """ + A context manager wrapper around logon_user(). This will automatically + Close the logon_token when the context manager is exited. + """ + hToken: Optional[HANDLE] = None + try: + hToken = logon_user(username, password) + yield hToken + finally: + if hToken is not None and not CloseHandle(hToken): + raise WinError() + + +def environment_block_for_user(logon_token: HANDLE) -> c_void_p: + """ + Create an Environment Block for a given logon_token and return it. + Per https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-createenvironmentblock + "The environment block is an array of null-terminated Unicode strings. The list ends with two nulls (\0\0)." + + Returns: + Pointer to the environment block + + Raises: + OSError - If there is an error creating the block + + Notes: + 1) The returned block *MUST* be deallocated with DestroyEnvironmentBlock when done + 2) Destroying an environment block while it is in use (e.g. while the process it was passed + to is still running) WILL result in a hard to debug crash in ntdll.dll. So, don't do that! + """ + environment = c_void_p() + if not CreateEnvironmentBlock(byref(environment), logon_token, False): + raise WinError() + return environment + + +@contextmanager +def environment_block_for_user_context(logon_token: HANDLE) -> Generator[c_void_p, None, None]: + """As environment_block_for_user, but as a context manager. This will automatically + Destroy the environment block when exiting the context. + """ + lp_environment: Optional[c_void_p] = None + try: + lp_environment = environment_block_for_user(logon_token) + yield lp_environment + finally: + if lp_environment is not None and not DestroyEnvironmentBlock(lp_environment): + raise WinError() + + +def environment_block_to_dict(block: c_void_p) -> dict[str, str]: + """Converts an environment block as returned from CreateEnvironmentBlock to a Python dict of key/value strings. + + An environment block is a void pointer. The pointer points to a sequence of strings. Each string is terminated with a + null character. The final string is terminated by an additional null character. + """ + assert block.value is not None + w_char_size = sizeof(c_wchar) + cur: int = block.value + key_val_str = cast(cur, c_wchar_p).value + env: dict[str, str] = {} + while key_val_str: + key, val = key_val_str.split("=", maxsplit=1) + env[key] = val + # Advance pointer by string length plus a null character + cur += (len(key_val_str) + 1) * w_char_size + key_val_str = cast(cur, c_wchar_p).value + + return env + + +def environment_block_from_dict(env: dict[str, str]) -> c_wchar_p: + """Converts a Python dictionary representation of an environment into a character buffer as expected by the + lpEnvironment argument to the CreateProcess* family of win32 functions. + + Note: The returned c_char_p is pointing to the internal contents of an immutable python string; that is + to say that it will be garbage collected, and the caller need not worry about deallocating it. + """ + # Create a string with null-character delimiters between each "key=value" string + null_delimited = "\0".join(f"{key}={value}" for key, value in env.items()) + # Add a final null-terminator character + env_block_str = null_delimited + "\0" + + return c_wchar_p(env_block_str) + + +def load_user_profile(token: HANDLE, username: str) -> PROFILEINFO: + """ + Load the user profile for the given logon token and user name + + NOTE: The caller *MUST* call unload_user_profile when finished with the user profile + + See: https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-loaduserprofilew + """ + profile_info = PROFILEINFO() + profile_info.dwSize = sizeof(PROFILEINFO) + profile_info.lpUserName = username + profile_info.dwFlags = PI_NOUI + profile_info.lpProfilePath = None + + if not LoadUserProfileW(token, byref(profile_info)): + raise WinError() + + return profile_info + + +def unload_user_profile(token: HANDLE, profile_info: PROFILEINFO) -> None: + """ + Unload the user profile for the given token and profile. + + See: https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-unloaduserprofile + """ + if not UnloadUserProfile(token, profile_info.hProfile): + raise WinError() diff --git a/src/openjd/sessions/_v1/_win32/_locate_executable.py b/src/openjd/sessions/_v1/_win32/_locate_executable.py new file mode 100644 index 00000000..db1ab86d --- /dev/null +++ b/src/openjd/sessions/_v1/_win32/_locate_executable.py @@ -0,0 +1,180 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import os +import sys +import shutil +from logging import INFO, getLogger +from logging.handlers import QueueHandler +from pathlib import Path +from queue import Empty, SimpleQueue +from threading import Lock +from typing import Optional, Sequence + +from .._session_user import SessionUser +from .._subprocess import LoggingSubprocess +from .._logging import LoggerAdapter + +_internal_logger_lock = Lock() +_internal_logger = getLogger("openjd_sessions_runner_base_internal_logger") +_internal_logger_adapter = LoggerAdapter(_internal_logger, extra=dict()) +_internal_logger.setLevel(INFO) +_internal_logger.propagate = False +_internal_logger_queue: SimpleQueue = SimpleQueue() +_internal_logger.addHandler(QueueHandler(_internal_logger_queue)) + + +def locate_windows_executable( + args: Sequence[str], + user: Optional[SessionUser], + os_env_vars: Optional[dict[str, Optional[str]]], + working_dir: str, +) -> Sequence[str]: + cmd_path = Path(args[0]) + if cmd_path.is_absolute(): + # If it's an absolute path (e.g. C:\Foo\Bar.exe or C:\Foo\Bar) then we just return + # and leave it up to the OS to resolve the executable's extention. + + # TODO: Do we actually still want to do the find as a check to see if the command exists & + # is executable? This would catch stuff like 'c:\Foo\test.ps1' as a command (which fails) + return args + + return_args = list(args) + if user is None: + return_args[0] = _locate_for_same_user(cmd_path, os_env_vars, working_dir) + else: + return_args[0] = _locate_for_other_user(cmd_path, os_env_vars, working_dir, user) + return return_args + + +def _get_path_var_for_shutil_which( + os_env_vars: Optional[dict[str, Optional[str]]], working_dir: str +): + path_var: Optional[str] = None + if os_env_vars: + env_var_keys = {k.lower(): k for k in os_env_vars} + path_var = os_env_vars.get(env_var_keys["path"]) if "path" in env_var_keys else None + if path_var is None: + path_var = os.environ.get("PATH", "") + path_var = "%s;%s" % (working_dir, path_var) + return path_var + + +def _locate_for_same_user( + command: Path, os_env_vars: Optional[dict[str, Optional[str]]], working_dir: str +) -> str: + path_var: Optional[str] = _get_path_var_for_shutil_which(os_env_vars, working_dir) + exe = str(shutil.which(str(command), path=path_var)) + if not exe: + raise RuntimeError("Could not find executable file: %s" % command) + return exe + + +def _locate_for_other_user( + command: Path, + os_env_vars: Optional[dict[str, Optional[str]]], + working_dir: str, + user: SessionUser, +) -> str: # pragma: nocover + # Running as a potentially different user, so it's possible that + # this process doesn't have read access to the executable file's location. + # Thus, we need to rely on running a subprocess as the user to be able + # to find the executable. + + if len(command.parts) > 1: + # Windows cannot find executables by relative location + # i.e. where "dir\test.bat" + # + # Even if that worked, we'd have to prepend the relative part of the command + # to the path and then search for only the command.name. But, we don't generally + # have the user's PATH env var value. + # + # So, for both of those reasons we just return the command and let the action fail out + # naturally. + return str(command) + + # Prevent issues that might arise by having multiple Actions trying to start up + # concurrently -- grab a lock. + with _internal_logger_lock: + # Drain the message queue to ensure nothing remains from previous runs. + try: + while True: + _internal_logger_queue.get(block=False) + except Empty: + pass # Will happen when the queue is fully empty + + # When running in a service context, we want to call the non-service Python binary + sys_executable = sys.executable.lower().replace("pythonservice.exe", "python.exe") + + # In the subprocess code, we avoid exit code 1 as that is returned if a Python exception is thrown. + exit_code_success = 2 + exit_code_could_not_find_exe = 3 + + path_var: Optional[str] = _get_path_var_for_shutil_which(os_env_vars, working_dir) + process = LoggingSubprocess( + logger=_internal_logger_adapter, + args=[ + sys_executable, + "-c", + # Command injection here is possible, but it's irrelevant. The command is running + # as the given user. No need for an attacker to be fancy here, they could just run + # the desired attack command directly in the job template. + "import shutil, sys, pathlib\n" + + f"cmd = shutil.which({str(command)!r}, path={path_var!r})\n" + + "if cmd:\n" + + " print(str(pathlib.Path(cmd).absolute()))\n" + + f" sys.exit({exit_code_success})\n" + + f"sys.exit({exit_code_could_not_find_exe})\n", + ], + user=user, + os_env_vars=os_env_vars, + working_dir=str(working_dir), + ) + process.run() # blocking call + exit_code = process.exit_code + + # We're seeing random errors when trying to run an Action's command immediately after this + # outside of Session 0; theory is that maybe this has something to do with running two + # CreateProcessWithLogonW calls back-to-back with little time inbetween. So, explicitly + # delete the process object to try to force some cleanup of handles that maybe help the + # profile get unloaded. (this seems like it might be doing the trick) + # Error: + # [WinError 1018] Illegal operation attempted on a registry key that has been marked for deletion + del process + + if exit_code == exit_code_could_not_find_exe: + raise RuntimeError(f"Could not find executable file: {command}") + + # Parse the output + try: + while True: + record = _internal_logger_queue.get(block=False) + message = record.getMessage() + if "Output:" in message: + break + + if exit_code == exit_code_success: + exe_record = _internal_logger_queue.get(block=False) + # The line of output with the result of 'shutil.which' is the location of the command + return exe_record.getMessage() + except Empty: + raise RuntimeError( + f"Could not run Python as user {user.user} to find executable {command} in PATH.\n" + + f"The host configuration must allow users to run {sys_executable}." + ) + + # Collect the error output from the subprocess + error_messages = [] + try: + while True: + record = _internal_logger_queue.get(block=False) + error_messages.append(record.getMessage()) + except Empty: + pass + + # Something went wrong in launching sys_executable. + # Because this scenario may be difficult to diagnose, we include more context. + raise RuntimeError( + f"Could not run Python as user {user.user} to find executable {command} in PATH.\n" + + f"The host configuration must allow users to run {sys_executable}.\n\nError output:\n" + + "\n".join(error_messages) + ) diff --git a/src/openjd/sessions/_v1/_win32/_popen_as_user.py b/src/openjd/sessions/_v1/_win32/_popen_as_user.py new file mode 100644 index 00000000..33bdfea3 --- /dev/null +++ b/src/openjd/sessions/_v1/_win32/_popen_as_user.py @@ -0,0 +1,255 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import os +import sys + +# This assertion short-circuits mypy from type checking this module on platforms other than Windows +# https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks +assert sys.platform == "win32" + +from typing import Any, Optional, cast +import ctypes +from ctypes.wintypes import HANDLE +from subprocess import list2cmdline, Popen +from subprocess import Handle # type: ignore # linter doesn't know it exists +import platform +from ._api import ( + # Constants + LOGON_WITH_PROFILE, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + STARTF_USESHOWWINDOW, + STARTF_USESTDHANDLES, + SW_HIDE, + # Structures + PROCESS_INFORMATION, + STARTUPINFO, + STARTUPINFOEX, + # Functions + CloseHandle, + CreateProcessAsUserW, + CreateProcessWithLogonW, + DestroyEnvironmentBlock, + UpdateProcThreadAttribute, +) +from ._helpers import ( + environment_block_for_user, + # environment_block_for_user_context, + environment_block_from_dict, + environment_block_to_dict, + logon_user_context, +) +from .._session_user import WindowsSessionUser + +if platform.python_implementation() != "CPython": + raise RuntimeError( + f"Not compatible with the {platform.python_implementation} of Python. Please use CPython." + ) + +CREATE_UNICODE_ENVIRONMENT = 0x00000400 +EXTENDED_STARTUPINFO_PRESENT = 0x00080000 + + +def inherit_handles(startup_info: STARTUPINFOEX, handles: tuple[int]) -> ctypes.Array: + """Set the given 'startup_info' to have the subprocess inherit the given handles, and only the + given handles. + + Returns: + - The allocated list of handles. Per the Win32 APIs, you must ensure that this buffer is + only deallocated *after* the attribute list in the startup_info has been deallocated. + """ + handles_list = (HANDLE * len(handles))() + for i, h in enumerate(handles): + handles_list[i] = h + if not UpdateProcThreadAttribute( + startup_info.lpAttributeList, + 0, # reserved and must be 0 + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + ctypes.byref(handles_list), + ctypes.sizeof(handles_list), + None, # reserved and must be null + None, # reserved and must be null + ): + raise ctypes.WinError() + return handles_list + + +class PopenWindowsAsUser(Popen): + """Class to run a process as another user on Windows. + Derived from Popen, it defines the _execute_child() method to call CreateProcessWithLogonW. + """ + + def __init__(self, user: WindowsSessionUser, *args: Any, **kwargs: Any): + """ + Arguments: + username (str): Name of user to run subprocess as + password (str): Password for username + args (Any): Popen constructor args + kwargs (Any): Popen constructor kwargs + https://docs.python.org/3/library/subprocess.html#popen-constructor + """ + self.user = user + super(PopenWindowsAsUser, self).__init__(*args, **kwargs) + + def _execute_child( + self, + args, + executable, + preexec_fn, + close_fds, + pass_fds, + cwd, + env, + startupinfo, + creationflags, + shell, + p2cread, + p2cwrite, + c2pread, + c2pwrite, + errread, + errwrite, + restore_signals, + start_new_session, + *additional_args, + **kwargs, + ): + """Execute program (MS Windows version). + Calls CreateProcessWithLogonW to run a process as another user. + """ + + assert not pass_fds, "pass_fds not supported on Windows." + + commandline = args if isinstance(args, str) else list2cmdline(args) + # CreateProcess* may modify the commandline, so copy it to a mutable buffer + cmdline = ctypes.create_unicode_buffer(commandline) + + if executable is not None: + executable = os.fsdecode(executable) + + if cwd is not None: + cwd = os.fsdecode(cwd) + + # Initialize structures + si = STARTUPINFO() + si.cb = ctypes.sizeof(STARTUPINFO) + pi = PROCESS_INFORMATION() + + use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) + if use_std_handles: + si.hStdInput = int(p2cread) + si.hStdOutput = int(c2pwrite) + si.hStdError = int(errwrite) + si.dwFlags |= STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW + # Ensure that the console window is hidden + si.wShowWindow = SW_HIDE + + sys.audit("subprocess.Popen", executable, args, cwd, env, self.user.user) + + def _merge_environment( + user_env: ctypes.c_void_p, env: dict[str, Optional[str]] + ) -> ctypes.c_wchar_p: + user_env_dict = cast(dict[str, Optional[str]], environment_block_to_dict(user_env)) + # All environment keys are converted to uppercase. This is because while Windows environment variables are case insensitive, the win32 api + # allows you to store multiple keys of mixed case, for example the win32 api would allow you + # to set both "PATH" and "Path" as environment variable keys. This leads to undefined behaviour when calling one of the keys. + user_env_dict = {k.upper(): v for k, v in user_env_dict.items()} + upper_cased_env = {k.upper(): v for k, v in env.items()} + user_env_dict.update(**upper_cased_env) + result = {k: v for k, v in user_env_dict.items() if v is not None} + return environment_block_from_dict(result) + + env_ptr = ctypes.c_void_p(0) + try: + if self.user.password is not None: + with logon_user_context(self.user.user, self.user.password) as logon_token: + env_ptr = environment_block_for_user(logon_token) + if env: + env_block = _merge_environment(env_ptr, env) + else: + env_block = env_ptr + + # https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createprocesswithlogonw + if not CreateProcessWithLogonW( + self.user.user, + None, # TODO: Domains not yet supported + self.user.password, + LOGON_WITH_PROFILE, + executable, + cmdline, + creationflags | CREATE_UNICODE_ENVIRONMENT, + env_block, + cwd, + ctypes.byref(si), + ctypes.byref(pi), + ): + # Raises: OSError + raise ctypes.WinError() + elif self.user.logon_token is not None: + siex = STARTUPINFOEX() + ctypes.memmove( + ctypes.pointer(siex.StartupInfo), ctypes.pointer(si), ctypes.sizeof(STARTUPINFO) + ) + siex.StartupInfo.cb = ctypes.sizeof(STARTUPINFOEX) + creationflags |= EXTENDED_STARTUPINFO_PRESENT + + handles_list: Optional[ctypes.Array] = None + handles_to_inherit = tuple(int(h) for h in (p2cread, c2pwrite, errwrite) if h != -1) + try: + # Allocate the lpAttributeList array of the STARTUPINFOEX structure so that it + # has sufficient space for a single attribute. We only have a single attribute that + # we're setting -- namely a PROC_THREAD_ATTRIBUTE_HANDLE_LIST that itself contains a list of handles -- + # so this is sufficient. + siex.allocate_attribute_list(1) + + # Note: We must ensure that 'handles_list' must persist until the + # attribute list is destroyed using DeleteProcThreadAttributeList. We do this by holding on + # to a reference to it until after the finally block of this try. + handles_list = inherit_handles( # noqa: F841 # ignore: assigned but not used + siex, handles_to_inherit + ) + + # From https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessasuserw + # If the lpEnvironment parameter is NULL, the new process inherits the environment of the calling process. + # CreateProcessAsUser does not automatically modify the environment block to include environment variables specific to + # the user represented by hToken. For example, the USERNAME and USERDOMAIN variables are inherited from the calling + # process if lpEnvironment is NULL. It is your responsibility to prepare the environment block for the new process and + # specify it in lpEnvironment. + + env_ptr = environment_block_for_user(self.user.logon_token) + if env: + env_block = _merge_environment(env_ptr, env) + else: + env_block = env_ptr + + if not CreateProcessAsUserW( + self.user.logon_token, + executable, + cmdline, + None, + None, + True, + creationflags | CREATE_UNICODE_ENVIRONMENT, + env_block, + cwd, + ctypes.byref(siex), + ctypes.byref(pi), + ): + # Raises: OSError + raise ctypes.WinError() + finally: + siex.deallocate_attribute_list() + else: + raise NotImplementedError("Unexpected case for WindowsSessionUser properties") + finally: + if env_ptr.value is not None: + DestroyEnvironmentBlock(env_ptr) + # Child is launched. Close the parent's copy of those pipe + # handles that only the child should have open. + self._close_pipe_fds(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) + + # Retain the process handle, but close the thread handle + CloseHandle(pi.hThread) + + self._child_created = True + self.pid = pi.dwProcessId + self._handle = Handle(pi.hProcess) diff --git a/src/openjd/sessions/_v1/py.typed b/src/openjd/sessions/_v1/py.typed new file mode 100644 index 00000000..7ef21167 --- /dev/null +++ b/src/openjd/sessions/_v1/py.typed @@ -0,0 +1 @@ +# Marker file that indicates this package supports typing diff --git a/test/openjd/sessions_v0/__init__.py b/test/openjd/sessions_v0/__init__.py new file mode 100644 index 00000000..8d929cc8 --- /dev/null +++ b/test/openjd/sessions_v0/__init__.py @@ -0,0 +1 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/openjd/sessions/conftest.py b/test/openjd/sessions_v0/conftest.py similarity index 100% rename from test/openjd/sessions/conftest.py rename to test/openjd/sessions_v0/conftest.py diff --git a/test/openjd/sessions_v0/support_files/__init__.py b/test/openjd/sessions_v0/support_files/__init__.py new file mode 100644 index 00000000..8d929cc8 --- /dev/null +++ b/test/openjd/sessions_v0/support_files/__init__.py @@ -0,0 +1 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/openjd/sessions/support_files/app_20s_run.py b/test/openjd/sessions_v0/support_files/app_20s_run.py similarity index 100% rename from test/openjd/sessions/support_files/app_20s_run.py rename to test/openjd/sessions_v0/support_files/app_20s_run.py diff --git a/test/openjd/sessions/support_files/app_20s_run.sh b/test/openjd/sessions_v0/support_files/app_20s_run.sh old mode 100755 new mode 100644 similarity index 100% rename from test/openjd/sessions/support_files/app_20s_run.sh rename to test/openjd/sessions_v0/support_files/app_20s_run.sh diff --git a/test/openjd/sessions/support_files/app_20s_run_ignore_signal.py b/test/openjd/sessions_v0/support_files/app_20s_run_ignore_signal.py similarity index 100% rename from test/openjd/sessions/support_files/app_20s_run_ignore_signal.py rename to test/openjd/sessions_v0/support_files/app_20s_run_ignore_signal.py diff --git a/test/openjd/sessions/support_files/output_signal_sender.c b/test/openjd/sessions_v0/support_files/output_signal_sender.c similarity index 100% rename from test/openjd/sessions/support_files/output_signal_sender.c rename to test/openjd/sessions_v0/support_files/output_signal_sender.c diff --git a/test/openjd/sessions/support_files/run_app_20s_run.py b/test/openjd/sessions_v0/support_files/run_app_20s_run.py similarity index 100% rename from test/openjd/sessions/support_files/run_app_20s_run.py rename to test/openjd/sessions_v0/support_files/run_app_20s_run.py diff --git a/test/openjd/sessions/test_action_filter.py b/test/openjd/sessions_v0/test_action_filter.py similarity index 100% rename from test/openjd/sessions/test_action_filter.py rename to test/openjd/sessions_v0/test_action_filter.py diff --git a/test/openjd/sessions/test_embedded_files.py b/test/openjd/sessions_v0/test_embedded_files.py similarity index 100% rename from test/openjd/sessions/test_embedded_files.py rename to test/openjd/sessions_v0/test_embedded_files.py diff --git a/test/openjd/sessions/test_importable.py b/test/openjd/sessions_v0/test_importable.py similarity index 100% rename from test/openjd/sessions/test_importable.py rename to test/openjd/sessions_v0/test_importable.py diff --git a/test/openjd/sessions/test_os_checker.py b/test/openjd/sessions_v0/test_os_checker.py similarity index 100% rename from test/openjd/sessions/test_os_checker.py rename to test/openjd/sessions_v0/test_os_checker.py diff --git a/test/openjd/sessions/test_path_mapping.py b/test/openjd/sessions_v0/test_path_mapping.py similarity index 100% rename from test/openjd/sessions/test_path_mapping.py rename to test/openjd/sessions_v0/test_path_mapping.py diff --git a/test/openjd/sessions/test_redacted_env.py b/test/openjd/sessions_v0/test_redacted_env.py similarity index 100% rename from test/openjd/sessions/test_redacted_env.py rename to test/openjd/sessions_v0/test_redacted_env.py diff --git a/test/openjd/sessions/test_redaction.py b/test/openjd/sessions_v0/test_redaction.py similarity index 100% rename from test/openjd/sessions/test_redaction.py rename to test/openjd/sessions_v0/test_redaction.py diff --git a/test/openjd/sessions/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py similarity index 100% rename from test/openjd/sessions/test_runner_base.py rename to test/openjd/sessions_v0/test_runner_base.py diff --git a/test/openjd/sessions/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py similarity index 100% rename from test/openjd/sessions/test_runner_env_script.py rename to test/openjd/sessions_v0/test_runner_env_script.py diff --git a/test/openjd/sessions/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py similarity index 100% rename from test/openjd/sessions/test_runner_step_script.py rename to test/openjd/sessions_v0/test_runner_step_script.py diff --git a/test/openjd/sessions/test_session.py b/test/openjd/sessions_v0/test_session.py similarity index 100% rename from test/openjd/sessions/test_session.py rename to test/openjd/sessions_v0/test_session.py diff --git a/test/openjd/sessions/test_session_run_subprocess.py b/test/openjd/sessions_v0/test_session_run_subprocess.py similarity index 100% rename from test/openjd/sessions/test_session_run_subprocess.py rename to test/openjd/sessions_v0/test_session_run_subprocess.py diff --git a/test/openjd/sessions/test_session_user.py b/test/openjd/sessions_v0/test_session_user.py similarity index 100% rename from test/openjd/sessions/test_session_user.py rename to test/openjd/sessions_v0/test_session_user.py diff --git a/test/openjd/sessions/test_subprocess.py b/test/openjd/sessions_v0/test_subprocess.py similarity index 100% rename from test/openjd/sessions/test_subprocess.py rename to test/openjd/sessions_v0/test_subprocess.py diff --git a/test/openjd/sessions/test_tempdir.py b/test/openjd/sessions_v0/test_tempdir.py similarity index 100% rename from test/openjd/sessions/test_tempdir.py rename to test/openjd/sessions_v0/test_tempdir.py diff --git a/test/openjd/sessions/test_windows_process_killer.py b/test/openjd/sessions_v0/test_windows_process_killer.py similarity index 100% rename from test/openjd/sessions/test_windows_process_killer.py rename to test/openjd/sessions_v0/test_windows_process_killer.py diff --git a/test/openjd/sessions_v1/__init__.py b/test/openjd/sessions_v1/__init__.py new file mode 100644 index 00000000..8d929cc8 --- /dev/null +++ b/test/openjd/sessions_v1/__init__.py @@ -0,0 +1 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/openjd/sessions_v1/conftest.py b/test/openjd/sessions_v1/conftest.py new file mode 100644 index 00000000..c19f11d2 --- /dev/null +++ b/test/openjd/sessions_v1/conftest.py @@ -0,0 +1,22 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import sys +import pytest + + +def pytest_collection_modifyitems(config, items): + mark_expr = config.getoption("markexpr", False) + if not mark_expr: + config.option.markexpr = "not requires_cap_kill" + else: + config.option.markexpr = mark_expr + + +@pytest.fixture(scope="function") +def session_id() -> str: + return "some Id" + + +@pytest.fixture(scope="function") +def python_exe() -> str: + return sys.executable diff --git a/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_scenario.yaml b/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_scenario.yaml new file mode 100644 index 00000000..33e5b923 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_scenario.yaml @@ -0,0 +1,21 @@ +name: "Env.File.* available in environment script let bindings" +description: > + Verifies that Env.File.* symbols are available in EnvironmentScript let bindings, + allowing let bindings to coerce the file path to a path type and access its properties. + The embedded file content should also be written correctly using the full symbol table. + +run_on: all + +job_template_file: "env_file_let_bindings_template.yaml" + +job_parameters: + Greeting: "world" + +expect: + success: true + output_contains: + - "env_file_exists=True" + - "env_file_content=greeting=world" + - "env_file_has_parent=true" + - "env_exit=ok" + - "task_ran=ok" diff --git a/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_template.yaml b/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_template.yaml new file mode 100644 index 00000000..68aa8006 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/env_file_let_bindings/env_file_let_bindings_template.yaml @@ -0,0 +1,45 @@ +specificationVersion: jobtemplate-2023-09 +name: Env File Let Bindings Test +extensions: + - EXPR +parameterDefinitions: + - name: Greeting + type: STRING + default: "hello" +jobEnvironments: + - name: TestEnv + script: + embeddedFiles: + - name: config + type: TEXT + data: "greeting={{Param.Greeting}}" + let: + - "cfg = Env.File.config" + - "cfg_has_parent = len(string(cfg.parent)) > 0" + actions: + onEnter: + command: python + args: + - "-c" + - | + import os + path = r'{{ cfg }}' + exists = os.path.isfile(path) + content = open(path).read() if exists else '' + print(f'env_file_exists={exists}') + print(f'env_file_content={content}') + print(r'env_file_has_parent={{ cfg_has_parent }}') + onExit: + command: python + args: + - "-c" + - "print('env_exit=ok')" +steps: + - name: Step1 + script: + actions: + onRun: + command: python + args: + - "-c" + - "print('task_ran=ok')" diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_scenario.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_scenario.yaml new file mode 100644 index 00000000..ab113339 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_scenario.yaml @@ -0,0 +1,77 @@ +name: "Let bindings comprehensive test" +description: "Verifies step-level and script-level let bindings with various types and chaining" + +run_on: all + +job_template_file: "let_bindings_template.yaml" + +job_parameters: + BaseInt: 10 + BaseFloat: 2.5 + BasePath: "/input/scene.exr" + BaseString: "hello" + Items: [1, 2, 3, 4, 5] + FrameRange: "1-10" + Tags: ["alpha", "beta", "gamma"] + InputFiles: ["/data/file1.exr", "/data/file2.exr", "/data/file3.exr"] + +expect: + success: true + output_contains: + # Job environment let bindings + - "job_env_enter_msg=JobEnv onEnter: hello" + - "job_env_exit_msg=JobEnv onExit: hello_done" + - "job_env_computed=110" + # Step environment let bindings + - "step_env_enter_msg=StepEnv onEnter: hello" + - "step_env_exit_msg=StepEnv onExit: hello_done" + - "step_env_doubled=20" + # Embedded file with let bindings (frame 1) + - "output_name=hello_world_1.exr" + - "combined_value=21" + # Embedded file with let bindings (frame 2) + - "output_name=hello_world_2.exr" + - "combined_value=22" + # Step-level bindings (BaseInt=10, so step_int=20) + - "step_int=20" + - "step_float=3.0" + - "step_string=hello_world" + - "step_list_sum=15" + - "step_chained=25" + - "step_conditional=big" + - "step_path_stem=scene" + - "step_list_len=5" + - "step_min=1" + - "step_max=5" + # Boolean and comparison operations + - "step_bool_and=true" + - "step_bool_or=true" + - "step_comparison=true" + - "step_chained_compare=true" + # List comprehension + - "step_doubled=[2, 4, 6, 8, 10]" + # Nested path operations + - "step_path_suffix=.exr" + # RANGE_EXPR bindings + - "step_range_len=10" + # LIST[STRING] bindings + - "step_tags_len=3" + - "step_first_tag=alpha" + - 'step_tags_upper=["alpha_TAG", "beta_TAG", "gamma_TAG"]' + # LIST[PATH] bindings + - "step_files_len=3" + - "step_first_file_stem=file1" + - 'step_file_stems=["file1", "file2", "file3"]' + # Task-level bindings for frame 1 (step_int//10 = 2, so frames 1-2) + - "task_frame=1" + - "task_combined=21" + - "task_output_name=hello_world_1.exr" + - "task_msg=Frame 1 of 2" + # Task-level bindings for frame 2 + - "task_frame=2" + - "task_combined=22" + - "task_output_name=hello_world_2.exr" + - "task_msg=Frame 2 of 2" + # Direct param access + - "Param.BaseInt=10" + - "Param.BasePath.stem=scene" diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_template.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_template.yaml new file mode 100644 index 00000000..ac7ebb55 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/let_bindings_template.yaml @@ -0,0 +1,183 @@ +# Standalone template - run with: openjd run let_bindings_template.yaml +specificationVersion: jobtemplate-2023-09 +name: Let Bindings Comprehensive Test +extensions: + - EXPR +parameterDefinitions: + - name: BaseInt + type: INT + default: 10 + - name: BaseFloat + type: FLOAT + default: 2.5 + - name: BasePath + type: PATH + default: "/input/scene.exr" + - name: BaseString + type: STRING + default: "hello" + - name: Items + type: LIST[INT] + default: [1, 2, 3, 4, 5] + - name: FrameRange + type: RANGE_EXPR + default: "1-10" + - name: Tags + type: LIST[STRING] + default: ["alpha", "beta", "gamma"] + - name: InputFiles + type: LIST[PATH] + default: ["/data/file1.exr", "/data/file2.exr", "/data/file3.exr"] + +# Job-level environment with let bindings +jobEnvironments: + - name: JobEnv + script: + let: + - 'job_env_enter_msg = "JobEnv onEnter: " + Param.BaseString' + - 'job_env_exit_msg = "JobEnv onExit: " + Param.BaseString + "_done"' + - job_env_computed = Param.BaseInt + 100 + actions: + onEnter: + command: bash + args: + - "-c" + - | + echo "=== Job Environment onEnter ===" + echo "job_env_enter_msg={{job_env_enter_msg}}" + echo "job_env_computed={{job_env_computed}}" + onExit: + command: bash + args: + - "-c" + - | + echo "=== Job Environment onExit ===" + echo "job_env_exit_msg={{job_env_exit_msg}}" + +steps: + - name: TestLetBindings + # Step-level environment with let bindings + stepEnvironments: + - name: StepEnv + script: + let: + - 'step_env_enter_msg = "StepEnv onEnter: " + Param.BaseString' + - 'step_env_exit_msg = "StepEnv onExit: " + Param.BaseString + "_done"' + - step_env_doubled = Param.BaseInt * 2 + actions: + onEnter: + command: bash + args: + - "-c" + - | + echo "=== Step Environment onEnter ===" + echo "step_env_enter_msg={{step_env_enter_msg}}" + echo "step_env_doubled={{step_env_doubled}}" + onExit: + command: bash + args: + - "-c" + - | + echo "=== Step Environment onExit ===" + echo "step_env_exit_msg={{step_env_exit_msg}}" + # Step-level let bindings - evaluated once per step at job creation + let: + - step_int = Param.BaseInt * 2 + - step_float = Param.BaseFloat + 0.5 + - step_string = Param.BaseString + "_world" + - step_list_sum = sum(Param.Items) + - step_chained = step_int + 5 + - step_conditional = "big" if step_int > 15 else "small" + - step_list_len = len(Param.Items) + - step_min = min(Param.Items) + - step_max = max(Param.Items) + # Boolean and comparison operations + - step_bool_and = step_int > 10 and step_float > 2.0 + - step_bool_or = step_int < 10 or step_float > 2.0 + - step_comparison = step_int == 20 + - step_chained_compare = 0 < step_min <= step_max < 10 + # List comprehension + - step_doubled = [x * 2 for x in Param.Items] + # RANGE_EXPR operations - can be used in range expressions + - step_range_len = len(Param.FrameRange) + # LIST[STRING] operations + - step_tags_len = len(Param.Tags) + - step_first_tag = Param.Tags[0] + - step_tags_upper = [t + "_TAG" for t in Param.Tags] + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: INT + range: "1-{{ step_int // 10 }}" + script: + # Script-level let bindings - evaluated at task runtime, can use Task.Param + # PATH and LIST[PATH] ``Param.*`` are also visible here (task scope); + # they're NOT visible in step-level let bindings (template scope), so + # bindings that touch PATH-typed parameters live here even though the + # asserted output names are ``step_*``. + let: + - task_frame = Task.Param.Frame + - task_combined = step_int + Task.Param.Frame + - task_output_name = step_string + "_" + string(Task.Param.Frame) + ".exr" + - task_msg = "Frame " + string(Task.Param.Frame) + " of " + string(step_int // 10) + # PATH operations on Param.BasePath (PATH-typed; task scope only). + - step_path = Param.BasePath.parent / "output" + - step_path_stem = Param.BasePath.stem + - step_path_suffix = Param.BasePath.suffix + # LIST[PATH] operations on Param.InputFiles (LIST_PATH; task scope only). + - step_files_len = len(Param.InputFiles) + - step_first_file_stem = Param.InputFiles[0].stem + - step_file_stems = [f.stem for f in Param.InputFiles] + # Embedded file using let binding values + embeddedFiles: + - name: config + type: TEXT + filename: "config_{{task_frame}}.txt" + data: | + # Config for frame {{task_frame}} + output_name={{task_output_name}} + combined_value={{task_combined}} + actions: + onRun: + command: bash + args: + - "-c" + - | + echo "=== Embedded file content ===" + cat {{repr_sh(Task.File.config)}} + echo "=== Step-level bindings ===" + echo step_int={{repr_sh(string(step_int))}} + echo step_float={{repr_sh(string(step_float))}} + echo step_path={{repr_sh(step_path)}} + echo step_string={{repr_sh(step_string)}} + echo step_list_sum={{repr_sh(string(step_list_sum))}} + echo step_chained={{repr_sh(string(step_chained))}} + echo step_conditional={{repr_sh(step_conditional)}} + echo step_path_stem={{repr_sh(step_path_stem)}} + echo step_list_len={{repr_sh(string(step_list_len))}} + echo step_min={{repr_sh(string(step_min))}} + echo step_max={{repr_sh(string(step_max))}} + echo step_bool_and={{repr_sh(string(step_bool_and))}} + echo step_bool_or={{repr_sh(string(step_bool_or))}} + echo step_comparison={{repr_sh(string(step_comparison))}} + echo step_chained_compare={{repr_sh(string(step_chained_compare))}} + echo step_doubled={{repr_sh(string(step_doubled))}} + echo step_path_suffix={{repr_sh(step_path_suffix)}} + echo "=== RANGE_EXPR bindings ===" + echo step_range_len={{repr_sh(string(step_range_len))}} + echo "=== LIST[STRING] bindings ===" + echo step_tags_len={{repr_sh(string(step_tags_len))}} + echo step_first_tag={{repr_sh(step_first_tag)}} + echo step_tags_upper={{repr_sh(string(step_tags_upper))}} + echo "=== LIST[PATH] bindings ===" + echo step_files_len={{repr_sh(string(step_files_len))}} + echo step_first_file_stem={{repr_sh(step_first_file_stem)}} + echo step_file_stems={{repr_sh(string(step_file_stems))}} + echo "=== Task-level bindings ===" + echo task_frame={{repr_sh(string(task_frame))}} + echo task_combined={{repr_sh(string(task_combined))}} + echo task_output_name={{repr_sh(task_output_name)}} + echo task_msg={{repr_sh(task_msg)}} + echo "=== Direct param access ===" + echo Param.BaseInt={{repr_sh(string(Param.BaseInt))}} + echo Param.BasePath.stem={{repr_sh(Param.BasePath.stem)}} diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_scenario.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_scenario.yaml new file mode 100644 index 00000000..b4c17653 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_scenario.yaml @@ -0,0 +1,17 @@ +name: "Let bindings with host-context symbols" +description: "Verifies that let bindings in script, environment, and simple action contexts can reference Session.*, Task.Param.*, and other host-context symbols" + +run_on: posix + +job_template_file: "let_host_context_template.yaml" + +job_parameters: + OutputDir: "/renders" + +expect: + success: true + output_contains: + - "ENV_WORK_OK=true" + - "WORK_OUT_OK=true" + - "FRAME_PATH=/renders/1" + - "FRAME_PATH=/renders/2" diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_template.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_template.yaml new file mode 100644 index 00000000..a7590ac8 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/let_host_context_template.yaml @@ -0,0 +1,53 @@ +specificationVersion: jobtemplate-2023-09 +name: Let Bindings Host Context Symbols +extensions: + - EXPR + - FEATURE_BUNDLE_1 +parameterDefinitions: + - name: OutputDir + type: PATH + default: "/renders" +steps: + - name: ScriptLetWithSession + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: INT + range: "1-2" + stepEnvironments: + - name: Setup + script: + let: + - env_work = Session.WorkingDirectory / 'env_output' + actions: + onEnter: + command: python + args: + - "-c" + - | + import os + p = r'{{env_work}}' + assert p.endswith('env_output'), f'bad env_work: {p}' + print(f'ENV_WORK_OK=true') + script: + let: + - work_out = Session.WorkingDirectory / 'output' + - frame_path = Param.OutputDir / string(Task.Param.Frame) + actions: + onRun: + command: python + args: + - "-c" + - | + import os + w = r'{{work_out}}' + assert w.endswith('output'), f'bad work_out: {w}' + print(f'WORK_OUT_OK=true') + print(f'FRAME_PATH={{frame_path}}') + + - name: SimpleActionLetWithSession + bash: + let: + - sa_work = Session.WorkingDirectory / 'simple' + script: | + echo "SA_WORK_ENDS=$(echo {{repr_sh(sa_work)}} | grep -c 'simple')" diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_scenario.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_scenario.yaml new file mode 100644 index 00000000..74a1424f --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_scenario.yaml @@ -0,0 +1,30 @@ +name: "Step-level let bindings available in stepEnvironments" +description: > + Verifies that step-level let bindings are available in stepEnvironment + variables, onEnter scripts, and onExit scripts. + +run_on: posix + +job_template_file: "step_let_in_step_env_template.yaml" + +job_parameters: + BaseValue: 42 + +expect: + success: true + output_contains: + # onEnter sees step-level let bindings + - "enter_base=42" + - "enter_doubled=84" + - "enter_label=step_42" + # onExit sees step-level let bindings + - "exit_base=42" + - "exit_doubled=84" + - "exit_label=step_42" + # Environment variables resolved from step-level let bindings + - "env_MY_BASE=42" + - "env_MY_DOUBLED=84" + - "env_MY_LABEL=step_42" + # Task also sees step-level let bindings + - "task_base=42" + - "task_doubled=84" diff --git a/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_template.yaml b/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_template.yaml new file mode 100644 index 00000000..847474d5 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/let_bindings/step_let_in_step_env_template.yaml @@ -0,0 +1,56 @@ +specificationVersion: jobtemplate-2023-09 +name: Step Let Bindings in Step Environments +extensions: + - EXPR +parameterDefinitions: + - name: BaseValue + type: INT + default: 42 +steps: + - name: TestStep + let: + - base = Param.BaseValue + - doubled = base * 2 + - label = "step_" + string(base) + stepEnvironments: + - name: TestEnv + script: + actions: + onEnter: + command: bash + args: + - "-c" + - | + echo "enter_base={{base}}" + echo "enter_doubled={{doubled}}" + echo "enter_label={{label}}" + onExit: + command: bash + args: + - "-c" + - | + echo "exit_base={{base}}" + echo "exit_doubled={{doubled}}" + echo "exit_label={{label}}" + - name: VarEnv + variables: + MY_BASE: "{{ base }}" + MY_DOUBLED: "{{ doubled }}" + MY_LABEL: "{{ label }}" + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo "task_base={{base}}" + echo "task_doubled={{doubled}}" + echo "env_MY_BASE=$MY_BASE" + echo "env_MY_DOUBLED=$MY_DOUBLED" + echo "env_MY_LABEL=$MY_LABEL" + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: INT + range: [1] diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_scenario.yaml new file mode 100644 index 00000000..cdba1ead --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_scenario.yaml @@ -0,0 +1,26 @@ +name: "CHUNK[INT] task parameter with expressions" +description: "Verifies CHUNK[INT] task parameters work correctly with list(), len(), and sum() expressions" + +run_on: all + +job_template_file: "all_task_param_types_template.yaml" + +job_parameters: + Multiplier: 10 + +# Only run the CHUNK[INT] step to keep test focused +step: TestChunkInt + +expect: + success: true + output_contains: + # Verify CHUNK[INT] works with list() conversion + - "CHUNK-INT as list: [1, 2, 3, 4, 5]" + - "CHUNK-INT count: 5" + - "CHUNK-INT first: 1" + - "CHUNK-INT sum: 15" + # Second chunk + - "CHUNK-INT as list: [6, 7, 8, 9, 10]" + - "CHUNK-INT count: 5" + - "CHUNK-INT first: 6" + - "CHUNK-INT sum: 40" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_template.yaml new file mode 100644 index 00000000..ee6ae090 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/all_task_param_types_template.yaml @@ -0,0 +1,61 @@ +# Tests all task parameter types with expressions +specificationVersion: jobtemplate-2023-09 +name: All Task Parameter Types Test +extensions: + - EXPR + - TASK_CHUNKING +parameterDefinitions: + - name: Multiplier + type: INT + default: 10 +steps: + - name: TestAllTypes + parameterSpace: + taskParameterDefinitions: + - name: IntParam + type: INT + range: "1-2" + - name: FloatParam + type: FLOAT + range: [1.5, 2.5] + - name: StringParam + type: STRING + range: ["alpha", "beta"] + - name: PathParam + type: PATH + range: ["/tmp/a.txt", "/tmp/b.txt"] + combination: "(IntParam, FloatParam, StringParam, PathParam)" + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo INT: {{repr_sh(string(Task.Param.IntParam))}} x {{repr_sh(string(Param.Multiplier))}} = {{repr_sh(string(Task.Param.IntParam * Param.Multiplier))}} + echo FLOAT: {{repr_sh(string(Task.Param.FloatParam))}} + echo STRING: {{repr_sh(Task.Param.StringParam.upper())}} + echo PATH: {{repr_sh(Task.Param.PathParam)}} + echo PATH.stem: {{repr_sh(Task.Param.PathParam.stem)}} + + - name: TestChunkInt + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: CHUNK[INT] + range: "1-10" + chunks: + defaultTaskCount: 5 + rangeConstraint: CONTIGUOUS + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo CHUNK-INT range: {{repr_sh(string(Task.Param.Frame))}} + echo CHUNK-INT as list: {{repr_sh(string(list(Task.Param.Frame)))}} + echo CHUNK-INT count: {{repr_sh(string(len(Task.Param.Frame)))}} + echo CHUNK-INT first: {{repr_sh(string(list(Task.Param.Frame)[0]))}} + echo CHUNK-INT sum: {{repr_sh(string(sum(list(Task.Param.Frame))))}} diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/all_types_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/all_types_scenario.yaml new file mode 100644 index 00000000..a6e43f81 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/all_types_scenario.yaml @@ -0,0 +1,44 @@ +name: "All parameter types basic test" +description: "Verifies all parameter types work correctly in expressions" + +run_on: all + +job_template_file: "all_types_template.yaml" + +job_parameters: + StringParam: "world" + IntParam: 100 + FloatParam: 2.5 + PathParam: "/test/path/file.txt" + BoolParam: false + RangeExprParam: "10-15" + ListStringParam: ["x", "y", "z"] + ListIntParam: [10, 20, 30] + ListFloatParam: [1.5, 2.5] + ListPathParam: ["/path/a.txt", "/path/b.txt"] + ListBoolParam: [false, true] + ListListIntParam: [[5, 6], [7, 8, 9]] + +expect: + success: true + output_contains: + - "StringParam=world" + - "IntParam=100" + - "IntParam+1=101" + - "FloatParam=2.5" + - "PathParam.name=file.txt" + - "BoolParam=false" + - "RangeExprParam=10-15" + - "len-RangeExprParam=6" + - "ListStringParam-0=x" + - "sum-ListIntParam=60" + - "ListPathParam-0-name=a.txt" + - "ListListIntParam-0-1=6" + +expect_posix: + output_contains: + - "PathParam=/test/path/file.txt" + +expect_windows: + output_contains: + - "PathParam=\\test\\path\\file.txt" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/all_types_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/all_types_template.yaml new file mode 100644 index 00000000..48d5a29e --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/all_types_template.yaml @@ -0,0 +1,70 @@ +# Standalone template - run with: openjd run all_types_template.yaml +specificationVersion: jobtemplate-2023-09 +name: All Parameter Types Test +extensions: + - EXPR +parameterDefinitions: + - name: StringParam + type: STRING + default: "hello" + - name: IntParam + type: INT + default: 42 + - name: FloatParam + type: FLOAT + default: 3.14 + - name: PathParam + type: PATH + default: "default/path" + - name: BoolParam + type: BOOL + default: true + - name: RangeExprParam + type: RANGE_EXPR + default: "1-5" + - name: ListStringParam + type: LIST[STRING] + default: ["a", "b", "c"] + - name: ListIntParam + type: LIST[INT] + default: [1, 2, 3] + - name: ListFloatParam + type: LIST[FLOAT] + default: [1.1, 2.2] + - name: ListPathParam + type: LIST[PATH] + default: ["path/one", "path/two"] + - name: ListBoolParam + type: LIST[BOOL] + default: [true, false] + - name: ListListIntParam + type: LIST[LIST[INT]] + default: [[1, 2], [3, 4]] +steps: + - name: TestAllTypes + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo StringParam={{repr_sh(Param.StringParam)}} + echo IntParam={{repr_sh(string(Param.IntParam))}} + echo IntParam+1={{repr_sh(string(Param.IntParam + 1))}} + echo FloatParam={{repr_sh(string(Param.FloatParam))}} + echo PathParam={{repr_sh(Param.PathParam)}} + echo PathParam.name={{repr_sh(Param.PathParam.name)}} + echo BoolParam={{repr_sh(string(Param.BoolParam))}} + echo RangeExprParam={{repr_sh(Param.RangeExprParam)}} + echo len-RangeExprParam={{repr_sh(string(len(Param.RangeExprParam)))}} + echo ListStringParam={{repr_sh(string(Param.ListStringParam))}} + echo ListStringParam-0={{repr_sh(Param.ListStringParam[0])}} + echo ListIntParam={{repr_sh(string(Param.ListIntParam))}} + echo sum-ListIntParam={{repr_sh(string(sum(Param.ListIntParam)))}} + echo ListFloatParam={{repr_sh(string(Param.ListFloatParam))}} + echo ListPathParam={{repr_sh(string(Param.ListPathParam))}} + echo ListPathParam-0-name={{repr_sh(Param.ListPathParam[0].name)}} + echo ListBoolParam={{repr_sh(string(Param.ListBoolParam))}} + echo ListListIntParam={{repr_sh(string(Param.ListListIntParam))}} + echo ListListIntParam-0-1={{repr_sh(string(Param.ListListIntParam[0][1]))}} diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_posix_to_win_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_posix_to_win_scenario.yaml new file mode 100644 index 00000000..51e6b0a3 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_posix_to_win_scenario.yaml @@ -0,0 +1,26 @@ +name: "apply_path_mapping posix-to-windows on RawParam produces Param" +description: "Verifies apply_path_mapping(RawParam.X) equals Param.X with posix-to-windows mapping" + +run_on: windows + +job_template_file: "apply_path_mapping_win_to_posix_template.yaml" + +job_parameters: + InputPath: "/home/test/file.txt" + +path_mapping_rules: + - source_path_format: posix + source_path: "/home" + destination_path: "C:\\Users" + +expect: + success: true + output_contains: + - 'Param.InputPath="C:\Users\test\file.txt"' + - 'RawParam.InputPath="/home/test/file.txt"' + - 'apply_path_mapping(RawParam.InputPath)="C:\Users\test\file.txt"' + - 'RawParam.InputPath.apply_path_mapping()="C:\Users\test\file.txt"' + - 'Param.InputPath.parent="C:\Users\test"' + - 'apply_path_mapping(RawParam.InputPath).parent="C:\Users\test"' + - 'Param.InputPath.name="file.txt"' + - 'apply_path_mapping(RawParam.InputPath).name="file.txt"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_scenario.yaml new file mode 100644 index 00000000..0ca8aa84 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_scenario.yaml @@ -0,0 +1,26 @@ +name: "apply_path_mapping on RawParam produces Param" +description: "Verifies apply_path_mapping(RawParam.X) equals Param.X" + +run_on: posix + +job_template_file: "apply_path_mapping_win_to_posix_template.yaml" + +job_parameters: + InputPath: "C:\\Users\\test\\file.txt" + +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Users" + destination_path: "/home" + +expect: + success: true + output_contains: + - 'Param.InputPath="/home/test/file.txt"' + - 'RawParam.InputPath="C:\\Users\\test\\file.txt"' + - 'apply_path_mapping(RawParam.InputPath)="/home/test/file.txt"' + - 'RawParam.InputPath.apply_path_mapping()="/home/test/file.txt"' + - 'Param.InputPath.parent="/home/test"' + - 'apply_path_mapping(RawParam.InputPath).parent="/home/test"' + - 'Param.InputPath.name="file.txt"' + - 'apply_path_mapping(RawParam.InputPath).name="file.txt"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_template.yaml new file mode 100644 index 00000000..30a3092e --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/apply_path_mapping_win_to_posix_template.yaml @@ -0,0 +1,25 @@ +specificationVersion: jobtemplate-2023-09 +name: apply_path_mapping Test +extensions: + - EXPR +parameterDefinitions: + - name: InputPath + type: PATH + default: "default/input/path" +steps: + - name: TestApplyPathMapping + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo 'Param.InputPath={{repr_json(Param.InputPath)}}' + echo 'RawParam.InputPath={{repr_json(RawParam.InputPath)}}' + echo 'apply_path_mapping(RawParam.InputPath)={{repr_json(apply_path_mapping(RawParam.InputPath))}}' + echo 'RawParam.InputPath.apply_path_mapping()={{repr_json(RawParam.InputPath.apply_path_mapping())}}' + echo 'Param.InputPath.parent={{repr_json(Param.InputPath.parent)}}' + echo 'apply_path_mapping(RawParam.InputPath).parent={{repr_json(apply_path_mapping(RawParam.InputPath).parent)}}' + echo 'Param.InputPath.name={{repr_json(Param.InputPath.name)}}' + echo 'apply_path_mapping(RawParam.InputPath).name={{repr_json(apply_path_mapping(RawParam.InputPath).name)}}' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/basic_task_param_types_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/basic_task_param_types_scenario.yaml new file mode 100644 index 00000000..11f8f231 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/basic_task_param_types_scenario.yaml @@ -0,0 +1,35 @@ +name: "Basic task parameter types with expressions" +description: "Verifies INT, FLOAT, STRING, and PATH task parameters work correctly with expressions" + +run_on: all + +job_template_file: "all_task_param_types_template.yaml" + +job_parameters: + Multiplier: 10 + +step: TestAllTypes + +expect: + success: true + output_contains: + # First task: IntParam=1, FloatParam=1.5, StringParam=alpha, PathParam=/tmp/a.txt + - "INT: 1 x 10 = 10" + - "FLOAT: 1.5" + - "STRING: ALPHA" + - "PATH.stem: a" + # Second task: IntParam=2, FloatParam=2.5, StringParam=beta, PathParam=/tmp/b.txt + - "INT: 2 x 10 = 20" + - "FLOAT: 2.5" + - "STRING: BETA" + - "PATH.stem: b" + +expect_posix: + output_contains: + - "PATH: /tmp/a.txt" + - "PATH: /tmp/b.txt" + +expect_windows: + output_contains: + - "PATH: \\tmp\\a.txt" + - "PATH: \\tmp\\b.txt" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_posix_to_win_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_posix_to_win_scenario.yaml new file mode 100644 index 00000000..d49be5ed --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_posix_to_win_scenario.yaml @@ -0,0 +1,26 @@ +name: "LIST[PATH] parameter with posix-to-windows path mapping" +description: "Verifies Param.X is list[path] with mapping, RawParam.X is list[string] unmapped" + +run_on: windows + +job_template_file: "list_path_param_win_to_posix_template.yaml" + +job_parameters: + InputPaths: + - "/home/test/file1.txt" + - "/home/test/file2.txt" + +path_mapping_rules: + - source_path_format: posix + source_path: "/home" + destination_path: "C:\\Users" + +expect: + success: true + output_contains: + - 'Param.InputPaths=["C:\Users\test\file1.txt", "C:\Users\test\file2.txt"]' + - 'RawParam.InputPaths=["/home/test/file1.txt", "/home/test/file2.txt"]' + - 'Param.InputPaths[0]="C:\Users\test\file1.txt"' + - 'RawParam.InputPaths[0]="/home/test/file1.txt"' + - 'Param.InputPaths[0].name="file1.txt"' + - 'RawParam.InputPaths[0].upper()="/HOME/TEST/FILE1.TXT"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_scenario.yaml new file mode 100644 index 00000000..86de285d --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_scenario.yaml @@ -0,0 +1,26 @@ +name: "LIST[PATH] parameter with path mapping" +description: "Verifies Param.X is list[path] with mapping, RawParam.X is list[string] unmapped" + +run_on: posix + +job_template_file: "list_path_param_win_to_posix_template.yaml" + +job_parameters: + InputPaths: + - "C:\\Users\\test\\file1.txt" + - "C:\\Users\\test\\file2.txt" + +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Users" + destination_path: "/home" + +expect: + success: true + output_contains: + - 'Param.InputPaths=["/home/test/file1.txt", "/home/test/file2.txt"]' + - 'RawParam.InputPaths=["C:\\Users\\test\\file1.txt", "C:\\Users\\test\\file2.txt"]' + - 'Param.InputPaths[0]="/home/test/file1.txt"' + - 'RawParam.InputPaths[0]="C:\\Users\\test\\file1.txt"' + - 'Param.InputPaths[0].name="file1.txt"' + - 'RawParam.InputPaths[0].upper()="C:\\USERS\\TEST\\FILE1.TXT"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_template.yaml new file mode 100644 index 00000000..0ddea4a4 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/list_path_param_win_to_posix_template.yaml @@ -0,0 +1,26 @@ +# Standalone template - run with: openjd run list_path_param_mapping_template.yaml +specificationVersion: jobtemplate-2023-09 +name: LIST[PATH] Parameter Mapping Test +extensions: + - EXPR +parameterDefinitions: + - name: InputPaths + type: LIST[PATH] + default: + - "default/path/one" + - "default/path/two" +steps: + - name: TestListPathMapping + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo 'Param.InputPaths={{repr_json(Param.InputPaths)}}' + echo 'RawParam.InputPaths={{repr_json(RawParam.InputPaths)}}' + echo 'Param.InputPaths[0]={{repr_json(Param.InputPaths[0])}}' + echo 'RawParam.InputPaths[0]={{repr_json(RawParam.InputPaths[0])}}' + echo 'Param.InputPaths[0].name={{repr_json(Param.InputPaths[0].name)}}' + echo 'RawParam.InputPaths[0].upper()={{repr_json(RawParam.InputPaths[0].upper())}}' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/path_param_posix_to_win_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_posix_to_win_scenario.yaml new file mode 100644 index 00000000..8ce497f3 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_posix_to_win_scenario.yaml @@ -0,0 +1,22 @@ +name: "PATH parameter with posix-to-windows path mapping" +description: "Verifies Param.X has path type with mapping applied, RawParam.X is unmapped string" + +run_on: windows + +job_template_file: "path_param_win_to_posix_template.yaml" + +job_parameters: + InputPath: "/home/test/file.txt" + +path_mapping_rules: + - source_path_format: posix + source_path: "/home" + destination_path: "C:\\Users" + +expect: + success: true + output_contains: + - 'Param.InputPath="C:\Users\test\file.txt"' + - 'RawParam.InputPath="/home/test/file.txt"' + - 'Param.InputPath.name="file.txt"' + - 'RawParam.InputPath.upper()="/HOME/TEST/FILE.TXT"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_scenario.yaml new file mode 100644 index 00000000..66644fa3 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_scenario.yaml @@ -0,0 +1,22 @@ +name: "PATH parameter with path mapping" +description: "Verifies Param.X has path type with mapping applied, RawParam.X is unmapped string" + +run_on: posix + +job_template_file: "path_param_win_to_posix_template.yaml" + +job_parameters: + InputPath: "C:\\Users\\test\\file.txt" + +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Users" + destination_path: "/home" + +expect: + success: true + output_contains: + - 'Param.InputPath="/home/test/file.txt"' + - 'RawParam.InputPath="C:\\Users\\test\\file.txt"' + - 'Param.InputPath.name="file.txt"' + - 'RawParam.InputPath.upper()="C:\\USERS\\TEST\\FILE.TXT"' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_template.yaml new file mode 100644 index 00000000..89b03bd2 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/path_param_win_to_posix_template.yaml @@ -0,0 +1,22 @@ +# Standalone template - run with: openjd run path_param_mapping_template.yaml -p InputPath=/some/path +specificationVersion: jobtemplate-2023-09 +name: PATH Parameter Mapping Test +extensions: + - EXPR +parameterDefinitions: + - name: InputPath + type: PATH + default: "default/input/path" +steps: + - name: TestPathMapping + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo 'Param.InputPath={{repr_json(Param.InputPath)}}' + echo 'RawParam.InputPath={{repr_json(RawParam.InputPath)}}' + echo 'Param.InputPath.name={{repr_json(Param.InputPath.name)}}' + echo 'RawParam.InputPath.upper()={{repr_json(RawParam.InputPath.upper())}}' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_params_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_params_scenario.yaml new file mode 100644 index 00000000..410f4462 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_params_scenario.yaml @@ -0,0 +1,20 @@ +name: "Task parameter iteration" +description: "Verifies task parameters work with parameter space iteration" + +run_on: all + +job_template_file: "task_params_template.yaml" + +job_parameters: + Prefix: "render" + +expect: + success: true + output_contains: + - "Task.Param.Frame=1" + - "Task.Param.Frame=2" + - "Task.Param.Frame=3" + - "Param.Prefix=render" + - "Output=render_0001" + - "Output=render_0002" + - "Output=render_0003" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_params_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_params_template.yaml new file mode 100644 index 00000000..a38e5ec7 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_params_template.yaml @@ -0,0 +1,26 @@ +# Standalone template - run with: openjd run task_params_template.yaml +specificationVersion: jobtemplate-2023-09 +name: Task Parameters Test +extensions: + - EXPR +parameterDefinitions: + - name: Prefix + type: STRING + default: "frame" +steps: + - name: RenderFrames + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: INT + range: "1-3" + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo "Task.Param.Frame={{Task.Param.Frame}}" + echo "Param.Prefix={{Param.Prefix}}" + echo "Output={{(Param.Prefix + '_####').with_number(Task.Param.Frame)}}" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_scenario.yaml new file mode 100644 index 00000000..6cacc5be --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_scenario.yaml @@ -0,0 +1,31 @@ +name: "PATH task parameter from LIST[PATH] job parameter (posix-to-win)" +description: "Verifies PATH task parameters derived from LIST[PATH] job parameter with posix-to-windows path mapping" + +run_on: windows + +job_template_file: "task_path_from_list_path_posix_to_win_template.yaml" + +job_parameters: + InputDirs: + - "/mnt/projects/shot01" + - "/mnt/projects/shot02" + +path_mapping_rules: + - source_path_format: posix + source_path: "/mnt/projects" + destination_path: "C:\\Projects" + +expect: + success: true + output_contains: + # First task - shot01/contents.json with path mapping + - "Processing directory based on metadata" + - 'Task.Param.MetadataFile=C:\Projects\shot01\contents.json' + - 'Task.RawParam.MetadataFile=/mnt/projects/shot01/contents.json' + - 'Task.Param.MetadataFile.parent=C:\Projects\shot01' + - 'Task.RawParam.MetadataFile.upper=/MNT/PROJECTS/SHOT01/CONTENTS.JSON' + # Second task + - 'Task.Param.MetadataFile=C:\Projects\shot02\contents.json' + - 'Task.RawParam.MetadataFile=/mnt/projects/shot02/contents.json' + - 'Task.Param.MetadataFile.parent=C:\Projects\shot02' + - 'Task.RawParam.MetadataFile.upper=/MNT/PROJECTS/SHOT02/CONTENTS.JSON' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_template.yaml new file mode 100644 index 00000000..5ad34970 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_posix_to_win_template.yaml @@ -0,0 +1,29 @@ +# Tests PATH task parameter derived from LIST[PATH] job parameter (posix-to-win) +specificationVersion: jobtemplate-2023-09 +name: Task Path From List Path Posix To Win Test +extensions: + - EXPR +parameterDefinitions: + - name: InputDirs + type: LIST[PATH] + objectType: DIRECTORY + dataFlow: IN +steps: + - name: ProcessDirectories + parameterSpace: + taskParameterDefinitions: + - name: MetadataFile + type: PATH + range: "{{[dir + '/contents.json' for dir in RawParam.InputDirs]}}" + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo Processing directory based on metadata + echo Task.Param.MetadataFile={{repr_sh(Task.Param.MetadataFile)}} + echo Task.RawParam.MetadataFile={{repr_sh(Task.RawParam.MetadataFile)}} + echo Task.Param.MetadataFile.parent={{repr_sh(Task.Param.MetadataFile.parent)}} + echo Task.RawParam.MetadataFile.upper={{repr_sh(Task.RawParam.MetadataFile.upper())}} diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_scenario.yaml new file mode 100644 index 00000000..96e64abf --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_scenario.yaml @@ -0,0 +1,32 @@ +name: "PATH task parameter from LIST[PATH] job parameter" +description: "Verifies PATH task parameters derived from LIST[PATH] job parameter with path mapping" + +run_on: posix + +job_template_file: "task_path_from_list_path_template.yaml" + +job_parameters: + InputDirs: + - "C:\\Projects\\shot01" + - "C:\\Projects\\shot02" + +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Projects" + destination_path: "/mnt/projects" + +expect: + success: true + output_contains: + # First task - shot01/contents.json with path mapping + - "Processing directory based on metadata" + - "Task.Param.MetadataFile=/mnt/projects/shot01/contents.json" + - "Task.RawParam.MetadataFile=C:\\Projects\\shot01/contents.json" + - "Task.Param.MetadataFile.parent=/mnt/projects/shot01" + # Verify Task.RawParam is string type (has .upper()) + - "Task.RawParam.MetadataFile.upper=C:\\PROJECTS\\SHOT01/CONTENTS.JSON" + # Second task - shot02/contents.json with path mapping + - "Task.Param.MetadataFile=/mnt/projects/shot02/contents.json" + - "Task.RawParam.MetadataFile=C:\\Projects\\shot02/contents.json" + - "Task.Param.MetadataFile.parent=/mnt/projects/shot02" + - "Task.RawParam.MetadataFile.upper=C:\\PROJECTS\\SHOT02/CONTENTS.JSON" diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_template.yaml new file mode 100644 index 00000000..8d7e533a --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_from_list_path_template.yaml @@ -0,0 +1,29 @@ +# Tests PATH task parameter derived from LIST[PATH] job parameter +specificationVersion: jobtemplate-2023-09 +name: Task Path From List Path Job Param Test +extensions: + - EXPR +parameterDefinitions: + - name: InputDirs + type: LIST[PATH] + objectType: DIRECTORY + dataFlow: IN +steps: + - name: ProcessDirectories + parameterSpace: + taskParameterDefinitions: + - name: MetadataFile + type: PATH + range: "{{[dir + '/contents.json' for dir in RawParam.InputDirs]}}" + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo Processing directory based on metadata + echo Task.Param.MetadataFile={{repr_sh(Task.Param.MetadataFile)}} + echo Task.RawParam.MetadataFile={{repr_sh(Task.RawParam.MetadataFile)}} + echo Task.Param.MetadataFile.parent={{repr_sh(Task.Param.MetadataFile.parent)}} + echo Task.RawParam.MetadataFile.upper={{repr_sh(Task.RawParam.MetadataFile.upper())}} diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_scenario.yaml new file mode 100644 index 00000000..96f3e5fb --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_scenario.yaml @@ -0,0 +1,26 @@ +name: "PATH task parameter with posix-to-windows path mapping" +description: "Verifies PATH task parameters have posix-to-windows path mapping rules applied" + +run_on: windows + +job_template_file: "task_path_param_mapping_posix_to_win_template.yaml" + +path_mapping_rules: + - source_path_format: posix + source_path: "/mnt/projects" + destination_path: "C:\\Projects" + +expect: + success: true + output_contains: + # First task - path mapping applied to Task.Param, raw preserved in Task.RawParam + - 'Task.Param.InputFile=C:\Projects\render\frame001.exr' + - 'Task.RawParam.InputFile=/mnt/projects/render/frame001.exr' + - 'Task.Param.InputFile.parent=C:\Projects\render' + - 'Task.Param.InputFile.name=frame001.exr' + - 'Task.RawParam.InputFile.upper=/MNT/PROJECTS/RENDER/FRAME001.EXR' + # Second task + - 'Task.Param.InputFile=C:\Projects\render\frame002.exr' + - 'Task.RawParam.InputFile=/mnt/projects/render/frame002.exr' + - 'Task.Param.InputFile.name=frame002.exr' + - 'Task.RawParam.InputFile.upper=/MNT/PROJECTS/RENDER/FRAME002.EXR' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_template.yaml new file mode 100644 index 00000000..a4003077 --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_posix_to_win_template.yaml @@ -0,0 +1,24 @@ +# Tests PATH task parameter with posix-to-windows path mapping rules applied +specificationVersion: jobtemplate-2023-09 +name: Task Path Parameter Posix To Win Mapping Test +extensions: + - EXPR +steps: + - name: TestPathMapping + parameterSpace: + taskParameterDefinitions: + - name: InputFile + type: PATH + range: ["/mnt/projects/render/frame001.exr", "/mnt/projects/render/frame002.exr"] + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo Task.Param.InputFile={{repr_sh(Task.Param.InputFile)}} + echo Task.RawParam.InputFile={{repr_sh(Task.RawParam.InputFile)}} + echo Task.Param.InputFile.parent={{repr_sh(Task.Param.InputFile.parent)}} + echo Task.Param.InputFile.name={{repr_sh(Task.Param.InputFile.name)}} + echo Task.RawParam.InputFile.upper={{repr_sh(Task.RawParam.InputFile.upper())}} diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_scenario.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_scenario.yaml new file mode 100644 index 00000000..fee7ef3f --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_scenario.yaml @@ -0,0 +1,27 @@ +name: "PATH task parameter with path mapping" +description: "Verifies PATH task parameters have path mapping rules applied" + +run_on: posix + +job_template_file: "task_path_param_mapping_template.yaml" + +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Projects" + destination_path: "/mnt/projects" + +expect: + success: true + output_contains: + # First task - path mapping applied to Task.Param, raw preserved in Task.RawParam + - 'Task.Param.InputFile=/mnt/projects/render/frame001.exr' + - 'Task.RawParam.InputFile=C:\Projects\render\frame001.exr' + - 'Task.Param.InputFile.parent=/mnt/projects/render' + - 'Task.Param.InputFile.name=frame001.exr' + # Verify Task.RawParam is string type (has .upper(), path type does not) + - 'Task.RawParam.InputFile.upper=C:\PROJECTS\RENDER\FRAME001.EXR' + # Second task + - 'Task.Param.InputFile=/mnt/projects/render/frame002.exr' + - 'Task.RawParam.InputFile=C:\Projects\render\frame002.exr' + - 'Task.Param.InputFile.name=frame002.exr' + - 'Task.RawParam.InputFile.upper=C:\PROJECTS\RENDER\FRAME002.EXR' diff --git a/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_template.yaml b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_template.yaml new file mode 100644 index 00000000..29d8686b --- /dev/null +++ b/test/openjd/sessions_v1/scenarios/parameter_types/task_path_param_mapping_template.yaml @@ -0,0 +1,24 @@ +# Tests PATH task parameter with path mapping rules applied +specificationVersion: jobtemplate-2023-09 +name: Task Path Parameter Path Mapping Test +extensions: + - EXPR +steps: + - name: TestPathMapping + parameterSpace: + taskParameterDefinitions: + - name: InputFile + type: PATH + range: ["C:\\Projects\\render\\frame001.exr", "C:\\Projects\\render\\frame002.exr"] + script: + actions: + onRun: + command: bash + args: + - "-c" + - | + echo Task.Param.InputFile={{repr_sh(Task.Param.InputFile)}} + echo Task.RawParam.InputFile={{repr_sh(Task.RawParam.InputFile)}} + echo Task.Param.InputFile.parent={{repr_sh(Task.Param.InputFile.parent)}} + echo Task.Param.InputFile.name={{repr_sh(Task.Param.InputFile.name)}} + echo Task.RawParam.InputFile.upper={{repr_sh(Task.RawParam.InputFile.upper())}} diff --git a/test/openjd/sessions_v1/support_files/__init__.py b/test/openjd/sessions_v1/support_files/__init__.py new file mode 100644 index 00000000..8d929cc8 --- /dev/null +++ b/test/openjd/sessions_v1/support_files/__init__.py @@ -0,0 +1 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/openjd/sessions_v1/support_files/app_20s_run.py b/test/openjd/sessions_v1/support_files/app_20s_run.py new file mode 100644 index 00000000..e384616d --- /dev/null +++ b/test/openjd/sessions_v1/support_files/app_20s_run.py @@ -0,0 +1,30 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# A simple app for use in testing of the LoggingSubprocess. +# Prints out an increasing series of integers (0, 1, 2, ...) +# every second for 10 seconds +# +# Hook SIGTERM (posix) or CTRL_BREAK_EVENT (windows) and print "Trapped" +# and exit if we get the signal + +import signal +import sys +import time + + +def hook(handle, frame): + print("Trapped") + sys.stdout.flush() + sys.exit(1) + + +if sys.platform.startswith("win"): + signal.signal(signal.SIGBREAK, hook) + signal.signal(signal.SIGINT, hook) +else: + signal.signal(signal.SIGTERM, hook) + +for i in range(0, 20): + print(f"Log from test {str(i)}") + sys.stdout.flush() + time.sleep(1) diff --git a/test/openjd/sessions_v1/support_files/app_20s_run.sh b/test/openjd/sessions_v1/support_files/app_20s_run.sh new file mode 100644 index 00000000..e756cf65 --- /dev/null +++ b/test/openjd/sessions_v1/support_files/app_20s_run.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +PYTHON="$1" + +SCRIPT=$(dirname $0)/app_20s_run.py + +"$PYTHON" "$SCRIPT" +exit $? \ No newline at end of file diff --git a/test/openjd/sessions_v1/support_files/app_20s_run_ignore_signal.py b/test/openjd/sessions_v1/support_files/app_20s_run_ignore_signal.py new file mode 100644 index 00000000..111421e6 --- /dev/null +++ b/test/openjd/sessions_v1/support_files/app_20s_run_ignore_signal.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +# As app_20s_run.py except it does not exit when it gets a SIGTERM/SIGBREAK + +import signal +import sys +import time + + +def hook(handle, frame): + print("Trapped") + sys.stdout.flush() + + +if sys.platform.startswith("win"): + signal.signal(signal.SIGBREAK, hook) + signal.signal(signal.SIGINT, hook) +else: + signal.signal(signal.SIGTERM, hook) + +for i in range(0, 20): + print(i) + sys.stdout.flush() + time.sleep(1) diff --git a/test/openjd/sessions_v1/support_files/output_signal_sender.c b/test/openjd/sessions_v1/support_files/output_signal_sender.c new file mode 100644 index 00000000..87a0adc1 --- /dev/null +++ b/test/openjd/sessions_v1/support_files/output_signal_sender.c @@ -0,0 +1,43 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +// This is a minimal C program that sleeps until it receives a SIGTERM signal +// and outputs the process ID of the sender. + +#include +#include +#include +#include +#include +#include +#include +#include + +static bool received_signal = false; + +static void signal_handler(int sig, siginfo_t *siginfo, void *context) { + // Output PID of the sending process + int pid_of_sending_process = (int) siginfo->si_pid; + printf("%d\n", pid_of_sending_process); + + // Tell main loop to exit + received_signal = true; +} + +int main(int argc, char *argv[]) { + // register signal handler + struct sigaction signal_action; + signal_action.sa_sigaction = *signal_handler; + // get details about the signal + signal_action.sa_flags |= SA_SIGINFO; + if(sigaction(SIGTERM, &signal_action, NULL) != 0) { + printf("Could not register signal handler\n"); + return errno; + } + + // sleep until SIGINT received + while(!received_signal) { + sleep(1); + } + + return 0; +} diff --git a/test/openjd/sessions_v1/support_files/run_app_20s_run.py b/test/openjd/sessions_v1/support_files/run_app_20s_run.py new file mode 100644 index 00000000..864603cc --- /dev/null +++ b/test/openjd/sessions_v1/support_files/run_app_20s_run.py @@ -0,0 +1,28 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import sys +import subprocess +import time +from pathlib import Path + +proc = subprocess.Popen( + args=[ + sys.executable, + str(Path(__file__).parent / "app_20s_run.py"), + ], + stdout=subprocess.PIPE, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + encoding="utf-8", +) + +if proc.stdout is not None: + for line in iter(proc.stdout.readline, ""): + line = line.rstrip("\n\r") + print(line) + sys.stdout.flush() + +for i in range(0, 20): + print(f"Log from runner {str(i)}") + sys.stdout.flush() + time.sleep(1) diff --git a/test/openjd/sessions_v1/test_importable.py b/test/openjd/sessions_v1/test_importable.py new file mode 100644 index 00000000..91046e07 --- /dev/null +++ b/test/openjd/sessions_v1/test_importable.py @@ -0,0 +1,9 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + +def test_openjd_importable(): + import openjd # noqa: F401 + + +def test_importable(): + import openjd.sessions._v1 # noqa: F401 diff --git a/test/openjd/sessions_v1/test_pickle.py b/test/openjd/sessions_v1/test_pickle.py new file mode 100644 index 00000000..fa7c483d --- /dev/null +++ b/test/openjd/sessions_v1/test_pickle.py @@ -0,0 +1,195 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pickle round-trip tests for the ``openjd.sessions._v1`` value types. + +The Group A enums (``SessionState``, ``ActionState``, ``ScriptRunnerState``) +and Group B value types (``ActionStatus``, ``ActionResult``, +``PosixSessionUser``) all support pickle. ``Session`` itself does not — +it owns live OS resources (subprocess, working directory, file handles) +and explicitly rejects pickling. + +Cross-reference: ``reports/sessions-bindings-quality-evaluation-report.md`` +recommendations #7 and #9. +""" + +import os +import pickle +import sys + +import pytest + + +# ── Group A: enums ─────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "value", + [ + "openjd.sessions._v1.SessionState.READY", + "openjd.sessions._v1.SessionState.RUNNING", + "openjd.sessions._v1.SessionState.CANCELING", + "openjd.sessions._v1.SessionState.READY_ENDING", + "openjd.sessions._v1.SessionState.ENDED", + "openjd.sessions._v1.ActionState.RUNNING", + "openjd.sessions._v1.ActionState.SUCCESS", + "openjd.sessions._v1.ActionState.FAILED", + "openjd.sessions._v1.ActionState.CANCELED", + "openjd.sessions._v1.ActionState.TIMEOUT", + "openjd.sessions._v1.ScriptRunnerState.READY", + "openjd.sessions._v1.ScriptRunnerState.RUNNING", + "openjd.sessions._v1.ScriptRunnerState.CANCELING", + "openjd.sessions._v1.ScriptRunnerState.CANCELED", + "openjd.sessions._v1.ScriptRunnerState.TIMEOUT", + "openjd.sessions._v1.ScriptRunnerState.FAILED", + "openjd.sessions._v1.ScriptRunnerState.SUCCESS", + ], +) +def test_enum_round_trip(value): + import importlib + + module_path, attr = value.rsplit(".", 1) + cls_path, cls = module_path.rsplit(".", 1) + module = importlib.import_module(cls_path) + enum_cls = getattr(module, cls) + v = getattr(enum_cls, attr) + loaded = pickle.loads(pickle.dumps(v)) + assert loaded == v + assert loaded is v + + +# ── Group B: ActionStatus ──────────────────────────────────────── + + +def test_action_status_round_trip_minimal(): + from openjd.sessions._v1 import ActionState, ActionStatus + + status = ActionStatus(state=ActionState.RUNNING) + loaded = pickle.loads(pickle.dumps(status)) + assert loaded.state == status.state + assert loaded.progress is None + assert loaded.status_message is None + assert loaded.fail_message is None + assert loaded.exit_code is None + assert loaded.started_at is None + assert loaded.ended_at is None + + +def test_action_status_round_trip_full_user_constructed(): + from openjd.sessions._v1 import ActionState, ActionStatus + + status = ActionStatus( + state=ActionState.SUCCESS, + progress=100.0, + status_message="ok", + fail_message=None, + exit_code=0, + ) + loaded = pickle.loads(pickle.dumps(status)) + assert loaded.state == status.state + assert loaded.progress == 100.0 + assert loaded.status_message == "ok" + assert loaded.fail_message is None + assert loaded.exit_code == 0 + + +def test_action_status_round_trip_failed(): + from openjd.sessions._v1 import ActionState, ActionStatus + + status = ActionStatus( + state=ActionState.FAILED, + fail_message="non-zero exit", + exit_code=1, + ) + loaded = pickle.loads(pickle.dumps(status)) + assert loaded.state == ActionState.FAILED + assert loaded.fail_message == "non-zero exit" + assert loaded.exit_code == 1 + + +def test_action_status_pickle_with_timestamps(): + """``started_at`` / ``ended_at`` are normally set internally by the + runner; the pickle reducer goes through ``_from_state`` which + accepts them. Round-trip via the private classmethod.""" + import datetime + + from openjd.sessions._v1 import ActionState, ActionStatus + + started = datetime.datetime(2024, 1, 15, 12, 30, 0, tzinfo=datetime.timezone.utc) + ended = datetime.datetime(2024, 1, 15, 12, 35, 42, tzinfo=datetime.timezone.utc) + status = ActionStatus._from_state( + state=ActionState.SUCCESS, + exit_code=0, + started_at=started, + ended_at=ended, + ) + loaded = pickle.loads(pickle.dumps(status)) + assert loaded.state == ActionState.SUCCESS + assert loaded.started_at == started + assert loaded.ended_at == ended + + +# ── Group B: ActionResult ──────────────────────────────────────── + + +def test_action_result_round_trip(): + from openjd.sessions._v1 import ActionResult, ActionState + + result = ActionResult(state=ActionState.SUCCESS, exit_code=0, stdout="hello\n") + loaded = pickle.loads(pickle.dumps(result)) + assert loaded.state == result.state + assert loaded.exit_code == result.exit_code + assert loaded.stdout == result.stdout + + +def test_action_result_round_trip_minimal(): + """Default ``stdout=""`` and ``exit_code=None`` round-trip.""" + from openjd.sessions._v1 import ActionResult, ActionState + + result = ActionResult(state=ActionState.CANCELED) + loaded = pickle.loads(pickle.dumps(result)) + assert loaded.state == result.state + assert loaded.exit_code is None + assert loaded.stdout == "" + + +# ── Group B: PosixSessionUser ──────────────────────────────────── + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-only") +def test_posix_session_user_round_trip(): + from openjd.sessions._v1 import PosixSessionUser + + user_name = os.environ.get("USER", "nobody") + user = PosixSessionUser(user_name) + loaded = pickle.loads(pickle.dumps(user)) + assert loaded.user == user.user + assert loaded.group == user.group + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-only") +def test_posix_session_user_round_trip_with_group(): + from openjd.sessions._v1 import PosixSessionUser + + # Use the current user/group; PosixSessionUser doesn't validate + # they exist on the system in its constructor. + user_name = os.environ.get("USER", "nobody") + user = PosixSessionUser(user_name, group=user_name) + loaded = pickle.loads(pickle.dumps(user)) + assert loaded.user == user_name + assert loaded.group == user_name + + +# ── Pickled-bytes shape sanity ────────────────────────────────── + + +def test_pickled_action_state_qualifies_under_canonical_module(): + from openjd.sessions._v1 import ActionState + + data = pickle.dumps(ActionState.SUCCESS) + # The pickled object must reference the canonical module so that + # consumers reading pickled bytes (Deadline Cloud worker-agent IPC + # in particular) resolve it correctly. + assert b"openjd.sessions._v1" in data + assert b"ActionState" in data + assert b"SUCCESS" in data diff --git a/test/openjd/sessions_v1/test_session_scenarios.py b/test/openjd/sessions_v1/test_session_scenarios.py new file mode 100644 index 00000000..839df7f3 --- /dev/null +++ b/test/openjd/sessions_v1/test_session_scenarios.py @@ -0,0 +1,275 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""YAML-based session scenario tests. + +Each scenario YAML file references a standalone job template that can be run +independently with `openjd run`, plus test-specific parameters and expectations. + +Scenario YAML Format +-------------------- + +```yaml +name: "Human-readable test name" +description: "What this scenario tests" + +# Platform restriction (optional, default: all) +# Values: all, windows, posix +run_on: posix + +# Path to job template file (required) +job_template_file: "my_template.yaml" + +# Job parameter values (optional) +job_parameters: + ParamName: "value" + ListParam: ["a", "b"] + +# Path mapping rules (optional) +path_mapping_rules: + - source_path_format: windows + source_path: "C:\\Users" + destination_path: "/home" + +# Expected behavior (optional) +expect: + success: true # default: true + output_contains: + - "pattern that must appear in output" + output_excludes: + - "pattern that must NOT appear" + +# Platform-specific expectations (optional, merged into expect) +expect_posix: + output_contains: + - "posix-only pattern" +expect_windows: + output_contains: + - "windows-only pattern" +``` + +File Naming Convention +---------------------- +- Scenario files must end with `_scenario.yaml` +- Template files typically end with `_template.yaml` +- Templates can be run standalone: `openjd run my_template.yaml` +""" + +import os +from pathlib import Path +from typing import Any + +import pytest +import yaml # type: ignore[import-untyped] + +from openjd.model._v1 import ( + create_job, + decode_job_template, +) +from openjd.model._v1.types import ( + JobParameterType, + JobParameterValue, +) +from openjd.model._v1.job import ( + StepParameterSpaceIterator, +) +from openjd.sessions._v1 import Session, PathMappingRule, SessionState + + +SCENARIOS_DIR = Path(__file__).parent / "scenarios" + + +def discover_scenarios() -> list[Path]: + """Find all scenario YAML files (ending with _scenario.yaml).""" + if not SCENARIOS_DIR.exists(): + return [] + return sorted(SCENARIOS_DIR.rglob("*_scenario.yaml")) + + +def should_run(scenario: dict[str, Any]) -> bool: + """Check if scenario should run on current platform based on run_on.""" + run_on = scenario.get("run_on", "all") + if run_on not in ("all", "windows", "posix"): + raise ValueError(f"Invalid run_on '{run_on}'. Must be 'all', 'windows', or 'posix'") + if run_on == "all": + return True + is_windows = os.name == "nt" + if run_on == "windows": + return is_windows + if run_on == "posix": + return not is_windows + return True + + +def parse_path_mapping_rules(rules_data: list[dict]) -> list[PathMappingRule]: + """Convert YAML path mapping rules to PathMappingRule objects.""" + return [PathMappingRule.from_dict(r) for r in rules_data] + + +def build_parameter_values( + params: dict[str, Any], job_template: Any +) -> dict[str, JobParameterValue]: + """Build JobParameterValue dict from scenario parameters.""" + # Get type info from template's parameter definitions. The v1 + # binding exposes `JobParameterType` as a pyclass enum without + # a value-from-string constructor; pass the variant directly. + # `.name` is the identifier (e.g. "STRING", "LIST_PATH"), + # `.as_str()` is the spec form (e.g. "STRING", "LIST[PATH]") + # — different shapes, so we hold onto the variant itself. + type_map: dict[str, JobParameterType] = {} + for param_def in getattr(job_template, "parameterDefinitions", []) or []: + type_map[param_def.name] = param_def.type + + result = {} + for name, value in params.items(): + ptype = type_map.get(name, JobParameterType.STRING) + result[name] = JobParameterValue(type=ptype, value=value) + return result + + +# Session states the runner waits to settle into between actions. +# In the v1 binding `SessionState` is a pyclass enum (no `.value` +# attribute); compare against the enum variants directly. +_QUIESCENT_STATES = { + SessionState.READY, + SessionState.ENDED, + SessionState.READY_ENDING, +} + + +class TestSessionScenarios: + """Run YAML-defined session scenarios.""" + + @pytest.mark.parametrize( + "scenario_path", + discover_scenarios(), + ids=lambda p: str(p.relative_to(SCENARIOS_DIR)).replace("/", "::"), + ) + def test_scenario(self, scenario_path: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture): + """Execute a single scenario and verify expectations.""" + scenario = yaml.safe_load(scenario_path.read_text()) + + # Check platform + if not should_run(scenario): + run_on = scenario.get("run_on", "all") + pytest.skip(f"Scenario only runs on {run_on}") + + # Load referenced template + template_file = scenario.get("job_template_file") + if template_file: + template_path = scenario_path.parent / template_file + template_dict = yaml.safe_load(template_path.read_text()) + else: + pytest.fail("Scenario must specify job_template_file") + + # Decode template + extensions = template_dict.get("extensions", []) + job_template = decode_job_template(template=template_dict, supported_extensions=extensions) + + # Build parameter values + job_params = build_parameter_values(scenario.get("job_parameters", {}), job_template) + + # Build path mapping rules + path_rules = parse_path_mapping_rules(scenario.get("path_mapping_rules", [])) + + # Create job to get step script + job = create_job(job_template=job_template, job_parameter_values=job_params) + + # Select step (by name or default to first) + step_name = scenario.get("step") + if step_name: + step = next((s for s in job.steps if s.name == step_name), None) + if not step: + pytest.fail(f"Step '{step_name}' not found in template") + else: + step = job.steps[0] + step_script = step.script + + # Track environment identifiers for exit + job_env_ids = [] + step_env_ids = [] + + # Run session and iterate through parameter space + with Session( + session_id="scenario-test", + job_parameter_values=job_params, + path_mapping_rules=path_rules if path_rules else None, + session_root_directory=tmp_path, + ) as session: + # Enter job environments + for env in job.jobEnvironments or []: + env_id = session.enter_environment(environment=env) + job_env_ids.append(env_id) + while session.state not in _QUIESCENT_STATES: + import time + + time.sleep(0.01) + + # Enter step environments + for env in step.stepEnvironments or []: + env_id = session.enter_environment( + environment=env, + resolved_symtab=step.resolved_symtab, + ) + step_env_ids.append(env_id) + while session.state not in _QUIESCENT_STATES: + import time + + time.sleep(0.01) + + # Run tasks + for task_params in StepParameterSpaceIterator(space=step.parameterSpace): + session.run_task( + step_script=step_script, + task_parameter_values=task_params, + resolved_symtab=step.resolved_symtab, + ) + while session.state not in _QUIESCENT_STATES: + import time + + time.sleep(0.01) + + # Exit step environments (reverse order) + for env_id in reversed(step_env_ids): + session.exit_environment( + identifier=env_id, + resolved_symtab=step.resolved_symtab, + ) + while session.state not in _QUIESCENT_STATES: + import time + + time.sleep(0.01) + + # Exit job environments (reverse order) + for env_id in reversed(job_env_ids): + session.exit_environment(identifier=env_id) + while session.state not in _QUIESCENT_STATES: + import time + + time.sleep(0.01) + + # Collect output from logs + captured_output = caplog.messages + + # Verify expectations + expect = scenario.get("expect", {}) + + # Merge platform-specific expectations + is_windows = os.name == "nt" + platform_expect = scenario.get("expect_windows" if is_windows else "expect_posix", {}) + # Merge platform-specific output_contains/output_excludes into main expect + for key in ("output_contains", "output_excludes"): + if key in platform_expect: + expect.setdefault(key, []).extend(platform_expect[key]) + + if expect.get("success", True): + # Check for failure indicators in output + for msg in captured_output: + assert "openjd_fail:" not in msg.lower(), f"Unexpected failure: {msg}" + + for pattern in expect.get("output_contains", []): + found = any(pattern in msg for msg in captured_output) + assert found, f"Expected output containing '{pattern}' not found in: {captured_output}" + + for pattern in expect.get("output_excludes", []): + found = any(pattern in msg for msg in captured_output) + assert not found, f"Unexpected output containing '{pattern}' found"