From c9194f59a05289e4d2b4f311c2e64f5ed484ea24 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 10:38:19 +0000 Subject: [PATCH 1/7] Replace copy with move --- .../task_runner/executers/arbitrary_commands_executer.py | 2 +- task-runner/task_runner/executers/mpi_base_executer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index 6b75382c..be1136ec 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -28,7 +28,7 @@ def execute(self): self.args.run_subprocess_dir) # Copy the input files to the artifacts directory - shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) + shutil.move(src=input_dir, dst=self.artifacts_dir) original_username = None if self.commands_user: diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py index 71e9bcdc..457ce50a 100644 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ b/task-runner/task_runner/executers/mpi_base_executer.py @@ -63,4 +63,4 @@ def execute(self): is_mpi=True) self.run_subprocess(cmd, working_dir=sim_dir) - shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) + shutil.move(src=sim_dir, dst=self.artifacts_dir) From 18cc3f1ddde394a812654a5cb74bc830fe98233d Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 15:37:12 +0000 Subject: [PATCH 2/7] Revert "Replace copy with move" This reverts commit c9194f59a05289e4d2b4f311c2e64f5ed484ea24. --- .../task_runner/executers/arbitrary_commands_executer.py | 2 +- task-runner/task_runner/executers/mpi_base_executer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index be1136ec..6b75382c 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -28,7 +28,7 @@ def execute(self): self.args.run_subprocess_dir) # Copy the input files to the artifacts directory - shutil.move(src=input_dir, dst=self.artifacts_dir) + shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) original_username = None if self.commands_user: diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py index 457ce50a..71e9bcdc 100644 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ b/task-runner/task_runner/executers/mpi_base_executer.py @@ -63,4 +63,4 @@ def execute(self): is_mpi=True) self.run_subprocess(cmd, working_dir=sim_dir) - shutil.move(src=sim_dir, dst=self.artifacts_dir) + shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) From 8f58069150ca58abea3c96e24d728a8b296ae54c Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:34:07 +0000 Subject: [PATCH 3/7] Move sim_dir into artifacts_dir when creating artifacts --- .../task_runner/executers/arbitrary_commands_executer.py | 6 ------ task-runner/task_runner/executers/base_executer.py | 9 ++++++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index 6b75382c..661f1aa7 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -2,7 +2,6 @@ import getpass import os -import shutil from task_runner import executers from task_runner.utils import files @@ -18,8 +17,6 @@ def execute(self): else: global_env = {} - input_dir = os.path.join(self.working_dir, self.args.sim_dir) - run_subprocess_dir = self.artifacts_dir_container if hasattr(self.args, @@ -27,9 +24,6 @@ def execute(self): run_subprocess_dir = os.path.join(self.artifacts_dir_container, self.args.run_subprocess_dir) - # Copy the input files to the artifacts directory - shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) - original_username = None if self.commands_user: original_username = getpass.getuser() diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index cb639c24..901c48e6 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -4,6 +4,7 @@ its usage. """ import os +import shutil import threading import time from abc import ABC, abstractmethod @@ -91,8 +92,14 @@ def __init__( self.args = named_tuple_constructor(**extra_params) logging.info("Working directory: %s", self.working_dir) + + # Move the inputs from sim_dir/ to artifacts_dir/ so that artifacts_dir/ + # already contains the inputs, avoiding the need for copying + shutil.move( + src=f"{self.working_dir}/{self.args.sim_dir}", + dest=self.artifacts_dir, + ) - os.makedirs(self.artifacts_dir) logging.info("Created output directory: %s", self.output_dir) logging.info("Created artifacts directory: %s", self.artifacts_dir) From 30ed5d7a09bfa2ce6ddd66e6f105df0837791215 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:35:58 +0000 Subject: [PATCH 4/7] Fix param name --- task-runner/task_runner/executers/base_executer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index 901c48e6..0f1c3332 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -97,7 +97,7 @@ def __init__( # already contains the inputs, avoiding the need for copying shutil.move( src=f"{self.working_dir}/{self.args.sim_dir}", - dest=self.artifacts_dir, + dst=self.artifacts_dir, ) logging.info("Created output directory: %s", self.output_dir) From 90346e4ade89b04253c186627e47737457c3d6fd Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:41:13 +0000 Subject: [PATCH 5/7] Run ruff and yapf --- task-runner/task_runner/executers/base_executer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index 0f1c3332..873a9405 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -92,7 +92,7 @@ def __init__( self.args = named_tuple_constructor(**extra_params) logging.info("Working directory: %s", self.working_dir) - + # Move the inputs from sim_dir/ to artifacts_dir/ so that artifacts_dir/ # already contains the inputs, avoiding the need for copying shutil.move( From f5d95efcafffe03a7dc95b4592e7b59d892b9e10 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:44:53 +0000 Subject: [PATCH 6/7] Delete deprecated mpi executer --- .../executers/mpi_base_executer.py | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 task-runner/task_runner/executers/mpi_base_executer.py diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py deleted file mode 100644 index 71e9bcdc..00000000 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ /dev/null @@ -1,66 +0,0 @@ -"""This file provides a class to implement MPI executers.""" -import os -import shutil -from typing import Any - -from task_runner import executers -from task_runner.executers import mpi_configuration - -# Instructions inside Docker containers are run by the root user (as default), -# so we need to allow Open MPI to be run as root. This is usually strongly -# discouraged, but necessary to run Open MPI inside a container. For further -# details, see https://www.open-mpi.org/doc/v4.1/man1/mpirun.1.php#toc25. -MPI_ALLOW = "--allow-run-as-root" -MPI_DISTRIBUTION_FILENAME = "machinefile" - - -class MPIExecuter(executers.BaseExecuter): - """Implementation of a general MPI Executer.""" - - def __init__( - self, - working_dir, - container_image, - mpi_config: mpi_configuration.MPIClusterConfiguration, - exec_command_logger: executers.ExecCommandLogger, - extra_params: dict[str, Any], - sim_binary, - file_type, - sim_specific_input_filename, - ): - super().__init__(working_dir, container_image, mpi_config, - exec_command_logger, extra_params) - self.sim_binary = sim_binary - self.sim_specific_input_filename = sim_specific_input_filename - self.file_type = file_type - - def execute(self): - sim_dir = os.path.join(self.working_dir, self.args.sim_dir) - input_filename = self.args.input_filename - - input_file_full_path = os.path.join(sim_dir, input_filename) - - if not os.path.exists(input_file_full_path): - if os.path.exists(f"{input_file_full_path}.{self.file_type}"): - input_filename = f"{input_file_full_path}.{self.file_type}" - else: - raise ValueError( - f"A file with name {input_filename} doesn't exist.") - - if self.args.n_vcpus: - self.mpi_config.extra_args.extend(["-np", f"{self.args.n_vcpus}"]) - - use_hwthread = bool(self.args.use_hwthread) - - if use_hwthread: - self.mpi_config.extra_args.extend(["--use-hwthread-cpus"]) - # Renaming input file as the simulator expects it to be - os.rename(input_file_full_path, - os.path.join(sim_dir, self.sim_specific_input_filename)) - - cmd = executers.Command(self.sim_binary + " " + - self.sim_specific_input_filename, - is_mpi=True) - self.run_subprocess(cmd, working_dir=sim_dir) - - shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) From c5d9be22ea8a2822404b0751a94aa67c87f77db2 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:45:39 +0000 Subject: [PATCH 7/7] Delete mpi executer from init --- task-runner/task_runner/executers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/task-runner/task_runner/executers/__init__.py b/task-runner/task_runner/executers/__init__.py index 18483513..f7e4facf 100644 --- a/task-runner/task_runner/executers/__init__.py +++ b/task-runner/task_runner/executers/__init__.py @@ -7,7 +7,6 @@ from .exec_command_logger import ExecCommandLogger # noqa: I001 from .base_executer import BaseExecuter, ExecuterSubProcessError # noqa: I001 from .command import Command # noqa: I001 -from .mpi_base_executer import MPIExecuter # noqa: I001 from .mpi_configuration import MPIClusterConfiguration # noqa: I001 from .subprocess_tracker import SubprocessTracker # noqa: I001 from . import (