diff --git a/task-runner/task_runner/executers/__init__.py b/task-runner/task_runner/executers/__init__.py index 18483513..f7e4facf 100644 --- a/task-runner/task_runner/executers/__init__.py +++ b/task-runner/task_runner/executers/__init__.py @@ -7,7 +7,6 @@ from .exec_command_logger import ExecCommandLogger # noqa: I001 from .base_executer import BaseExecuter, ExecuterSubProcessError # noqa: I001 from .command import Command # noqa: I001 -from .mpi_base_executer import MPIExecuter # noqa: I001 from .mpi_configuration import MPIClusterConfiguration # noqa: I001 from .subprocess_tracker import SubprocessTracker # noqa: I001 from . import ( diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index 6b75382c..661f1aa7 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -2,7 +2,6 @@ import getpass import os -import shutil from task_runner import executers from task_runner.utils import files @@ -18,8 +17,6 @@ def execute(self): else: global_env = {} - input_dir = os.path.join(self.working_dir, self.args.sim_dir) - run_subprocess_dir = self.artifacts_dir_container if hasattr(self.args, @@ -27,9 +24,6 @@ def execute(self): run_subprocess_dir = os.path.join(self.artifacts_dir_container, self.args.run_subprocess_dir) - # Copy the input files to the artifacts directory - shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) - original_username = None if self.commands_user: original_username = getpass.getuser() diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index cb639c24..873a9405 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -4,6 +4,7 @@ its usage. """ import os +import shutil import threading import time from abc import ABC, abstractmethod @@ -92,7 +93,13 @@ def __init__( logging.info("Working directory: %s", self.working_dir) - os.makedirs(self.artifacts_dir) + # Move the inputs from sim_dir/ to artifacts_dir/ so that artifacts_dir/ + # already contains the inputs, avoiding the need for copying + shutil.move( + src=f"{self.working_dir}/{self.args.sim_dir}", + dst=self.artifacts_dir, + ) + logging.info("Created output directory: %s", self.output_dir) logging.info("Created artifacts directory: %s", self.artifacts_dir) diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py deleted file mode 100644 index 71e9bcdc..00000000 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ /dev/null @@ -1,66 +0,0 @@ -"""This file provides a class to implement MPI executers.""" -import os -import shutil -from typing import Any - -from task_runner import executers -from task_runner.executers import mpi_configuration - -# Instructions inside Docker containers are run by the root user (as default), -# so we need to allow Open MPI to be run as root. This is usually strongly -# discouraged, but necessary to run Open MPI inside a container. For further -# details, see https://www.open-mpi.org/doc/v4.1/man1/mpirun.1.php#toc25. -MPI_ALLOW = "--allow-run-as-root" -MPI_DISTRIBUTION_FILENAME = "machinefile" - - -class MPIExecuter(executers.BaseExecuter): - """Implementation of a general MPI Executer.""" - - def __init__( - self, - working_dir, - container_image, - mpi_config: mpi_configuration.MPIClusterConfiguration, - exec_command_logger: executers.ExecCommandLogger, - extra_params: dict[str, Any], - sim_binary, - file_type, - sim_specific_input_filename, - ): - super().__init__(working_dir, container_image, mpi_config, - exec_command_logger, extra_params) - self.sim_binary = sim_binary - self.sim_specific_input_filename = sim_specific_input_filename - self.file_type = file_type - - def execute(self): - sim_dir = os.path.join(self.working_dir, self.args.sim_dir) - input_filename = self.args.input_filename - - input_file_full_path = os.path.join(sim_dir, input_filename) - - if not os.path.exists(input_file_full_path): - if os.path.exists(f"{input_file_full_path}.{self.file_type}"): - input_filename = f"{input_file_full_path}.{self.file_type}" - else: - raise ValueError( - f"A file with name {input_filename} doesn't exist.") - - if self.args.n_vcpus: - self.mpi_config.extra_args.extend(["-np", f"{self.args.n_vcpus}"]) - - use_hwthread = bool(self.args.use_hwthread) - - if use_hwthread: - self.mpi_config.extra_args.extend(["--use-hwthread-cpus"]) - # Renaming input file as the simulator expects it to be - os.rename(input_file_full_path, - os.path.join(sim_dir, self.sim_specific_input_filename)) - - cmd = executers.Command(self.sim_binary + " " + - self.sim_specific_input_filename, - is_mpi=True) - self.run_subprocess(cmd, working_dir=sim_dir) - - shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True)