diff --git a/src/aind_data_upload_utils/check_directories_job.py b/src/aind_data_upload_utils/check_directories_job.py index 8688b01..4adbec0 100644 --- a/src/aind_data_upload_utils/check_directories_job.py +++ b/src/aind_data_upload_utils/check_directories_job.py @@ -202,10 +202,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/check_metadata_job.py b/src/aind_data_upload_utils/check_metadata_job.py index 4cfc986..43841d7 100644 --- a/src/aind_data_upload_utils/check_metadata_job.py +++ b/src/aind_data_upload_utils/check_metadata_job.py @@ -111,10 +111,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/copy_metadata_files.py b/src/aind_data_upload_utils/copy_metadata_files.py index 330cd66..641358a 100644 --- a/src/aind_data_upload_utils/copy_metadata_files.py +++ b/src/aind_data_upload_utils/copy_metadata_files.py @@ -76,10 +76,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/create_s5_commands_job.py b/src/aind_data_upload_utils/create_s5_commands_job.py index aedd550..e4d24de 100644 --- a/src/aind_data_upload_utils/create_s5_commands_job.py +++ b/src/aind_data_upload_utils/create_s5_commands_job.py @@ -217,10 +217,12 @@ def run_job(self) -> None: "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_folders_job.py b/src/aind_data_upload_utils/delete_folders_job.py index d2a992a..eca3bdc 100644 --- a/src/aind_data_upload_utils/delete_folders_job.py +++ b/src/aind_data_upload_utils/delete_folders_job.py @@ -84,10 +84,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_source_folders_job.py b/src/aind_data_upload_utils/delete_source_folders_job.py index 55f141f..d4e2acf 100644 --- a/src/aind_data_upload_utils/delete_source_folders_job.py +++ b/src/aind_data_upload_utils/delete_source_folders_job.py @@ -169,9 +169,16 @@ def _remove_metadata_directory( metadata_dir = self.job_settings.directories.metadata_dir for metadata_file in metadata_files_in_both_places: local_file = Path(metadata_dir) / metadata_file - if not self.job_settings.dry_run: + if not self.job_settings.dry_run and os.path.isfile(local_file): logging.info(f"Removing {local_file}") os.remove(local_file) + elif not self.job_settings.dry_run and not os.path.isfile( + local_file + ): + logging.warning( + f"{local_file} not found! It may have been in a parent " + f"directory already removed." + ) else: logging.info(f"(DRYRUN): os.remove('{local_file}')") is_empty = True @@ -229,10 +236,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/src/aind_data_upload_utils/delete_staging_folder_job.py b/src/aind_data_upload_utils/delete_staging_folder_job.py index 3c0910e..df39c84 100644 --- a/src/aind_data_upload_utils/delete_staging_folder_job.py +++ b/src/aind_data_upload_utils/delete_staging_folder_job.py @@ -189,10 +189,12 @@ def run_job(self): "--job-settings", required=False, type=str, - help=(r""" + help=( + r""" Instead of init args the job settings can optionally be passed in as a json string in the command line. - """), + """ + ), ) cli_args = parser.parse_args(sys_args) main_job_settings = JobSettings.model_validate_json(cli_args.job_settings) diff --git a/tests/test_delete_source_folders_job.py b/tests/test_delete_source_folders_job.py index e6f5e86..3fab802 100644 --- a/tests/test_delete_source_folders_job.py +++ b/tests/test_delete_source_folders_job.py @@ -299,11 +299,15 @@ def test_s3_check_modalities_to_delete_filter( ) self.assertEqual(5, len(captured.output)) + @patch("os.path.isfile") @patch("os.scandir") - def test_remove_metadata_directory(self, mock_scandir: MagicMock): - """Tests remove_metadata_directory method when empty.""" + def test_remove_metadata_directory( + self, mock_scandir: MagicMock, mock_isfile: MagicMock + ): + """Tests remove_metadata_directory method when files are there.""" mock_scandir.return_value.__enter__.return_value = [] + mock_isfile.return_value = True metadata_files = {"subject.json", "data_description.json"} with self.assertLogs(level="INFO") as captured: self.actual_run_job._remove_metadata_directory( @@ -313,13 +317,34 @@ def test_remove_metadata_directory(self, mock_scandir: MagicMock): self.assertEqual(2, len(self.mock_remove.mock_calls)) self.mock_rmdir.assert_called_once() + @patch("os.path.isfile") + @patch("os.scandir") + def test_remove_metadata_directory_already_removed( + self, mock_scandir: MagicMock, mock_isfile: MagicMock + ): + """Tests remove_metadata_directory method when the parent directory + is already removed.""" + + mock_scandir.return_value.__enter__.return_value = [] + mock_isfile.return_value = False + metadata_files = {"subject.json", "data_description.json"} + with self.assertLogs(level="INFO") as captured: + self.actual_run_job._remove_metadata_directory( + metadata_files_in_both_places=metadata_files + ) + self.assertEqual(3, len(captured.output)) + self.assertEqual(0, len(self.mock_remove.mock_calls)) + self.mock_rmdir.assert_called_once() + + @patch("os.path.isfile") @patch("os.scandir") def test_remove_metadata_directory_not_empty( - self, mock_scandir: MagicMock + self, mock_scandir: MagicMock, mock_isfile: MagicMock ): """Tests remove_metadata_directory method when not emptied.""" mock_scandir.return_value.__enter__.return_value = ["extra_folder"] + mock_isfile.return_value = True metadata_files = {"subject.json", "data_description.json"} with self.assertLogs(level="WARNING"): self.actual_run_job._remove_metadata_directory(