fix: adds check that files exist before removing them

src/aind_data_upload_utils/check_directories_job.py

@@ -202,10 +202,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/check_metadata_job.py

@@ -111,10 +111,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/copy_metadata_files.py

@@ -76,10 +76,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/create_s5_commands_job.py

@@ -217,10 +217,12 @@ def run_job(self) -> None:
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_folders_job.py

@@ -84,10 +84,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_source_folders_job.py

@@ -169,9 +169,16 @@ def _remove_metadata_directory(
         metadata_dir = self.job_settings.directories.metadata_dir
         for metadata_file in metadata_files_in_both_places:
             local_file = Path(metadata_dir) / metadata_file
-            if not self.job_settings.dry_run:
+            if not self.job_settings.dry_run and os.path.isfile(local_file):
                 logging.info(f"Removing {local_file}")
                 os.remove(local_file)
+            elif not self.job_settings.dry_run and not os.path.isfile(
+                local_file
+            ):
+                logging.warning(
+                    f"{local_file} not found! It may have been in a parent "
+                    f"directory already removed."
+                )
             else:
                 logging.info(f"(DRYRUN): os.remove('{local_file}')")
         is_empty = True
@@ -229,10 +236,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_staging_folder_job.py

@@ -189,10 +189,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

tests/test_delete_source_folders_job.py

@@ -299,11 +299,15 @@ def test_s3_check_modalities_to_delete_filter(
         )
         self.assertEqual(5, len(captured.output))
 
+    @patch("os.path.isfile")
     @patch("os.scandir")
-    def test_remove_metadata_directory(self, mock_scandir: MagicMock):
-        """Tests remove_metadata_directory method when empty."""
+    def test_remove_metadata_directory(
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
+    ):
+        """Tests remove_metadata_directory method when files are there."""
 
         mock_scandir.return_value.__enter__.return_value = []
+        mock_isfile.return_value = True
         metadata_files = {"subject.json", "data_description.json"}
         with self.assertLogs(level="INFO") as captured:
             self.actual_run_job._remove_metadata_directory(
@@ -313,13 +317,34 @@ def test_remove_metadata_directory(self, mock_scandir: MagicMock):
         self.assertEqual(2, len(self.mock_remove.mock_calls))
         self.mock_rmdir.assert_called_once()
 
+    @patch("os.path.isfile")
+    @patch("os.scandir")
+    def test_remove_metadata_directory_already_removed(
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
+    ):
+        """Tests remove_metadata_directory method when the parent directory
+        is already removed."""
+
+        mock_scandir.return_value.__enter__.return_value = []
+        mock_isfile.return_value = False
+        metadata_files = {"subject.json", "data_description.json"}
+        with self.assertLogs(level="INFO") as captured:
+            self.actual_run_job._remove_metadata_directory(
+                metadata_files_in_both_places=metadata_files
+            )
+        self.assertEqual(3, len(captured.output))
+        self.assertEqual(0, len(self.mock_remove.mock_calls))
+        self.mock_rmdir.assert_called_once()
+
+    @patch("os.path.isfile")
     @patch("os.scandir")
     def test_remove_metadata_directory_not_empty(
-        self, mock_scandir: MagicMock
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
     ):
         """Tests remove_metadata_directory method when not emptied."""
 
         mock_scandir.return_value.__enter__.return_value = ["extra_folder"]
+        mock_isfile.return_value = True
         metadata_files = {"subject.json", "data_description.json"}
         with self.assertLogs(level="WARNING"):
             self.actual_run_job._remove_metadata_directory(