Fix rm metadata dir

src/aind_data_upload_utils/check_directories_job.py

@@ -202,10 +202,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/check_metadata_job.py

@@ -111,10 +111,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/copy_metadata_files.py

@@ -76,10 +76,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/create_s5_commands_job.py

@@ -217,10 +217,12 @@ def run_job(self) -> None:
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_folders_job.py

@@ -84,10 +84,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_source_folders_job.py

@@ -169,23 +169,36 @@ def _remove_metadata_directory(
         metadata_dir = self.job_settings.directories.metadata_dir
         for metadata_file in metadata_files_in_both_places:
             local_file = Path(metadata_dir) / metadata_file
-            if not self.job_settings.dry_run:
+            if not self.job_settings.dry_run and os.path.isfile(local_file):
                 logging.info(f"Removing {local_file}")
                 os.remove(local_file)
+            elif not self.job_settings.dry_run and not os.path.isfile(
+                local_file
+            ):
+                logging.warning(
+                    f"{local_file} not found! It may have been in a parent "
+                    f"directory already removed."
+                )
             else:
                 logging.info(f"(DRYRUN): os.remove('{local_file}')")
         is_empty = True
-        with os.scandir(metadata_dir) as it:
-            if any(it):
-                is_empty = False
+        if os.path.isdir(metadata_dir):
+            with os.scandir(metadata_dir) as it:
+                if any(it):
+                    is_empty = False
         if not is_empty and not self.job_settings.dry_run:
             logging.warning(
                 f"There are extra files or folders found in {metadata_dir}! "
                 f"Will not auto-delete!"
             )
-        elif not self.job_settings.dry_run:
+        elif not self.job_settings.dry_run and os.path.isdir(metadata_dir):
             logging.info(f"Removing {metadata_dir}")
             os.rmdir(metadata_dir)
+        elif not os.path.isdir(metadata_dir):
+            logging.warning(
+                f"{metadata_dir} not found! It may have been in a parent "
+                f"directory already removed."
+            )
         else:
             logging.info(f"(DRYRUN): os.rmdir('{metadata_dir}')")
 
@@ -229,10 +242,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

src/aind_data_upload_utils/delete_staging_folder_job.py

@@ -189,10 +189,12 @@ def run_job(self):
         "--job-settings",
         required=False,
         type=str,
-        help=(r"""
+        help=(
+            r"""
             Instead of init args the job settings can optionally be passed in
             as a json string in the command line.
-            """),
+            """
+        ),
     )
     cli_args = parser.parse_args(sys_args)
     main_job_settings = JobSettings.model_validate_json(cli_args.job_settings)

tests/test_delete_source_folders_job.py

@@ -299,11 +299,15 @@ def test_s3_check_modalities_to_delete_filter(
         )
         self.assertEqual(5, len(captured.output))
 
+    @patch("os.path.isfile")
     @patch("os.scandir")
-    def test_remove_metadata_directory(self, mock_scandir: MagicMock):
-        """Tests remove_metadata_directory method when empty."""
+    def test_remove_metadata_directory(
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
+    ):
+        """Tests remove_metadata_directory method when files are there."""
 
         mock_scandir.return_value.__enter__.return_value = []
+        mock_isfile.return_value = True
         metadata_files = {"subject.json", "data_description.json"}
         with self.assertLogs(level="INFO") as captured:
             self.actual_run_job._remove_metadata_directory(
@@ -313,13 +317,34 @@ def test_remove_metadata_directory(self, mock_scandir: MagicMock):
         self.assertEqual(2, len(self.mock_remove.mock_calls))
         self.mock_rmdir.assert_called_once()
 
+    @patch("os.path.isfile")
+    @patch("os.scandir")
+    def test_remove_metadata_directory_already_removed(
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
+    ):
+        """Tests remove_metadata_directory method when the parent directory
+        is already removed."""
+
+        mock_scandir.return_value.__enter__.return_value = []
+        mock_isfile.return_value = False
+        metadata_files = {"subject.json", "data_description.json"}
+        with self.assertLogs(level="INFO") as captured:
+            self.actual_run_job._remove_metadata_directory(
+                metadata_files_in_both_places=metadata_files
+            )
+        self.assertEqual(3, len(captured.output))
+        self.assertEqual(0, len(self.mock_remove.mock_calls))
+        self.mock_rmdir.assert_called_once()
+
+    @patch("os.path.isfile")
     @patch("os.scandir")
     def test_remove_metadata_directory_not_empty(
-        self, mock_scandir: MagicMock
+        self, mock_scandir: MagicMock, mock_isfile: MagicMock
     ):
         """Tests remove_metadata_directory method when not emptied."""
 
         mock_scandir.return_value.__enter__.return_value = ["extra_folder"]
+        mock_isfile.return_value = True
         metadata_files = {"subject.json", "data_description.json"}
         with self.assertLogs(level="WARNING"):
             self.actual_run_job._remove_metadata_directory(
@@ -328,6 +353,29 @@ def test_remove_metadata_directory_not_empty(
         self.assertEqual(2, len(self.mock_remove.mock_calls))
         self.mock_rmdir.assert_not_called()
 
+    @patch("os.path.isdir")
+    @patch("os.path.isfile")
+    @patch("os.scandir")
+    def test_remove_metadata_directory_no_directory(
+        self,
+        mock_scandir: MagicMock,
+        mock_isfile: MagicMock,
+        mock_isdir: MagicMock,
+    ):
+        """Tests remove_metadata_directory method when no directory."""
+
+        mock_isdir.return_value = False
+        mock_isfile.return_value = False
+        metadata_files = {"subject.json", "data_description.json"}
+        with self.assertLogs(level="WARNING") as captured:
+            self.actual_run_job._remove_metadata_directory(
+                metadata_files_in_both_places=metadata_files
+            )
+        self.mock_remove.assert_not_called()
+        mock_scandir.assert_not_called()
+        self.mock_rmdir.assert_not_called()
+        self.assertEqual(3, len(captured.output))
+
     @patch("os.scandir")
     def test_remove_metadata_directory_dry_run(self, mock_scandir: MagicMock):
         """Tests remove_metadata_directory method when dry run set."""