⚡ Bolt: Parallelize folder deletion

main.py

@@ -651,7 +651,7 @@
             return [rule["PK"] for rule in folder_rules if rule.get("PK")]
         except httpx.HTTPError:
             return []
         except Exception as e:
             # We log error but don't stop the whole process;
             # individual folder failure shouldn't crash the sync
             log.warning(f"Error fetching rules for folder {folder_id}: {e}")
@@ -694,7 +694,7 @@
 
         log.info(f"Total existing rules across all folders: {len(all_rules)}")
         return all_rules
     except Exception as e:
         log.error(f"Failed to get existing rules: {sanitize_for_log(e)}")
         return set()
 
@@ -825,7 +825,7 @@
                             grp["PK"],
                         )
                         return str(grp["PK"])
             except Exception as e:
                 log.warning(f"Error fetching groups on attempt {attempt}: {e}")
 
             if attempt < MAX_RETRIES:
@@ -855,7 +855,7 @@
     status: int,
     hostnames: List[str],
     existing_rules: Set[str],
     client: httpx.Client,
 ) -> bool:
     if not hostnames:
         log.info("Folder %s - no rules to push", sanitize_for_log(folder_name))
@@ -1036,6 +1036,55 @@
 # --------------------------------------------------------------------------- #
 # 4. Main workflow
 # --------------------------------------------------------------------------- #
+def delete_existing_folders_parallel(
+    client: httpx.Client,
+    profile_id: str,
+    folder_data_list: List[Dict[str, Any]],
+    existing_folders: Dict[str, str],
+) -> bool:
+    """
+    Identifies and deletes existing folders that match the new folder list.
+    Deletes in parallel to save time.
+    Updates existing_folders in-place by removing deleted folders.
+    Returns True if any deletion occurred.
+    """
+    folders_to_delete = []
+    seen_deletions = set()
+
+    # Identify folders to delete
+    for folder_data in folder_data_list:
+        name = folder_data["group"]["group"].strip()
+        if name in existing_folders and name not in seen_deletions:
+            folders_to_delete.append((name, existing_folders[name]))
+            seen_deletions.add(name)
+
+    if not folders_to_delete:
+        return False
+
+    if not USE_COLORS:
+        log.info(f"Deleting {len(folders_to_delete)} folders in parallel...")
+
+    deletion_occurred = False
+    # Using 5 workers to be safe with rate limits for write operations
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_name = {
+            executor.submit(delete_folder, client, profile_id, name, fid): name
+            for name, fid in folders_to_delete
+        }
+
+        for future in concurrent.futures.as_completed(future_to_name):
+            name = future_to_name[future]
+            try:
+                if future.result():
+                    if name in existing_folders:
+                        del existing_folders[name]
+                    deletion_occurred = True
+            except Exception as e:
+                log.error(f"Failed to delete folder {sanitize_for_log(name)}: {e}")
+
+    return deletion_occurred
+
+
 def sync_profile(
     profile_id: str,
     folder_urls: Sequence[str],
@@ -1148,21 +1197,13 @@
 
             existing_folders = list_existing_folders(client, profile_id)
             if not no_delete:
-                deletion_occurred = False
-                for folder_data in folder_data_list:
-                    name = folder_data["group"]["group"].strip()
-                    if name in existing_folders:
-                        # Optimization: Maintain local state of folders to avoid re-fetching
-                        # delete_folder returns True on success
-                        if delete_folder(
-                            client, profile_id, name, existing_folders[name]
-                        ):
-                            del existing_folders[name]
-                            deletion_occurred = True
+                deletion_occurred = delete_existing_folders_parallel(
+                    client, profile_id, folder_data_list, existing_folders
+                )
 
                 # CRITICAL FIX: Increased wait time for massive folders to clear
                 if deletion_occurred:
                     if not USE_COLORS:
                         log.info(
                             "Waiting 60s for deletions to propagate (prevents 'Badware Hoster' zombie state)..."
                         )

tests/test_parallel_delete.py

@@ -0,0 +1,124 @@
+
+import concurrent.futures
+import time
+from unittest.mock import MagicMock
+import pytest
+import main
+
+def test_delete_existing_folders_parallel(monkeypatch):
+    """
+    Verify that delete_existing_folders_parallel:
+    1. Runs in parallel (speed check).
+    2. Updates existing_folders correctly.
+    3. Returns True if deletions occurred.
+    """
+
+    # Mock delete_folder to simulate network delay
+    # Each deletion takes 0.1s
+    mock_delete = MagicMock()
+    def slow_delete(client, profile_id, name, folder_id):
+        time.sleep(0.1)
+        return True
+
+    mock_delete.side_effect = slow_delete
+    monkeypatch.setattr(main, "delete_folder", mock_delete)
+
+    # Mock log to avoid clutter
+    mock_log = MagicMock()
+    monkeypatch.setattr(main, "log", mock_log)
+
+    # Setup
+    client = MagicMock()
+    profile_id = "test_profile"
+
+    # 10 folders to delete
+    # "group" -> "group" structure mimics the API response structure used in main.py
+    folder_data_list = [{"group": {"group": f"Folder {i}"}} for i in range(10)]
+
+    # All these folders exist in existing_folders
+    existing_folders = {f"Folder {i}": f"id_{i}" for i in range(10)}
+    original_len = len(existing_folders)
+
+    # Add one folder that exists but is NOT in folder_data_list (should NOT be deleted)
+    existing_folders["Keep Me"] = "id_keep"
+
+    # Add one folder in data list that does NOT exist (should check but skip deletion)
+    folder_data_list.append({"group": {"group": "New Folder"}})
+
+    start_time = time.time()
+    result = main.delete_existing_folders_parallel(
+        client, profile_id, folder_data_list, existing_folders
+    )
+    end_time = time.time()
+    duration = end_time - start_time
+
+    # Assertions
+    assert result is True
+
+    # Check that the 10 folders were deleted
+    for i in range(10):
+        assert f"Folder {i}" not in existing_folders
+
+    # Check that "Keep Me" is still there
+    assert "Keep Me" in existing_folders
+    assert existing_folders["Keep Me"] == "id_keep"
+
+    # Check that "New Folder" is not in existing_folders (it wasn't there to begin with)
+    assert "New Folder" not in existing_folders
+
+    # Check performance
+    # 10 tasks * 0.1s = 1.0s sequential time.
+    # With 5 workers, it should take roughly 0.2s + overhead.
+    # We assert it takes < 0.6s to be safe but prove parallelism.
+    assert duration < 0.6, f"Execution took {duration}s, expected < 0.6s (parallel)"
+
+    # Verify mock calls
+    assert mock_delete.call_count == 10
+
+def test_delete_existing_folders_parallel_no_deletions(monkeypatch):
+    """Verify behavior when no folders match."""
+    client = MagicMock()
+    profile_id = "test_profile"
+    folder_data_list = [{"group": {"group": "Folder A"}}]
+    existing_folders = {"Folder B": "id_b"} # No match
+
+    result = main.delete_existing_folders_parallel(
+        client, profile_id, folder_data_list, existing_folders
+    )
+
+    assert result is False
+    assert "Folder B" in existing_folders
+
+def test_delete_existing_folders_parallel_partial_failure(monkeypatch):
+    """Verify behavior when some deletions fail."""
+
+    # Mock delete_folder: Fail for even numbers
+    mock_delete = MagicMock()
+    def conditional_delete(client, profile_id, name, folder_id):
+        # name is "Folder i"
+        num = int(name.split()[1])
+        return num % 2 != 0 # Return True for odd (success), False for even (fail)
+
+    mock_delete.side_effect = conditional_delete
+    monkeypatch.setattr(main, "delete_folder", mock_delete)
+
+    client = MagicMock()
+    profile_id = "test_profile"
+    folder_data_list = [{"group": {"group": f"Folder {i}"}} for i in range(10)]
+    existing_folders = {f"Folder {i}": f"id_{i}" for i in range(10)}
+
+    result = main.delete_existing_folders_parallel(
+        client, profile_id, folder_data_list, existing_folders
+    )
+
+    assert result is True # At least one succeeded
+
+    # Check results
+    for i in range(10):
+        name = f"Folder {i}"
+        if i % 2 != 0:
+            # Odd: Succeeded, should be removed
+            assert name not in existing_folders
+        else:
+            # Even: Failed, should remain
+            assert name in existing_folders