diff --git a/main.py b/main.py index 86792da4..83b318be 100644 --- a/main.py +++ b/main.py @@ -1036,6 +1036,55 @@ def _process_single_folder( # --------------------------------------------------------------------------- # # 4. Main workflow # --------------------------------------------------------------------------- # +def delete_existing_folders_parallel( + client: httpx.Client, + profile_id: str, + folder_data_list: List[Dict[str, Any]], + existing_folders: Dict[str, str], +) -> bool: + """ + Identifies and deletes existing folders that match the new folder list. + Deletes in parallel to save time. + Updates existing_folders in-place by removing deleted folders. + Returns True if any deletion occurred. + """ + folders_to_delete = [] + seen_deletions = set() + + # Identify folders to delete + for folder_data in folder_data_list: + name = folder_data["group"]["group"].strip() + if name in existing_folders and name not in seen_deletions: + folders_to_delete.append((name, existing_folders[name])) + seen_deletions.add(name) + + if not folders_to_delete: + return False + + if not USE_COLORS: + log.info(f"Deleting {len(folders_to_delete)} folders in parallel...") + + deletion_occurred = False + # Using 5 workers to be safe with rate limits for write operations + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + future_to_name = { + executor.submit(delete_folder, client, profile_id, name, fid): name + for name, fid in folders_to_delete + } + + for future in concurrent.futures.as_completed(future_to_name): + name = future_to_name[future] + try: + if future.result(): + if name in existing_folders: + del existing_folders[name] + deletion_occurred = True + except Exception as e: + log.error(f"Failed to delete folder {sanitize_for_log(name)}: {e}") + + return deletion_occurred + + def sync_profile( profile_id: str, folder_urls: Sequence[str], @@ -1148,17 +1197,9 @@ def _fetch_if_valid(url: str): existing_folders = list_existing_folders(client, profile_id) if not no_delete: - deletion_occurred = False - for folder_data in folder_data_list: - name = folder_data["group"]["group"].strip() - if name in existing_folders: - # Optimization: Maintain local state of folders to avoid re-fetching - # delete_folder returns True on success - if delete_folder( - client, profile_id, name, existing_folders[name] - ): - del existing_folders[name] - deletion_occurred = True + deletion_occurred = delete_existing_folders_parallel( + client, profile_id, folder_data_list, existing_folders + ) # CRITICAL FIX: Increased wait time for massive folders to clear if deletion_occurred: diff --git a/tests/test_parallel_delete.py b/tests/test_parallel_delete.py new file mode 100644 index 00000000..78b59f4c --- /dev/null +++ b/tests/test_parallel_delete.py @@ -0,0 +1,124 @@ + +import concurrent.futures +import time +from unittest.mock import MagicMock +import pytest +import main + +def test_delete_existing_folders_parallel(monkeypatch): + """ + Verify that delete_existing_folders_parallel: + 1. Runs in parallel (speed check). + 2. Updates existing_folders correctly. + 3. Returns True if deletions occurred. + """ + + # Mock delete_folder to simulate network delay + # Each deletion takes 0.1s + mock_delete = MagicMock() + def slow_delete(client, profile_id, name, folder_id): + time.sleep(0.1) + return True + + mock_delete.side_effect = slow_delete + monkeypatch.setattr(main, "delete_folder", mock_delete) + + # Mock log to avoid clutter + mock_log = MagicMock() + monkeypatch.setattr(main, "log", mock_log) + + # Setup + client = MagicMock() + profile_id = "test_profile" + + # 10 folders to delete + # "group" -> "group" structure mimics the API response structure used in main.py + folder_data_list = [{"group": {"group": f"Folder {i}"}} for i in range(10)] + + # All these folders exist in existing_folders + existing_folders = {f"Folder {i}": f"id_{i}" for i in range(10)} + original_len = len(existing_folders) + + # Add one folder that exists but is NOT in folder_data_list (should NOT be deleted) + existing_folders["Keep Me"] = "id_keep" + + # Add one folder in data list that does NOT exist (should check but skip deletion) + folder_data_list.append({"group": {"group": "New Folder"}}) + + start_time = time.time() + result = main.delete_existing_folders_parallel( + client, profile_id, folder_data_list, existing_folders + ) + end_time = time.time() + duration = end_time - start_time + + # Assertions + assert result is True + + # Check that the 10 folders were deleted + for i in range(10): + assert f"Folder {i}" not in existing_folders + + # Check that "Keep Me" is still there + assert "Keep Me" in existing_folders + assert existing_folders["Keep Me"] == "id_keep" + + # Check that "New Folder" is not in existing_folders (it wasn't there to begin with) + assert "New Folder" not in existing_folders + + # Check performance + # 10 tasks * 0.1s = 1.0s sequential time. + # With 5 workers, it should take roughly 0.2s + overhead. + # We assert it takes < 0.6s to be safe but prove parallelism. + assert duration < 0.6, f"Execution took {duration}s, expected < 0.6s (parallel)" + + # Verify mock calls + assert mock_delete.call_count == 10 + +def test_delete_existing_folders_parallel_no_deletions(monkeypatch): + """Verify behavior when no folders match.""" + client = MagicMock() + profile_id = "test_profile" + folder_data_list = [{"group": {"group": "Folder A"}}] + existing_folders = {"Folder B": "id_b"} # No match + + result = main.delete_existing_folders_parallel( + client, profile_id, folder_data_list, existing_folders + ) + + assert result is False + assert "Folder B" in existing_folders + +def test_delete_existing_folders_parallel_partial_failure(monkeypatch): + """Verify behavior when some deletions fail.""" + + # Mock delete_folder: Fail for even numbers + mock_delete = MagicMock() + def conditional_delete(client, profile_id, name, folder_id): + # name is "Folder i" + num = int(name.split()[1]) + return num % 2 != 0 # Return True for odd (success), False for even (fail) + + mock_delete.side_effect = conditional_delete + monkeypatch.setattr(main, "delete_folder", mock_delete) + + client = MagicMock() + profile_id = "test_profile" + folder_data_list = [{"group": {"group": f"Folder {i}"}} for i in range(10)] + existing_folders = {f"Folder {i}": f"id_{i}" for i in range(10)} + + result = main.delete_existing_folders_parallel( + client, profile_id, folder_data_list, existing_folders + ) + + assert result is True # At least one succeeded + + # Check results + for i in range(10): + name = f"Folder {i}" + if i % 2 != 0: + # Odd: Succeeded, should be removed + assert name not in existing_folders + else: + # Even: Failed, should remain + assert name in existing_folders