Skip to content

⚡ Bolt: Remove redundant API call to list folders #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
google-labs-jules wants to merge 1 commit into from
Closed

⚡ Bolt: Remove redundant API call to list folders #99

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,11 @@
log.error(f"Failed to list existing folders: {sanitize_for_log(e)}")
return {}

def get_all_existing_rules(client: httpx.Client, profile_id: str) -> Set[str]:
def get_all_existing_rules(

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning

Missing function or method docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Missing function docstring Warning

Missing function docstring
client: httpx.Client,

Check warning

Code scanning / Pylint (reported by Codacy)

Wrong hanging indentation before block (add 4 spaces). Warning

Wrong hanging indentation before block (add 4 spaces).
profile_id: str,
existing_folders_map: Optional[Dict[str, str]] = None,

Check warning

Code scanning / Pylint (reported by Codacy)

Wrong hanging indentation before block (add 4 spaces). Warning

Wrong hanging indentation before block (add 4 spaces).
) -> Set[str]:
all_rules = set()
all_rules_lock = threading.Lock()

Expand Down Expand Up @@ -356,7 +360,10 @@
pass

# Get rules from folders in parallel
folders = list_existing_folders(client, profile_id)
if existing_folders_map is not None:
folders = existing_folders_map
else:
folders = list_existing_folders(client, profile_id)

# Parallelize fetching rules from folders.
# Using 5 workers to be safe with rate limits, though GETs are usually cheaper.
Expand Down Expand Up @@ -521,7 +528,7 @@

if successful_batches == total_batches:
log.info("Folder %s – finished (%d new rules added)", sanitize_for_log(folder_name), len(filtered_hostnames))
return True

Check warning

Code scanning / Pylint (reported by Codacy)

Wrong hanging indentation before block (add 4 spaces). Warning

Wrong hanging indentation before block (add 4 spaces).
else:
log.error("Folder %s – only %d/%d batches succeeded", sanitize_for_log(folder_name), successful_batches, total_batches)
return False
Expand Down Expand Up @@ -649,16 +656,17 @@
for folder_data in folder_data_list:
name = folder_data["group"]["group"].strip()
if name in existing_folders:
delete_folder(client, profile_id, name, existing_folders[name])
deletion_occurred = True
if delete_folder(client, profile_id, name, existing_folders[name]):
del existing_folders[name]
deletion_occurred = True

# CRITICAL FIX: Increased wait time for massive folders to clear
if deletion_occurred:
if not USE_COLORS:
log.info("Waiting 60s for deletions to propagate (prevents 'Badware Hoster' zombie state)...")
countdown_timer(60, "Waiting for deletions to propagate")

existing_rules = get_all_existing_rules(client, profile_id)
existing_rules = get_all_existing_rules(client, profile_id, existing_folders)

with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_folder = {
Expand Down
Loading