From cdd4dd16636cdac64e06ab3360c88606ec818509 Mon Sep 17 00:00:00 2001 From: Viktor Arvidsson Date: Sun, 15 Mar 2026 12:24:18 +0100 Subject: [PATCH 1/2] fix(gmail): don't call gmail api when there's nothing to import --- src/fetcher/run.py | 117 ++++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/src/fetcher/run.py b/src/fetcher/run.py index cff98a6..3017c4a 100644 --- a/src/fetcher/run.py +++ b/src/fetcher/run.py @@ -100,21 +100,6 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str, inbox_label_id = None unread_label_id = None - if not dry_run: - try: - service = get_gmail_service( - gmail_cfg["credentials_path"], - gmail_cfg["token_path"], - ) - inbox_label_id = get_inbox_label_id(service, USER_ID) - unread_label_id = get_unread_label_id(service, USER_ID) - label_id = _ensure_label(service, label_name) if label_name else None - if label_name and not label_id: - label_id = _ensure_label(service, label_name) - except Exception as e: - logger.exception("Gmail API init failed: %s", e) - return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0} - for msg in messages_iter: if state.seen_hash(msg.message_hash): logger.debug("Skipping duplicate (hash): uid=%s", msg.uid) @@ -153,6 +138,21 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str, imported += 1 continue + if not service: + try: + service = get_gmail_service( + gmail_cfg["credentials_path"], + gmail_cfg["token_path"], + ) + inbox_label_id = get_inbox_label_id(service, USER_ID) + unread_label_id = get_unread_label_id(service, USER_ID) + label_id = _ensure_label(service, label_name) if label_name else None + if label_name and not label_id: + label_id = _ensure_label(service, label_name) + except Exception as e: + logger.exception("Gmail API init failed: %s", e) + return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0} + try: label_ids = [label_id] if label_id else [] # Fetch/polling only gets NEW mail (UID > last); always import as unread. @@ -267,19 +267,6 @@ def run_copy_all( inbox_label_id = None unread_label_id = None - if not dry_run: - try: - service = get_gmail_service( - gmail_cfg["credentials_path"], - gmail_cfg["token_path"], - ) - inbox_label_id = get_inbox_label_id(service, USER_ID) - unread_label_id = get_unread_label_id(service, USER_ID) - label_id = _ensure_label(service, label_name) if label_name else None - except Exception as e: - logger.exception("Gmail API init failed: %s", e) - return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0} - for msg in messages_iter: if state.seen_hash(msg.message_hash): logger.debug("Copy-all: skipping duplicate (hash): uid=%s", msg.uid) @@ -301,37 +288,6 @@ def run_copy_all( logger.warning("Delete (duplicate) failed for uid=%s: %s", msg.uid, e) continue - if not dry_run: - message_id_header = _parse_message_id_from_raw(msg.raw) - if message_id_header and gmail_has_message_with_id( - service, USER_ID, message_id_header - ): - logger.debug( - "Copy-all: skipping (already in Gmail by Message-ID): uid=%s", - msg.uid, - ) - skipped_duplicate += 1 - if delete_after_import: - try: - delete_and_expunge( - imap_cfg["host"], - int(imap_cfg.get("port", 993)), - imap_cfg["username"], - imap_cfg["password"], - mailbox, - msg.uid, - imap_cfg.get("use_ssl", True), - ) - state.set_last_processed_uid(mailbox, uid_validity, msg.uid) - deleted += 1 - except Exception as e: - logger.warning( - "Delete (already in Gmail) failed for uid=%s: %s", - msg.uid, - e, - ) - continue - if dry_run: logger.info( "[DRY-RUN] Copy-all would import uid=%s (hash=%s)", @@ -341,6 +297,49 @@ def run_copy_all( imported += 1 continue + if not service: + try: + service = get_gmail_service( + gmail_cfg["credentials_path"], + gmail_cfg["token_path"], + ) + inbox_label_id = get_inbox_label_id(service, USER_ID) + unread_label_id = get_unread_label_id(service, USER_ID) + label_id = _ensure_label(service, label_name) if label_name else None + except Exception as e: + logger.exception("Gmail API init failed: %s", e) + return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0} + + message_id_header = _parse_message_id_from_raw(msg.raw) + if message_id_header and gmail_has_message_with_id( + service, USER_ID, message_id_header + ): + logger.debug( + "Copy-all: skipping (already in Gmail by Message-ID): uid=%s", + msg.uid, + ) + skipped_duplicate += 1 + if delete_after_import: + try: + delete_and_expunge( + imap_cfg["host"], + int(imap_cfg.get("port", 993)), + imap_cfg["username"], + imap_cfg["password"], + mailbox, + msg.uid, + imap_cfg.get("use_ssl", True), + ) + state.set_last_processed_uid(mailbox, uid_validity, msg.uid) + deleted += 1 + except Exception as e: + logger.warning( + "Delete (already in Gmail) failed for uid=%s: %s", + msg.uid, + e, + ) + continue + try: label_ids = [label_id] if label_id else [] # Copy-all: preserve read/unread from ISP (\Seen). From fa20857367e9a2167d700b9cf5966dc086a64c82 Mon Sep 17 00:00:00 2001 From: Viktor Arvidsson Date: Sun, 15 Mar 2026 12:38:34 +0100 Subject: [PATCH 2/2] feat(imap): add optional since_date for limiting import to after a specific date --- src/fetcher/cli.py | 2 ++ src/fetcher/imap_client.py | 27 ++++++++++++++++++++++----- src/fetcher/run.py | 8 ++++++++ src/fetcher/web_ui.py | 23 +++++++++++++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/fetcher/cli.py b/src/fetcher/cli.py index 7db60ae..97c26e1 100644 --- a/src/fetcher/cli.py +++ b/src/fetcher/cli.py @@ -269,6 +269,7 @@ def _write_default_config(dest: Path) -> None: "mailbox": "INBOX", "use_ssl": True, "delete_after_import": True, + "since_date": None, }, "gmail": {"use_label": False, "label": "ISP Mail", "credentials_path": "credentials.json", "token_path": "token.json"}, "state": {"db_path": "state.db"}, @@ -301,6 +302,7 @@ def config_wizard_interactive() -> None: "mailbox": mailbox, "use_ssl": True, "delete_after_import": True, + "since_date": None, }, "gmail": {"use_label": bool(label), "label": label or "ISP Mail", "credentials_path": cred_path, "token_path": token_path}, "state": {"db_path": db_path}, diff --git a/src/fetcher/imap_client.py b/src/fetcher/imap_client.py index 7292e05..adc27ff 100644 --- a/src/fetcher/imap_client.py +++ b/src/fetcher/imap_client.py @@ -10,6 +10,7 @@ import imaplib import logging import ssl +import datetime from dataclasses import dataclass from typing import Iterator @@ -102,6 +103,15 @@ def get_uid_validity( pass +def _format_imap_date(date_val: str | datetime.date) -> str: + """Convert an ISO date string or date object to IMAP DD-MMM-YYYY.""" + if isinstance(date_val, str): + date_val = datetime.date.fromisoformat(date_val) + if not isinstance(date_val, datetime.date): + raise TypeError("since_date must be a date or ISO date string") + return date_val.strftime("%d-%b-%Y") + + def fetch_messages( host: str, port: int, @@ -110,11 +120,14 @@ def fetch_messages( mailbox: str = "INBOX", use_ssl: bool = True, last_processed_uid: int | None = None, + since: str | datetime.date | None = None, ) -> tuple[int, Iterator[FetchedMessage]]: """ Connect via IMAPS, select mailbox, return (uid_validity, iterator of new messages). - Messages are those with UID > last_processed_uid. Ordered by ascending UID. + Messages are those with UID > last_processed_uid and, if since is + provided, whose INTERNALDATE is on or after that date. + Caller must handle UIDVALIDITY changes (e.g. reset state when it changes). """ ssl_context = ssl.create_default_context() if use_ssl else None @@ -131,12 +144,16 @@ def fetch_messages( status = conn.status(mailbox, "(UIDVALIDITY)") # e.g. STATUS INBOX (UIDVALIDITY 12345) uid_validity = int(status[1][0].decode().split("UIDVALIDITY")[1].strip(" ()")) - # Search UIDs > last_processed_uid (all if last_processed_uid is None) + # Search UIDs > last_processed_uid (all if last_processed_uid is None) and optionally SINCE date. + criteria_parts: list[str] = [] if last_processed_uid is not None: - search_criteria = f"UID {last_processed_uid + 1}:*" + criteria_parts.append(f"UID {last_processed_uid + 1}:*") else: - search_criteria = "UID 1:*" - _, data = conn.uid("SEARCH", None, search_criteria) + criteria_parts.append("UID 1:*") + if since is not None: + imap_date = _format_imap_date(since) + criteria_parts.append(f"SINCE {imap_date}") + _, data = conn.uid("SEARCH", None, " ".join(criteria_parts)) if not data or not data[0]: try: conn.logout() diff --git a/src/fetcher/run.py b/src/fetcher/run.py index 3017c4a..cc09ba7 100644 --- a/src/fetcher/run.py +++ b/src/fetcher/run.py @@ -58,6 +58,9 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str, db_path = state_cfg.get("db_path", "state.db") mailbox = imap_cfg.get("mailbox", "INBOX") delete_after_import = imap_cfg.get("delete_after_import", True) + since_date = imap_cfg.get("since_date") + if since_date == "": + since_date = None use_label = gmail_cfg.get("use_label") if "use_label" in gmail_cfg else bool((gmail_cfg.get("label") or "").strip()) label_name = (gmail_cfg.get("label") or "ISP Mail").strip() if use_label else None @@ -87,6 +90,7 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str, mailbox=mailbox, use_ssl=imap_cfg.get("use_ssl", True), last_processed_uid=last_uid, + since=since_date, ) except Exception as e: logger.exception("IMAP fetch failed: %s", e) @@ -226,6 +230,9 @@ def run_copy_all( state_cfg = cfg.get("state", {}) db_path = state_cfg.get("db_path", "state.db") mailbox = imap_cfg.get("mailbox", "INBOX") + since_date = imap_cfg.get("since_date") + if since_date == "": + since_date = None use_label = gmail_cfg.get("use_label") if "use_label" in gmail_cfg else bool((gmail_cfg.get("label") or "").strip()) label_name = (gmail_cfg.get("label") or "ISP Mail").strip() if use_label else None @@ -254,6 +261,7 @@ def run_copy_all( mailbox=mailbox, use_ssl=imap_cfg.get("use_ssl", True), last_processed_uid=None, + since=since_date, ) except Exception as e: logger.exception("IMAP fetch failed: %s", e) diff --git a/src/fetcher/web_ui.py b/src/fetcher/web_ui.py index 50a43fd..9ded5d1 100644 --- a/src/fetcher/web_ui.py +++ b/src/fetcher/web_ui.py @@ -11,6 +11,7 @@ import os import threading import time +import datetime from contextlib import asynccontextmanager from pathlib import Path from typing import Any @@ -221,6 +222,7 @@ class ImapConfigSafe(BaseModel): mailbox: str use_ssl: bool delete_after_import: bool + since_date: str | None = None class GmailConfigSafe(BaseModel): @@ -432,6 +434,7 @@ def _default_config_response() -> ConfigResponse: mailbox="INBOX", use_ssl=True, delete_after_import=True, + since_date=None, ), gmail=GmailConfigSafe( use_label=False, @@ -476,6 +479,7 @@ def api_config(request: Request) -> ConfigResponse: mailbox=imap.get("mailbox", "INBOX"), use_ssl=imap.get("use_ssl", True), delete_after_import=imap.get("delete_after_import", True), + since_date=imap.get("since_date"), ), gmail=GmailConfigSafe( use_label=gmail.get("use_label") if "use_label" in gmail else bool((gmail.get("label") or "").strip()), @@ -503,6 +507,7 @@ class ConfigUpdate(BaseModel): imap_username: str | None = None imap_mailbox: str | None = None imap_use_ssl: bool | None = None + imap_since_date: str | None = None imap_password: str | None = None # stored in .env, not in config delete_after_import: bool | None = None gmail_use_label: bool | None = None @@ -535,6 +540,11 @@ def api_setup(request: Request, body: SetupBody) -> dict[str, str]: path = _get_config_path() if path.exists(): raise HTTPException(status_code=400, detail="Config already exists") + if body.imap_since_date: + try: + datetime.date.fromisoformat(body.imap_since_date) + except Exception as e: + raise HTTPException(status_code=400, detail="since_date must be YYYY-MM-DD") try: _verify_imap_credentials( host=body.imap_host, @@ -554,6 +564,7 @@ def api_setup(request: Request, body: SetupBody) -> dict[str, str]: "username": body.imap_username, "password_env": "IMAP_PASSWORD", "mailbox": body.imap_mailbox, + "since_date": body.imap_since_date, "use_ssl": body.imap_use_ssl, "delete_after_import": body.delete_after_import, }, @@ -616,6 +627,13 @@ def api_config_update(request: Request, update: ConfigUpdate) -> dict[str, str]: imap["username"] = update.imap_username if update.imap_mailbox is not None: imap["mailbox"] = update.imap_mailbox + if update.imap_since_date is not None: + try: + if update.imap_since_date != "": + datetime.date.fromisoformat(update.imap_since_date) + except Exception: + raise HTTPException(status_code=400, detail="since_date must be YYYY-MM-DD") + cfg.setdefault("imap", {})["since_date"] = update.imap_since_date if update.imap_use_ssl is not None: imap["use_ssl"] = update.imap_use_ssl if update.delete_after_import is not None: @@ -773,6 +791,7 @@ def api_status(request: Request) -> dict[str, Any]:

Mailbox is the IMAP folder to fetch from. INBOX is the main inbox where new mail arrives at your ISP; leave as INBOX unless you use a different folder.

+ @@ -827,6 +846,7 @@ def api_status(request: Request) -> dict[str, Any]:

Mailbox is the IMAP folder to fetch from. INBOX = main inbox where new mail arrives.

+ @@ -894,6 +914,7 @@ def api_status(request: Request) -> dict[str, Any]: document.getElementById('imap_use_ssl').checked = c.imap.use_ssl; document.getElementById('imap_username').value = c.imap.username; document.getElementById('imap_mailbox').value = c.imap.mailbox; + document.getElementById('imap_since_date').value = c.imap.since_date || ''; document.getElementById('delete_after_import').checked = c.imap.delete_after_import; document.getElementById('gmail_use_label').checked = c.gmail.use_label; document.getElementById('gmail_label_row').style.display = c.gmail.use_label ? 'block' : 'none'; @@ -1059,6 +1080,7 @@ def api_status(request: Request) -> dict[str, Any]: imap_use_ssl: document.getElementById('imap_use_ssl').checked, imap_username: document.getElementById('imap_username').value, imap_mailbox: document.getElementById('imap_mailbox').value, + imap_since_date: document.getElementById('imap_since_date').value || null, delete_after_import: document.getElementById('delete_after_import').checked, gmail_use_label: document.getElementById('gmail_use_label').checked, gmail_label: document.getElementById('gmail_label').value, @@ -1093,6 +1115,7 @@ def api_status(request: Request) -> dict[str, Any]: imap_username: document.getElementById('s_imap_username').value, imap_password: document.getElementById('s_imap_password').value, imap_mailbox: document.getElementById('s_imap_mailbox').value, + imap_since_date: document.getElementById('s_imap_since_date').value || null, delete_after_import: document.getElementById('s_delete_after_import').checked, gmail_use_label: document.getElementById('s_gmail_use_label').checked, gmail_label: document.getElementById('s_gmail_label').value