Skip to content

Import mail that is newer than a set date #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Cyriuz wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Import mail that is newer than a set date #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cdd4dd1
fix(gmail): don't call gmail api when there's nothing to import
Cyriuz Mar 15, 2026
fa20857
feat(imap): add optional since_date for limiting import to after a sp…
Cyriuz Mar 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/fetcher/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ def _write_default_config(dest: Path) -> None:
"mailbox": "INBOX",
"use_ssl": True,
"delete_after_import": True,
"since_date": None,
},
"gmail": {"use_label": False, "label": "ISP Mail", "credentials_path": "credentials.json", "token_path": "token.json"},
"state": {"db_path": "state.db"},
Expand Down Expand Up @@ -301,6 +302,7 @@ def config_wizard_interactive() -> None:
"mailbox": mailbox,
"use_ssl": True,
"delete_after_import": True,
"since_date": None,
},
"gmail": {"use_label": bool(label), "label": label or "ISP Mail", "credentials_path": cred_path, "token_path": token_path},
"state": {"db_path": db_path},
Expand Down
27 changes: 22 additions & 5 deletions src/fetcher/imap_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import imaplib
import logging
import ssl
import datetime
from dataclasses import dataclass
from typing import Iterator

Expand Down Expand Up @@ -102,6 +103,15 @@ def get_uid_validity(
pass


def _format_imap_date(date_val: str | datetime.date) -> str:
"""Convert an ISO date string or date object to IMAP DD-MMM-YYYY."""
if isinstance(date_val, str):
date_val = datetime.date.fromisoformat(date_val)
if not isinstance(date_val, datetime.date):
raise TypeError("since_date must be a date or ISO date string")
return date_val.strftime("%d-%b-%Y")


def fetch_messages(
host: str,
port: int,
Expand All @@ -110,11 +120,14 @@ def fetch_messages(
mailbox: str = "INBOX",
use_ssl: bool = True,
last_processed_uid: int | None = None,
since: str | datetime.date | None = None,
) -> tuple[int, Iterator[FetchedMessage]]:
"""
Connect via IMAPS, select mailbox, return (uid_validity, iterator of new messages).

Messages are those with UID > last_processed_uid. Ordered by ascending UID.
Messages are those with UID > last_processed_uid and, if since is
provided, whose INTERNALDATE is on or after that date.

Caller must handle UIDVALIDITY changes (e.g. reset state when it changes).
"""
ssl_context = ssl.create_default_context() if use_ssl else None
Expand All @@ -131,12 +144,16 @@ def fetch_messages(
status = conn.status(mailbox, "(UIDVALIDITY)")
# e.g. STATUS INBOX (UIDVALIDITY 12345)
uid_validity = int(status[1][0].decode().split("UIDVALIDITY")[1].strip(" ()"))
# Search UIDs > last_processed_uid (all if last_processed_uid is None)
# Search UIDs > last_processed_uid (all if last_processed_uid is None) and optionally SINCE date.
criteria_parts: list[str] = []
if last_processed_uid is not None:
search_criteria = f"UID {last_processed_uid + 1}:*"
criteria_parts.append(f"UID {last_processed_uid + 1}:*")
else:
search_criteria = "UID 1:*"
_, data = conn.uid("SEARCH", None, search_criteria)
criteria_parts.append("UID 1:*")
if since is not None:
imap_date = _format_imap_date(since)
criteria_parts.append(f"SINCE {imap_date}")
_, data = conn.uid("SEARCH", None, " ".join(criteria_parts))
if not data or not data[0]:
try:
conn.logout()
Expand Down
125 changes: 66 additions & 59 deletions src/fetcher/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str,
db_path = state_cfg.get("db_path", "state.db")
mailbox = imap_cfg.get("mailbox", "INBOX")
delete_after_import = imap_cfg.get("delete_after_import", True)
since_date = imap_cfg.get("since_date")
if since_date == "":
since_date = None
use_label = gmail_cfg.get("use_label") if "use_label" in gmail_cfg else bool((gmail_cfg.get("label") or "").strip())
label_name = (gmail_cfg.get("label") or "ISP Mail").strip() if use_label else None

Expand Down Expand Up @@ -87,6 +90,7 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str,
mailbox=mailbox,
use_ssl=imap_cfg.get("use_ssl", True),
last_processed_uid=last_uid,
since=since_date,
)
except Exception as e:
logger.exception("IMAP fetch failed: %s", e)
Expand All @@ -100,21 +104,6 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str,
inbox_label_id = None
unread_label_id = None

if not dry_run:
try:
service = get_gmail_service(
gmail_cfg["credentials_path"],
gmail_cfg["token_path"],
)
inbox_label_id = get_inbox_label_id(service, USER_ID)
unread_label_id = get_unread_label_id(service, USER_ID)
label_id = _ensure_label(service, label_name) if label_name else None
if label_name and not label_id:
label_id = _ensure_label(service, label_name)
except Exception as e:
logger.exception("Gmail API init failed: %s", e)
return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0}

for msg in messages_iter:
if state.seen_hash(msg.message_hash):
logger.debug("Skipping duplicate (hash): uid=%s", msg.uid)
Expand Down Expand Up @@ -153,6 +142,21 @@ def run_once(config_path: str | None = None, dry_run: bool = False) -> dict[str,
imported += 1
continue

if not service:
try:
service = get_gmail_service(
gmail_cfg["credentials_path"],
gmail_cfg["token_path"],
)
inbox_label_id = get_inbox_label_id(service, USER_ID)
unread_label_id = get_unread_label_id(service, USER_ID)
label_id = _ensure_label(service, label_name) if label_name else None
if label_name and not label_id:
label_id = _ensure_label(service, label_name)
except Exception as e:
logger.exception("Gmail API init failed: %s", e)
return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0}

try:
label_ids = [label_id] if label_id else []
# Fetch/polling only gets NEW mail (UID > last); always import as unread.
Expand Down Expand Up @@ -226,6 +230,9 @@ def run_copy_all(
state_cfg = cfg.get("state", {})
db_path = state_cfg.get("db_path", "state.db")
mailbox = imap_cfg.get("mailbox", "INBOX")
since_date = imap_cfg.get("since_date")
if since_date == "":
since_date = None
use_label = gmail_cfg.get("use_label") if "use_label" in gmail_cfg else bool((gmail_cfg.get("label") or "").strip())
label_name = (gmail_cfg.get("label") or "ISP Mail").strip() if use_label else None

Expand Down Expand Up @@ -254,6 +261,7 @@ def run_copy_all(
mailbox=mailbox,
use_ssl=imap_cfg.get("use_ssl", True),
last_processed_uid=None,
since=since_date,
)
except Exception as e:
logger.exception("IMAP fetch failed: %s", e)
Expand All @@ -267,19 +275,6 @@ def run_copy_all(
inbox_label_id = None
unread_label_id = None

if not dry_run:
try:
service = get_gmail_service(
gmail_cfg["credentials_path"],
gmail_cfg["token_path"],
)
inbox_label_id = get_inbox_label_id(service, USER_ID)
unread_label_id = get_unread_label_id(service, USER_ID)
label_id = _ensure_label(service, label_name) if label_name else None
except Exception as e:
logger.exception("Gmail API init failed: %s", e)
return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0}

for msg in messages_iter:
if state.seen_hash(msg.message_hash):
logger.debug("Copy-all: skipping duplicate (hash): uid=%s", msg.uid)
Expand All @@ -301,37 +296,6 @@ def run_copy_all(
logger.warning("Delete (duplicate) failed for uid=%s: %s", msg.uid, e)
continue

if not dry_run:
message_id_header = _parse_message_id_from_raw(msg.raw)
if message_id_header and gmail_has_message_with_id(
service, USER_ID, message_id_header
):
logger.debug(
"Copy-all: skipping (already in Gmail by Message-ID): uid=%s",
msg.uid,
)
skipped_duplicate += 1
if delete_after_import:
try:
delete_and_expunge(
imap_cfg["host"],
int(imap_cfg.get("port", 993)),
imap_cfg["username"],
imap_cfg["password"],
mailbox,
msg.uid,
imap_cfg.get("use_ssl", True),
)
state.set_last_processed_uid(mailbox, uid_validity, msg.uid)
deleted += 1
except Exception as e:
logger.warning(
"Delete (already in Gmail) failed for uid=%s: %s",
msg.uid,
e,
)
continue

if dry_run:
logger.info(
"[DRY-RUN] Copy-all would import uid=%s (hash=%s)",
Expand All @@ -341,6 +305,49 @@ def run_copy_all(
imported += 1
continue

if not service:
try:
service = get_gmail_service(
gmail_cfg["credentials_path"],
gmail_cfg["token_path"],
)
inbox_label_id = get_inbox_label_id(service, USER_ID)
unread_label_id = get_unread_label_id(service, USER_ID)
label_id = _ensure_label(service, label_name) if label_name else None
except Exception as e:
logger.exception("Gmail API init failed: %s", e)
return {"error": str(e), "imported": 0, "skipped_duplicate": 0, "deleted": 0}

message_id_header = _parse_message_id_from_raw(msg.raw)
if message_id_header and gmail_has_message_with_id(
service, USER_ID, message_id_header
):
logger.debug(
"Copy-all: skipping (already in Gmail by Message-ID): uid=%s",
msg.uid,
)
skipped_duplicate += 1
if delete_after_import:
try:
delete_and_expunge(
imap_cfg["host"],
int(imap_cfg.get("port", 993)),
imap_cfg["username"],
imap_cfg["password"],
mailbox,
msg.uid,
imap_cfg.get("use_ssl", True),
)
state.set_last_processed_uid(mailbox, uid_validity, msg.uid)
deleted += 1
except Exception as e:
logger.warning(
"Delete (already in Gmail) failed for uid=%s: %s",
msg.uid,
e,
)
continue

try:
label_ids = [label_id] if label_id else []
# Copy-all: preserve read/unread from ISP (\Seen).
Expand Down
23 changes: 23 additions & 0 deletions src/fetcher/web_ui.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import os
import threading
import time
import datetime
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -221,6 +222,7 @@ class ImapConfigSafe(BaseModel):
mailbox: str
use_ssl: bool
delete_after_import: bool
since_date: str | None = None


class GmailConfigSafe(BaseModel):
Expand Down Expand Up @@ -432,6 +434,7 @@ def _default_config_response() -> ConfigResponse:
mailbox="INBOX",
use_ssl=True,
delete_after_import=True,
since_date=None,
),
gmail=GmailConfigSafe(
use_label=False,
Expand Down Expand Up @@ -476,6 +479,7 @@ def api_config(request: Request) -> ConfigResponse:
mailbox=imap.get("mailbox", "INBOX"),
use_ssl=imap.get("use_ssl", True),
delete_after_import=imap.get("delete_after_import", True),
since_date=imap.get("since_date"),
),
gmail=GmailConfigSafe(
use_label=gmail.get("use_label") if "use_label" in gmail else bool((gmail.get("label") or "").strip()),
Expand Down Expand Up @@ -503,6 +507,7 @@ class ConfigUpdate(BaseModel):
imap_username: str | None = None
imap_mailbox: str | None = None
imap_use_ssl: bool | None = None
imap_since_date: str | None = None
imap_password: str | None = None # stored in .env, not in config
delete_after_import: bool | None = None
gmail_use_label: bool | None = None
Expand Down Expand Up @@ -535,6 +540,11 @@ def api_setup(request: Request, body: SetupBody) -> dict[str, str]:
path = _get_config_path()
if path.exists():
raise HTTPException(status_code=400, detail="Config already exists")
if body.imap_since_date:
try:
datetime.date.fromisoformat(body.imap_since_date)
except Exception as e:
raise HTTPException(status_code=400, detail="since_date must be YYYY-MM-DD")
try:
_verify_imap_credentials(
host=body.imap_host,
Expand All @@ -554,6 +564,7 @@ def api_setup(request: Request, body: SetupBody) -> dict[str, str]:
"username": body.imap_username,
"password_env": "IMAP_PASSWORD",
"mailbox": body.imap_mailbox,
"since_date": body.imap_since_date,
"use_ssl": body.imap_use_ssl,
"delete_after_import": body.delete_after_import,
},
Expand Down Expand Up @@ -616,6 +627,13 @@ def api_config_update(request: Request, update: ConfigUpdate) -> dict[str, str]:
imap["username"] = update.imap_username
if update.imap_mailbox is not None:
imap["mailbox"] = update.imap_mailbox
if update.imap_since_date is not None:
try:
if update.imap_since_date != "":
datetime.date.fromisoformat(update.imap_since_date)
except Exception:
raise HTTPException(status_code=400, detail="since_date must be YYYY-MM-DD")
cfg.setdefault("imap", {})["since_date"] = update.imap_since_date
if update.imap_use_ssl is not None:
imap["use_ssl"] = update.imap_use_ssl
if update.delete_after_import is not None:
Expand Down Expand Up @@ -773,6 +791,7 @@ def api_status(request: Request) -> dict[str, Any]:
<label>IMAP password <input id="s_imap_password" type="password" placeholder="Stored encrypted in .env"></label>
<label>Mailbox <input id="s_imap_mailbox" type="text" value="INBOX" title="IMAP folder to fetch from. INBOX is the main inbox where new mail arrives."></label>
<p class="hint" style="font-size:0.85rem; color:#666; margin-top:0;">Mailbox is the IMAP folder to fetch from. <strong>INBOX</strong> is the main inbox where new mail arrives at your ISP; leave as INBOX unless you use a different folder.</p>
<label>Only fetch mail newer than <input id="s_imap_since_date" type="date"></label>
<label style="margin-top:1rem;"><input type="checkbox" id="s_delete_after_import" checked> Delete emails from ISP after importing to Gmail</label>
<label><input type="checkbox" id="s_gmail_use_label"> Add a Gmail label to imported mail</label>
<div id="s_gmail_label_row" style="display:none;"><label>Label name <input id="s_gmail_label" type="text" value="ISP Mail" placeholder="e.g. ISP Mail"></label></div>
Expand Down Expand Up @@ -827,6 +846,7 @@ def api_status(request: Request) -> dict[str, Any]:
<label>IMAP password <input id="imap_password" type="password" placeholder="Leave blank to keep current (stored encrypted)"></label>
<label>Mailbox <input id="imap_mailbox" type="text" title="IMAP folder to fetch from. INBOX = main inbox."></label>
<p class="hint" style="font-size:0.85rem; color:#666; margin-top:0;">Mailbox is the IMAP folder to fetch from. <strong>INBOX</strong> = main inbox where new mail arrives.</p>
<label>Only fetch mail newer than <input id="imap_since_date" type="date"></label>
<label style="margin-top:1rem;"><input type="checkbox" id="delete_after_import"> Delete emails from ISP after importing to Gmail</label>
<label><input type="checkbox" id="gmail_use_label"> Add a Gmail label to imported mail</label>
<div id="gmail_label_row" style="display:none;"><label>Label name <input id="gmail_label" type="text" placeholder="e.g. ISP Mail"></label></div>
Expand Down Expand Up @@ -894,6 +914,7 @@ def api_status(request: Request) -> dict[str, Any]:
document.getElementById('imap_use_ssl').checked = c.imap.use_ssl;
document.getElementById('imap_username').value = c.imap.username;
document.getElementById('imap_mailbox').value = c.imap.mailbox;
document.getElementById('imap_since_date').value = c.imap.since_date || '';
document.getElementById('delete_after_import').checked = c.imap.delete_after_import;
document.getElementById('gmail_use_label').checked = c.gmail.use_label;
document.getElementById('gmail_label_row').style.display = c.gmail.use_label ? 'block' : 'none';
Expand Down Expand Up @@ -1059,6 +1080,7 @@ def api_status(request: Request) -> dict[str, Any]:
imap_use_ssl: document.getElementById('imap_use_ssl').checked,
imap_username: document.getElementById('imap_username').value,
imap_mailbox: document.getElementById('imap_mailbox').value,
imap_since_date: document.getElementById('imap_since_date').value || null,
delete_after_import: document.getElementById('delete_after_import').checked,
gmail_use_label: document.getElementById('gmail_use_label').checked,
gmail_label: document.getElementById('gmail_label').value,
Expand Down Expand Up @@ -1093,6 +1115,7 @@ def api_status(request: Request) -> dict[str, Any]:
imap_username: document.getElementById('s_imap_username').value,
imap_password: document.getElementById('s_imap_password').value,
imap_mailbox: document.getElementById('s_imap_mailbox').value,
imap_since_date: document.getElementById('s_imap_since_date').value || null,
delete_after_import: document.getElementById('s_delete_after_import').checked,
gmail_use_label: document.getElementById('s_gmail_use_label').checked,
gmail_label: document.getElementById('s_gmail_label').value
Expand Down