From 66b18ca21b4b126c4efe524ce28696f2f623a4c7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 16:35:39 +0200 Subject: [PATCH 1/7] feat(photos): add album CLI for bulk archive, copy, clear, and move Bulk operations on Immich albums from the command line. Supports archive/unarchive with pattern filtering (glob or alias like 'whatsapp'), copy/clear/move between albums with chunked batching, and album-scoped archive via --from. Uses persistent HTTP connections to avoid TCP connection floods. --- stacklets/photos/cli/album.py | 499 ++++++++++++++++++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 stacklets/photos/cli/album.py diff --git a/stacklets/photos/cli/album.py b/stacklets/photos/cli/album.py new file mode 100644 index 0000000..9d2b764 --- /dev/null +++ b/stacklets/photos/cli/album.py @@ -0,0 +1,499 @@ +""" +stack photos album — bulk operations on your Immich library + +Archive, delete, or inspect assets matching filename patterns directly +from the command line. Useful for cleaning up WhatsApp forwards, messenger +downloads, and other junk that pollutes your timeline. + +Usage: + stack photos album archive --pattern whatsapp # archive WhatsApp images + stack photos album archive --pattern 'IMG-*-WA*' # same thing, explicit glob + stack photos album archive --pattern whatsapp --keep-albums + stack photos album archive --pattern '*.gif' --dry-run # preview first + stack photos album archive --from "Imported" # archive an entire album + stack photos album unarchive --pattern whatsapp # undo — restore to timeline + + stack photos album copy --from "Imported" --to "Family" # copy all assets between albums + stack photos album copy --from "Imported" --to "Family" --pattern '*.jpg' + stack photos album clear --from "Imported" # remove all assets from an album + stack photos album clear --from "Imported" --pattern whatsapp + stack photos album move --from "Imported" --to "Family" # copy + clear in one pass +""" + +HELP = "Bulk archive/unarchive/copy/clear/move assets" + +import argparse +import fnmatch +import http.client +import json +import sys +import time + + +# ── HTTP client ────────────────────────────────────────────────────────────── + +class ImmichClient: + """Persistent HTTP connection to the Immich API.""" + + def __init__(self, host, port, api_key): + self.host = host + self.port = port + self.headers = { + "Content-Type": "application/json", + "x-api-key": api_key, + "Connection": "keep-alive", + } + self._conn = None + + def _connect(self): + if self._conn: + try: + self._conn.close() + except Exception: + pass + self._conn = http.client.HTTPConnection(self.host, self.port, timeout=60) + + def request(self, method, path, body=None, retries=2): + data = json.dumps(body).encode() if body else None + for attempt in range(1 + retries): + try: + if not self._conn: + self._connect() + self._conn.request(method, path, body=data, headers=self.headers) + resp = self._conn.getresponse() + raw = resp.read().decode() + status = resp.status + parsed = json.loads(raw) if raw else {} + return status, parsed + except (http.client.RemoteDisconnected, ConnectionError, OSError): + self._connect() + if attempt < retries: + time.sleep(2) + continue + return 0, {"message": "Connection failed (server not reachable)"} + + def get(self, path): + return self.request("GET", path) + + def put(self, path, body): + return self.request("PUT", path, body) + + def post(self, path, body): + return self.request("POST", path, body) + + def delete(self, path, body=None): + return self.request("DELETE", path, body) + + def close(self): + if self._conn: + try: + self._conn.close() + except Exception: + pass + self._conn = None + + +# ── Immich API wrappers ────────────────────────────────────────────────────── + +def _get_api_key(config): + secrets = config.get("secrets", {}) + return secrets.get("photos__IMPORT_API_KEY") or secrets.get("photos__API_KEY") + + +def _search_term(pattern): + """Extract the longest fixed substring from a glob pattern for server-side filtering.""" + parts = [p for p in pattern.replace("*", "\0").split("\0") if p] + return max(parts, key=len) if parts else None + + +def _search_assets(client, pattern): + """Search assets whose originalFileName matches the glob pattern.""" + term = _search_term(pattern) + assets = [] + page = 1 + while True: + body = {"page": page, "size": 1000} + if term: + body["originalFileName"] = term + status, resp = client.post("/api/search/metadata", body) + if status != 200: + print(f" ! Search failed (HTTP {status}): {resp}", file=sys.stderr) + return None + items = resp.get("assets", {}).get("items", []) + if not items: + break + for a in items: + if fnmatch.fnmatch(a.get("originalFileName", ""), pattern): + assets.append(a) + next_page = resp.get("assets", {}).get("nextPage") + if not next_page: + break + page = int(next_page) + return assets + + +def _chunked(items, size=100): + for i in range(0, len(items), size): + yield items[i:i + size] + + +def _update_assets(client, assets, is_archived): + """Update archive visibility in batches via the bulk endpoint.""" + visibility = "archive" if is_archived else "timeline" + updated = 0 + failed = [] + for chunk in _chunked(assets): + ids = [a["id"] for a in chunk] + status, resp = client.put("/api/assets", { + "ids": ids, + "visibility": visibility, + }) + if status in (200, 204): + updated += len(chunk) + else: + for a in chunk: + status, resp = client.put("/api/assets", { + "ids": [a["id"]], + "visibility": visibility, + }) + if status in (200, 204): + updated += 1 + else: + failed.append(a.get("originalFileName", a["id"])) + print(f" {updated + len(failed)}/{len(assets)}...", file=sys.stderr) + if failed: + print(f" ! {len(failed)} assets owned by a different user (no permission with this key):", + file=sys.stderr) + for name in failed: + print(f" {name}", file=sys.stderr) + print(f" hint: use --api-key with that user's key or an admin key to archive these too", + file=sys.stderr) + return updated + + +def _find_album_overlaps(client, asset_ids): + """Find which albums contain the given assets.""" + asset_id_set = set(asset_ids) + + status, albums = client.get("/api/albums") + if status != 200: + print(f" ! Failed to list albums (HTTP {status})", file=sys.stderr) + return [] + + overlaps = [] + for album in albums: + album_id = album["id"] + status, detail = client.get(f"/api/albums/{album_id}") + if status != 200: + continue + album_assets = detail.get("assets", []) + overlap = [a["id"] for a in album_assets if a["id"] in asset_id_set] + if overlap: + overlaps.append((album_id, album.get("albumName", "?"), overlap)) + + return overlaps + + +def _detach_from_albums(client, overlaps): + """Remove assets from albums using pre-computed overlaps.""" + total_removed = 0 + for album_id, album_name, ids in overlaps: + for chunk in _chunked(ids): + client.delete(f"/api/albums/{album_id}/assets", {"ids": chunk}) + print(f" removed {len(ids)} from \"{album_name}\"", file=sys.stderr) + total_removed += len(ids) + + return total_removed + + +def _resolve_album(client, name): + """Find an album by case-insensitive substring match. Returns (id, name) or None.""" + status, albums = client.get("/api/albums") + if status != 200: + print(f" ! Failed to list albums (HTTP {status})", file=sys.stderr) + return None + needle = name.lower() + matches = [(a["id"], a.get("albumName", "?")) for a in albums + if needle in a.get("albumName", "").lower()] + if not matches: + print(f" ! No album matching '{name}'", file=sys.stderr) + return None + exact = [(aid, aname) for aid, aname in matches if aname.lower() == needle] + if exact: + return exact[0] + if len(matches) > 1: + print(f" ! Ambiguous album name '{name}', matches:", file=sys.stderr) + for _, aname in matches: + print(f" {aname}", file=sys.stderr) + return None + return matches[0] + + +def _get_album_assets(client, album_id, pattern=None): + """Get all assets in an album, optionally filtered by glob pattern.""" + status, detail = client.get(f"/api/albums/{album_id}") + if status != 200: + print(f" ! Failed to read album (HTTP {status})", file=sys.stderr) + return None + assets = detail.get("assets", []) + if pattern: + assets = [a for a in assets if fnmatch.fnmatch(a.get("originalFileName", ""), pattern)] + return assets + + +def _add_to_album(client, album_id, asset_ids): + """Add assets to an album in chunks. Returns count added.""" + added = 0 + for chunk in _chunked(asset_ids): + status, resp = client.put(f"/api/albums/{album_id}/assets", {"ids": chunk}) + if status == 200: + success = [r for r in resp if r.get("success")] + added += len(success) + elif status == 204: + added += len(chunk) + else: + print(f" ! Failed to add {len(chunk)} assets (HTTP {status}): {resp}", file=sys.stderr) + print(f" {added}/{len(asset_ids)}...", file=sys.stderr) + return added + + +def _remove_from_album(client, album_id, asset_ids): + """Remove assets from an album in chunks. Returns count removed.""" + removed = 0 + for chunk in _chunked(asset_ids): + status, _ = client.delete(f"/api/albums/{album_id}/assets", {"ids": chunk}) + if status in (200, 204): + removed += len(chunk) + else: + print(f" ! Failed to remove {len(chunk)} assets (HTTP {status})", file=sys.stderr) + print(f" {removed}/{len(asset_ids)}...", file=sys.stderr) + return removed + + +def _create_album(client, name): + """Create a new album. Returns (id, name) or None.""" + status, resp = client.post("/api/albums", {"albumName": name}) + if status in (200, 201): + return resp["id"], resp.get("albumName", name) + print(f" ! Failed to create album '{name}' (HTTP {status}): {resp}", file=sys.stderr) + return None + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +PATTERN_ALIASES = { + "whatsapp": "IMG-*-WA*", +} + + +def _resolve_pattern(pattern): + """Expand known aliases like 'whatsapp' to their glob pattern.""" + if pattern: + return PATTERN_ALIASES.get(pattern.lower(), pattern) + return pattern + + +def _add_pattern_arg(sp): + sp.add_argument("--pattern", + help="Glob pattern or alias (e.g. 'IMG-*-WA*', 'whatsapp')") + + +def _parse_args(argv): + p = argparse.ArgumentParser(prog="stack photos album", description=HELP) + sub = p.add_subparsers(dest="action", required=True) + + for name in ("archive", "unarchive"): + sp = sub.add_parser(name, help=f"{name.title()} matching assets") + sp.add_argument("--from", dest="from_album", + help="Scope to assets in this album") + _add_pattern_arg(sp) + sp.add_argument("--api-key", + help="Immich API key (default: from secrets.toml)") + sp.add_argument("--dry-run", action="store_true", + help="Show what would be affected, don't change anything") + if name == "archive": + sp.add_argument("--keep-albums", action="store_true", + help="Don't remove archived assets from their albums (default: detach)") + + for name in ("copy", "move"): + sp = sub.add_parser(name, help=f"{name.title()} assets between albums") + sp.add_argument("--from", dest="from_album", required=True, + help="Source album name") + sp.add_argument("--to", dest="to_album", required=True, + help="Target album name (created if it doesn't exist)") + _add_pattern_arg(sp) + sp.add_argument("--api-key", + help="Immich API key (default: from secrets.toml)") + sp.add_argument("--dry-run", action="store_true", + help="Show what would be affected, don't change anything") + + sp = sub.add_parser("clear", help="Remove assets from an album") + sp.add_argument("--from", dest="from_album", required=True, + help="Album name to clear") + _add_pattern_arg(sp) + sp.add_argument("--api-key", + help="Immich API key (default: from secrets.toml)") + sp.add_argument("--dry-run", action="store_true", + help="Show what would be affected, don't change anything") + + opts = p.parse_args(argv) + opts.pattern = _resolve_pattern(getattr(opts, "pattern", None)) + if opts.action in ("archive", "unarchive"): + if not opts.pattern and not getattr(opts, "from_album", None): + p.error(f"{opts.action} requires --pattern or --from") + return opts + + +def _print_sample(assets): + sample = assets[:10] + print(f"\n Sample:", file=sys.stderr) + for a in sample: + print(f" {a.get('originalFileName', '?')}", file=sys.stderr) + if len(assets) > 10: + print(f" ... and {len(assets) - 10} more", file=sys.stderr) + + +def run(args, stacklet, config): + opts = _parse_args(args) + + api_key = opts.api_key or _get_api_key(config) + if not api_key: + return {"error": ( + "No Immich API key found. Create one in the Immich admin UI " + "(Administration > API Keys) and add it to .stack/secrets.toml " + "as photos__IMPORT_API_KEY = \"your-key\"" + )} + + port = stacklet.get("port", 42010) + client = ImmichClient("localhost", port, api_key) + + try: + if opts.action in ("archive", "unarchive"): + return _run_archive(client, opts) + return _run_album(client, opts) + finally: + client.close() + + +def _run_archive(client, opts): + is_archived = opts.action == "archive" + pattern = getattr(opts, "pattern", None) + from_album = getattr(opts, "from_album", None) + + if from_album: + src = _resolve_album(client, from_album) + if not src: + return {"error": f"Album '{from_album}' not found"} + src_id, src_name = src + print(f"\n Album: {src_name}", file=sys.stderr) + matched = _get_album_assets(client, src_id, pattern) + if matched is None: + return {"error": "Failed to read album"} + label = f" matching '{pattern}'" if pattern else "" + if not matched: + print(f" No assets{label} in \"{src_name}\"\n", file=sys.stderr) + return {"ok": True, "matched": 0} + print(f" {len(matched)} assets{label}", file=sys.stderr) + else: + print(f"\n Searching assets matching '{pattern}'...", file=sys.stderr) + matched = _search_assets(client, pattern) + if matched is None: + return {"error": "Failed to search assets"} + if not matched: + print(f" No assets match pattern '{pattern}'\n", file=sys.stderr) + return {"ok": True, "matched": 0} + print(f" {len(matched)} assets match '{pattern}'", file=sys.stderr) + + action = "archive" if is_archived else "unarchive" + _print_sample(matched) + + ids = [a["id"] for a in matched] + detach = is_archived and not getattr(opts, "keep_albums", False) + overlaps = [] + if detach: + print(f"\n Scanning albums...", file=sys.stderr) + overlaps = _find_album_overlaps(client, ids) + if overlaps: + print(f" Will detach from {len(overlaps)} album(s):", file=sys.stderr) + for _, name, ids_in_album in overlaps: + print(f" {name} ({len(ids_in_album)} assets)", file=sys.stderr) + else: + print(f" No album memberships found", file=sys.stderr) + + if opts.dry_run: + print(f"\n Dry run — no changes made.\n", file=sys.stderr) + return {"ok": True, "matched": len(matched), "would_update": len(matched)} + + print(f"\n Updating...", file=sys.stderr) + updated = _update_assets(client, matched, is_archived) + print(f" {action.title()}d {updated} assets.", file=sys.stderr) + + detached = 0 + if overlaps and updated: + print(f" Detaching from albums...", file=sys.stderr) + detached = _detach_from_albums(client, overlaps) + + print(file=sys.stderr) + return {"ok": True, "matched": len(matched), "updated": updated, "detached": detached} + + +def _run_album(client, opts): + action = opts.action + pattern = getattr(opts, "pattern", None) + + src = _resolve_album(client, opts.from_album) + if not src: + return {"error": f"Source album '{opts.from_album}' not found"} + src_id, src_name = src + + print(f"\n Source album: {src_name}", file=sys.stderr) + assets = _get_album_assets(client, src_id, pattern) + if assets is None: + return {"error": "Failed to read source album"} + + label = f" matching '{pattern}'" if pattern else "" + if not assets: + print(f" No assets{label} in \"{src_name}\"\n", file=sys.stderr) + return {"ok": True, "matched": 0} + + print(f" {len(assets)} assets{label}", file=sys.stderr) + _print_sample(assets) + + ids = [a["id"] for a in assets] + dst_id, dst_name = None, None + + if action in ("copy", "move"): + dst = _resolve_album(client, opts.to_album) + if not dst: + print(f" Creating album \"{opts.to_album}\"...", file=sys.stderr) + dst = _create_album(client, opts.to_album) + if not dst: + return {"error": f"Failed to create album '{opts.to_album}'"} + dst_id, dst_name = dst + print(f" Target album: {dst_name}", file=sys.stderr) + + if opts.dry_run: + print(f"\n Dry run — no changes made.\n", file=sys.stderr) + result = {"ok": True, "matched": len(assets)} + if action in ("copy", "move"): + result["would_copy"] = len(assets) + if action in ("clear", "move"): + result["would_clear"] = len(assets) + return result + + result = {"ok": True, "matched": len(assets)} + + if action in ("copy", "move"): + print(f"\n Copying to \"{dst_name}\"...", file=sys.stderr) + added = _add_to_album(client, dst_id, ids) + print(f" Copied {added} assets.", file=sys.stderr) + result["copied"] = added + + if action in ("clear", "move"): + print(f"\n Removing from \"{src_name}\"...", file=sys.stderr) + removed = _remove_from_album(client, src_id, ids) + print(f" Removed {removed} assets.", file=sys.stderr) + result["removed"] = removed + + print(file=sys.stderr) + return result From 18260fcb759d1ef851b7b0875d07277c855faba3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 16:35:50 +0200 Subject: [PATCH 2/7] docs(photos): document import and album CLI commands Add admin guide sections for bulk import (--source, --proceed, --force, --album) and album management (archive, copy, clear, move). Includes WhatsApp cleanup workflow and XNU TCP timestamp overflow runbook. --- docs/admin-guide.md | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 7779a45..09ccf7f 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -306,6 +306,54 @@ Install the Immich app on every phone, point it at `http://:42010`, | Port | `42010` | | Data | `~/famstack-data/photos/library/`, `~/famstack-data/photos/postgres/` | +#### Bulk import + +Import media from a mounted drive, NAS share, or local folder. The importer tracks progress per source path, so imports can be paused, resumed, and interleaved across multiple sources. + +```bash +./stack photos import --source /Volumes/files/Bilder/2020 # scan a folder +./stack photos import --status # show all sources + progress +./stack photos import --proceed 100 # upload next 100 files +./stack photos import --proceed 10 --dry-run # preview, no upload +./stack photos import --proceed 10 --verify # spot-check EXIF first +./stack photos import --proceed 500 --force # skip hash dedup (faster) +./stack photos import --album "Vacation 2023" # upload into a specific album +``` + +Deduplication is two-layered: a local SHA-256 hash ledger skips files already imported in previous runs, and Immich's server-side check catches files uploaded via the mobile app or web UI. The hash step can bottleneck large imports — use `--force` to skip it when you know there are no duplicates (Immich still deduplicates server-side). Switching `--source` starts a new cursor; going back to a previous source picks up where you left off. + +**WhatsApp images.** Phone backups often contain hundreds of WhatsApp forwarded images (`IMG-*-WA*`) that pollute the timeline. After importing, archive them in one pass: + +```bash +./stack photos album archive --pattern whatsapp +``` + +This hides them from the timeline, detaches them from albums, and keeps the files in the library. Use `unarchive` to bring them back. + +#### Album management + +Bulk operations on albums and assets. Archive junk, move assets between albums, or clear an album — all from the command line. + +```bash +./stack photos album archive --pattern whatsapp # archive WhatsApp images +./stack photos album archive --pattern '*.gif' # archive all GIFs +./stack photos album archive --from "Imported" # archive an entire album +./stack photos album archive --from "Imported" --pattern whatsapp # combine both +./stack photos album unarchive --pattern whatsapp # undo — restore to timeline +``` + +Archive detaches assets from albums by default. Use `--keep-albums` to keep album memberships. + +```bash +./stack photos album copy --from "Imported" --to "Family" # copy all assets +./stack photos album clear --from "Imported" # remove all from album +./stack photos album move --from "Imported" --to "2024" # copy + clear in one pass +``` + +Copy, clear, and move accept `--pattern` to filter by filename glob (e.g. `--pattern '*.jpg'`). The alias `whatsapp` expands to `IMG-*-WA*`. Album names are matched case-insensitively; `--to` creates the album if it doesn't exist. + +All commands support `--dry-run` to preview and `--api-key` to use a specific user's key (defaults to the import key from `secrets.toml`). + ### Documents (`docs`) Paperless-ngx archive with OCR and structured classification. Receipts, letters, contracts, tax documents. Search across everything by content. @@ -814,6 +862,40 @@ Run the diagnostic first. It logs in and lists the real folders: - **Wanted mail is missing.** `filter_noise` (default on) drops newsletters and automated mail; set `filter_noise = false` to keep everything. - **Threads render oddly on mobile.** Element X needs **beta features** enabled (Settings, then Labs) for threads. +### TCP connections stuck in TIME_WAIT (macOS) + +After ~50 days of uptime, new TCP connections start failing or the server feels sluggish despite low CPU. `netstat` shows thousands of TIME_WAIT connections that never expire: + +```bash +netstat -an | grep TIME_WAIT | wc -l +``` + +If this returns thousands (10k+), you've hit a [known XNU kernel bug](https://photon.codes/blog/we-found-a-ticking-time-bomb-in-macos-tcp-networking): a 32-bit unsigned integer overflow in `tcp_now` freezes the internal TCP timestamp clock after 49 days 17 hours 2 minutes 47 seconds of continuous uptime. Once frozen, TIME_WAIT connections never expire. + +**Diagnose:** + +```bash +# Check uptime — the bug triggers after ~49.7 days +uptime + +# Confirm TIME_WAIT pile-up +netstat -an | grep TIME_WAIT | wc -l + +# Check boot time precisely +sysctl kern.boottime +``` + +**Fix:** Reboot the Mac. No sysctl, no network restart, no OrbStack restart will clear it — only a full system reboot resets the kernel counter. + +**Prevent:** Reboot the server once a month. A simple cron job as a reminder: + +```bash +# Weekly uptime check — warns at 40 days so you can plan a reboot +(crontab -l 2>/dev/null; echo '0 9 * * 1 uptime_days=$(($(( $(date +\%s) - $(sysctl -n kern.boottime | grep -o "sec = [0-9]*" | grep -o "[0-9]*") )) / 86400)); [ $uptime_days -ge 40 ] && osascript -e "display notification \"Uptime: ${uptime_days} days — reboot soon (XNU TCP bug at 49d)\" with title \"famstack\""') | crontab - +``` + +After reboot, all containers come back automatically (OrbStack starts at login and restarts containers with `restart: unless-stopped`). + ### Something else ```bash From ee7362a38154b1a7198463f37142e73abefbf8d9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 16:36:40 +0200 Subject: [PATCH 3/7] feat(photos): add bulk import CLI with resumable cursor and dedup Scan a mounted source (NAS, external drive, local folder), build a manifest, and upload to Immich in controllable batches. Two-layer deduplication via local SHA-256 ledger and Immich server-side check. Per-source cursor allows pausing, resuming, and interleaving imports. --- stacklets/photos/cli/import.py | 610 +++++++++++++++++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 stacklets/photos/cli/import.py diff --git a/stacklets/photos/cli/import.py b/stacklets/photos/cli/import.py new file mode 100644 index 0000000..386d770 --- /dev/null +++ b/stacklets/photos/cli/import.py @@ -0,0 +1,610 @@ +""" +stack photos import — import media from any mounted source into Immich + +Scans a source path (NAS, external drive, local folder), builds a manifest +of all media files, and uploads them to Immich in controllable batches. +A cursor tracks progress per source path, so imports can be paused and +resumed across sessions — and you can switch between sources freely. + +Two-layer deduplication: + 1. Hash ledger: SHA-256 set from all previous imports (fast local skip) + 2. Immich server-side: catches files already uploaded via phone/web + +State layout: + {data_dir}/photos/import/ + hashes.txt # global — shared across all sources + sources/ + volumes-files-bilder-2020/ # per-source manifest + cursor + manifest.txt + state.json + volumes-files-bilder/ + manifest.txt + state.json + +Usage: + stack photos import --source /Volumes/files/Bilder/2020 # scan subfolder first + stack photos import --status # show all sources + stack photos import --proceed 100 # upload next 100 files + stack photos import --proceed 10 --dry-run # preview, no upload + stack photos import --proceed 10 --verify # spot-check EXIF first + + # later — switch to the full directory + stack photos import --source /Volumes/files/Bilder # new source, own cursor + stack photos import --proceed 1000 # continues from this source + + # go back to the subfolder + stack photos import --source /Volumes/files/Bilder/2020 --status +""" + +HELP = "Import media from a mounted source into Immich" + +import argparse +import hashlib +import json +import os +import random +import re +import subprocess +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path + +MEDIA_EXTENSIONS = { + ".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp", ".gif", ".bmp", + ".tiff", ".tif", ".avif", ".jxl", + ".mp4", ".mov", ".avi", ".mkv", ".wmv", ".m4v", ".3gp", ".mts", + ".dng", ".cr2", ".cr3", ".nef", ".arw", ".orf", ".rw2", ".raf", +} + +IMPORT_DIR = "photos/import" + + +# ── Path helpers ──────────────────────────────────────────────────────────── + +def _import_root(config): + data_dir = Path(config.get("data_dir", config.get("repo_root", "."))) + return data_dir / IMPORT_DIR + + +def _source_slug(source_path): + """Turn a source path into a filesystem-safe directory name.""" + resolved = str(Path(source_path).resolve()) + return re.sub(r"[^a-zA-Z0-9]+", "-", resolved).strip("-").lower() + + +def _source_dir(import_root, source_path): + return import_root / "sources" / _source_slug(source_path) + + +def _ledger_path(import_root): + return import_root / "hashes.txt" + + +# ── State management ──────────────────────────────────────────────────────── + +def _load_state(state_path): + if state_path.exists(): + return json.loads(state_path.read_text()) + return None + + +def _save_state(state_path, state): + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(json.dumps(state, indent=2) + "\n") + + +def _find_active_source(import_root): + """Find the most recently used source directory.""" + sources_dir = import_root / "sources" + if not sources_dir.exists(): + return None + candidates = [] + for d in sources_dir.iterdir(): + if not d.is_dir(): + continue + state_path = d / "state.json" + state = _load_state(state_path) + if state: + manifest = _load_manifest(d / "manifest.txt") + remaining = len(manifest) - state.get("cursor", 0) + if remaining > 0: + ts = state.get("last_used", state.get("created", "")) + candidates.append((ts, d, state)) + if not candidates: + return None + candidates.sort(reverse=True) + return candidates[0][1], candidates[0][2] + + +def _list_sources(import_root): + """List all source directories with their state.""" + sources_dir = import_root / "sources" + if not sources_dir.exists(): + return [] + result = [] + for d in sorted(sources_dir.iterdir()): + if not d.is_dir(): + continue + state = _load_state(d / "state.json") + if state: + manifest = _load_manifest(d / "manifest.txt") + remaining = len(manifest) - state.get("cursor", 0) + result.append((d, state, remaining)) + return result + + +# ── Manifest ──────────────────────────────────────────────────────────────── + +def _scan_media(root): + """Walk source tree, return sorted list of media file paths (relative to root).""" + files = [] + root = Path(root) + for dirpath, _, filenames in os.walk(root): + for f in filenames: + if Path(f).suffix.lower() in MEDIA_EXTENSIONS: + rel = (Path(dirpath) / f).relative_to(root) + files.append(str(rel)) + return sorted(files) + + +def _load_manifest(manifest_path): + if not manifest_path.exists(): + return [] + return [l.strip() for l in manifest_path.read_text().splitlines() if l.strip()] + + +def _save_manifest(manifest_path, files): + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text("\n".join(files) + "\n") + + +# ── Hash ledger ───────────────────────────────────────────────────────────── + +def _load_ledger(path): + if not path.exists(): + return set() + with open(path) as f: + return {line.strip() for line in f if line.strip()} + + +def _append_ledger(path, new_hashes): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a") as f: + for h in new_hashes: + f.write(h + "\n") + + +def _hash_file(path, buf_size=1 << 20): + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(buf_size): + h.update(chunk) + return h.hexdigest() + + +# ── Metadata verification ────────────────────────────────────────────────── + +def _verify_metadata(source_root, rel_paths, count=5): + if not shutil.which("exiftool"): + print(" ! exiftool not installed — install with: brew install exiftool", + file=sys.stderr) + return True + + sample = random.sample(rel_paths, min(count, len(rel_paths))) + print(f"\n Metadata spot-check ({len(sample)} files):\n", file=sys.stderr) + ok = True + for rel in sample: + full = Path(source_root) / rel + result = subprocess.run( + ["exiftool", "-DateTimeOriginal", "-FileType", "-ImageSize", "-s3", str(full)], + capture_output=True, text=True, + ) + fields = result.stdout.strip().splitlines() + has_date = len(fields) >= 1 and fields[0].strip() + icon = "+" if has_date else "!" + date_str = fields[0].strip() if has_date else "no DateTimeOriginal" + name = Path(rel).name + print(f" {icon} {name:40s} {date_str}", file=sys.stderr) + if not has_date: + ok = False + if not ok: + print("\n ! Some files lack DateTimeOriginal — Immich will fall back to file date", + file=sys.stderr) + return ok + + +# ── Upload ────────────────────────────────────────────────────────────────── + +def _get_api_key(config): + secrets = config.get("secrets", {}) + return secrets.get("photos__IMPORT_API_KEY") or secrets.get("photos__API_KEY") + + +def _get_immich_version(config): + env_file = Path(config.get("repo_root", "")) / "stacklets" / "photos" / ".env" + if env_file.exists(): + for line in env_file.read_text().splitlines(): + if line.startswith("IMMICH_VERSION="): + return line.split("=", 1)[1].strip().strip('"') + return "release" + + +def _upload_batch(source_root, rel_paths, immich_version, api_key, album_name): + """Copy files to a local temp dir, upload via sidecar container, clean up. + + Network/SMB mounts can't be bind-mounted into Docker containers on macOS, + so we stage files locally first. + """ + import shutil as _shutil + import tempfile + from shlex import quote + + staging = Path(tempfile.mkdtemp(prefix="immich-import-")) + try: + for rel in rel_paths: + src = Path(source_root) / rel + dst = staging / rel + dst.parent.mkdir(parents=True, exist_ok=True) + _shutil.copy2(str(src), str(dst)) + + staged_count = sum(1 for _ in staging.rglob("*") if _.is_file()) + print(f" staged {staged_count} files to {staging}", file=sys.stderr) + + staging_abs = str(staging) + album_flag = f" --album-name {quote(album_name)}" if album_name else "" + shell_cmd = ( + f"immich login-key http://stack-photos-server:2283 {quote(api_key)}" + f" && immich upload{album_flag} --recursive /import" + ) + cmd = [ + "docker", "run", "--rm", + "--entrypoint", "", + "--network", "stack", + "-v", f"{staging_abs}:/import:ro", + f"ghcr.io/immich-app/immich-server:{immich_version}", + "sh", "-c", shell_cmd, + ] + print(f" docker: {' '.join(cmd[:8])} ...", file=sys.stderr) + result = subprocess.run(cmd, timeout=7200, capture_output=True, text=True) + for line in (result.stdout + result.stderr).splitlines(): + if line.strip(): + print(f" immich: {line.strip()}", file=sys.stderr) + if result.returncode != 0: + print(f" exit code: {result.returncode}", file=sys.stderr) + return result.returncode == 0 + finally: + _shutil.rmtree(staging, ignore_errors=True) + + +def _upload_files(source_root, rel_paths, immich_version, api_key, album_name): + """Upload files in chunks, staging each batch to a temp dir.""" + chunk_size = 200 + print(f"\n Uploading {len(rel_paths)} files to album \"{album_name}\"...\n", + file=sys.stderr) + for i in range(0, len(rel_paths), chunk_size): + chunk = rel_paths[i:i + chunk_size] + if len(rel_paths) > chunk_size: + print(f" chunk {i // chunk_size + 1}: {len(chunk)} files", file=sys.stderr) + if not _upload_batch(source_root, chunk, immich_version, api_key, album_name): + return False + return True + + +# ── CLI argument parsing ──────────────────────────────────────────────────── + +def _parse_args(argv): + p = argparse.ArgumentParser(prog="stack photos import", description=HELP) + p.add_argument("--source", type=Path, + help="Path to mounted source (NAS, external drive). Scans and builds manifest.") + p.add_argument("--proceed", type=int, metavar="N", + help="Process next N files from the active source") + p.add_argument("--status", action="store_true", + help="Show import progress (all sources, or specific with --source)") + p.add_argument("--dry-run", action="store_true", + help="Show what would be uploaded, don't actually upload") + p.add_argument("--verify", action="store_true", + help="Spot-check EXIF metadata before uploading") + p.add_argument("--album", default="Imported", + help="Immich album name for uploaded files (default: Imported)") + p.add_argument("--force", action="store_true", + help="Skip hash ledger check — upload even if previously imported") + p.add_argument("--reset-cursor", action="store_true", + help="Reset cursor to 0 for the active source") + return p.parse_args(argv) + + +# ── Commands ──────────────────────────────────────────────────────────────── + +def _cmd_source(source_path, iroot): + """Scan source, build manifest, initialize or resume state.""" + if not source_path.is_dir(): + return {"error": f"Source path does not exist: {source_path}"} + + sdir = _source_dir(iroot, source_path) + + # Check for existing state — resume or rescan + existing_state = _load_state(sdir / "state.json") + if existing_state: + manifest = _load_manifest(sdir / "manifest.txt") + cursor = existing_state.get("cursor", 0) + remaining = len(manifest) - cursor + if remaining > 0: + print(f" Source already scanned — {remaining} files remaining at cursor {cursor}.", + file=sys.stderr) + print(f" Use --proceed N to continue uploading.\n", file=sys.stderr) + existing_state["last_used"] = datetime.now(timezone.utc).isoformat() + _save_state(sdir / "state.json", existing_state) + return {"ok": True, "resumed": True, "remaining": remaining} + + print(f" Scanning {source_path}...", file=sys.stderr) + files = _scan_media(source_path) + + if not files: + return {"error": f"No media files found in {source_path}"} + + _save_manifest(sdir / "manifest.txt", files) + state = { + "source": str(source_path.resolve()), + "total_scanned": len(files), + "total_queued": len(files), + "cursor": 0, + "uploaded": 0, + "dupes_skipped": 0, + "created": datetime.now(timezone.utc).isoformat(), + "last_used": datetime.now(timezone.utc).isoformat(), + } + _save_state(sdir / "state.json", state) + + print(f"\n Manifest built:", file=sys.stderr) + print(f" Source: {source_path}", file=sys.stderr) + print(f" Media files found: {len(files)}", file=sys.stderr) + print(f"\n Run 'stack photos import --proceed N' to start uploading.\n", + file=sys.stderr) + + return {"ok": True, "queued": len(files), "already_imported": already_known} + + +def _cmd_status(iroot, specific_source=None): + """Show import progress for all sources or a specific one.""" + if specific_source: + sdir = _source_dir(iroot, specific_source) + state = _load_state(sdir / "state.json") + if not state: + print(f" No import found for {specific_source}", file=sys.stderr) + print(f" Run: stack photos import --source {specific_source}\n", + file=sys.stderr) + return {"ok": True, "active": False} + manifest = _load_manifest(sdir / "manifest.txt") + _print_source_status(state, manifest) + return {"ok": True} + + sources = _list_sources(iroot) + if not sources: + print(" No imports in progress.", file=sys.stderr) + print(" Start with: stack photos import --source /path/to/media\n", + file=sys.stderr) + return {"ok": True, "active": False} + + ledger = _load_ledger(_ledger_path(iroot)) + print(f"\n Import overview ({len(ledger)} total hashes in ledger):\n", + file=sys.stderr) + + for sdir, state, remaining in sources: + source = state.get("source", "?") + uploaded = state.get("uploaded", 0) + queued = state.get("total_queued", 0) + status = "done" if remaining == 0 else f"{remaining} remaining" + marker = " *" if remaining > 0 else "" + print(f" {source}", file=sys.stderr) + print(f" {uploaded}/{queued} uploaded, {status}{marker}", file=sys.stderr) + + print(file=sys.stderr) + return {"ok": True, "sources": len(sources)} + + +def _print_source_status(state, manifest): + cursor = state.get("cursor", 0) + remaining = len(manifest) - cursor + + print(f"\n Import status:", file=sys.stderr) + print(f" Source: {state.get('source', '?')}", file=sys.stderr) + print(f" Total scanned: {state.get('total_scanned', '?')}", file=sys.stderr) + print(f" Already imported: {state.get('already_imported', 0)}", file=sys.stderr) + print(f" Queued: {state.get('total_queued', len(manifest))}", file=sys.stderr) + print(f" Uploaded: {state.get('uploaded', 0)}", file=sys.stderr) + print(f" Dupes skipped: {state.get('dupes_skipped', 0)}", file=sys.stderr) + print(f" Cursor: {cursor}/{len(manifest)}", file=sys.stderr) + print(f" Remaining: {remaining}", file=sys.stderr) + + if remaining > 0 and manifest: + next_files = manifest[cursor:cursor + 3] + print(f"\n Next up:", file=sys.stderr) + for f in next_files: + print(f" {f}", file=sys.stderr) + if remaining > 3: + print(f" ... and {remaining - 3} more", file=sys.stderr) + elif remaining == 0: + print(f"\n Import complete!", file=sys.stderr) + + print(file=sys.stderr) + + +def _cmd_proceed(n, iroot, config, source_path=None, dry_run=False, verify=False, album="Imported", force=False): + """Process next N files from a source.""" + # Resolve which source to use + if source_path: + sdir = _source_dir(iroot, source_path) + else: + found = _find_active_source(iroot) + if not found: + return {"error": "No active import. Run --source first."} + sdir, _ = found + + state_path = sdir / "state.json" + manifest_path = sdir / "manifest.txt" + ledger_p = _ledger_path(iroot) + + state = _load_state(state_path) + if not state: + return {"error": "No import in progress for this source. Run --source first."} + + manifest = _load_manifest(manifest_path) + source = state["source"] + cursor = state.get("cursor", 0) + + if cursor >= len(manifest): + print(" All files have been processed.\n", file=sys.stderr) + return {"ok": True, "message": "complete"} + + if not Path(source).is_dir(): + return {"error": f"Source not accessible: {source}\nIs the drive mounted?"} + + batch = manifest[cursor:cursor + n] + print(f"\n Processing {len(batch)} files (#{cursor + 1} – #{cursor + len(batch)}" + f" of {len(manifest)}):\n", file=sys.stderr) + + # Hash and dedup against ledger + ledger = set() if force else _load_ledger(ledger_p) + to_upload = [] + to_upload_hashes = [] + dupes = 0 + + for i, rel in enumerate(batch): + full = Path(source) / rel + if not full.exists(): + print(f" ! missing: {rel}", file=sys.stderr) + continue + + h = _hash_file(full) + if h in ledger: + dupes += 1 + else: + to_upload.append(rel) + to_upload_hashes.append(h) + + if (i + 1) % 100 == 0: + print(f" hashed {i + 1}/{len(batch)}...", file=sys.stderr) + + skip_msg = " (--force, hash check skipped)" if force else "" + print(f" {len(to_upload)} new, {dupes} duplicates skipped{skip_msg}", file=sys.stderr) + + if verify and to_upload: + _verify_metadata(source, to_upload) + + if dry_run: + print(f"\n Dry run — next {min(10, len(to_upload))} files:\n", file=sys.stderr) + for rel in to_upload[:10]: + print(f" {rel}", file=sys.stderr) + if len(to_upload) > 10: + print(f" ... and {len(to_upload) - 10} more", file=sys.stderr) + print(f"\n Would upload {len(to_upload)} files. No changes made.\n", + file=sys.stderr) + return {"ok": True, "would_upload": len(to_upload), "dupes": dupes} + + if not config["is_healthy"](): + return {"error": "Photos is not running — start it with 'stack up photos'"} + + api_key = _get_api_key(config) + if not api_key: + return {"error": ( + "No Immich API key found. Create one in the Immich admin UI " + "(Administration > API Keys) and add it to .stack/secrets.toml " + "as photos__IMPORT_API_KEY = \"your-key\"" + )} + + if not to_upload: + state["cursor"] = cursor + len(batch) + state["dupes_skipped"] = state.get("dupes_skipped", 0) + dupes + state["last_used"] = datetime.now(timezone.utc).isoformat() + _save_state(state_path, state) + print(f" All {dupes} files already imported. Cursor advanced.\n", + file=sys.stderr) + return {"ok": True, "uploaded": 0, "dupes": dupes} + + immich_version = _get_immich_version(config) + success = _upload_files(source, to_upload, immich_version, api_key, album) + + if success: + _append_ledger(ledger_p, to_upload_hashes) + state["cursor"] = cursor + len(batch) + state["uploaded"] = state.get("uploaded", 0) + len(to_upload) + state["dupes_skipped"] = state.get("dupes_skipped", 0) + dupes + state["last_used"] = datetime.now(timezone.utc).isoformat() + _save_state(state_path, state) + + remaining = len(manifest) - state["cursor"] + print(f" Uploaded {len(to_upload)} files. {remaining} remaining.\n", + file=sys.stderr) + return {"ok": True, "uploaded": len(to_upload), "dupes": dupes, + "remaining": remaining} + else: + return {"error": "Upload failed. Cursor not advanced — safe to retry."} + + +def _cmd_reset_cursor(iroot, source_path=None): + """Reset cursor to 0 for a source.""" + if source_path: + sdir = _source_dir(iroot, source_path) + else: + found = _find_active_source(iroot) + if not found: + return {"error": "No active import. Use --source to specify which one."} + sdir, _ = found + + state_path = sdir / "state.json" + state = _load_state(state_path) + if not state: + return {"error": "No import state found for this source."} + + old_cursor = state.get("cursor", 0) + state["cursor"] = 0 + state["uploaded"] = 0 + state["dupes_skipped"] = 0 + state["last_used"] = datetime.now(timezone.utc).isoformat() + _save_state(state_path, state) + + manifest = _load_manifest(sdir / "manifest.txt") + print(f" Cursor reset: {old_cursor} → 0 ({len(manifest)} files queued)", file=sys.stderr) + print(f" Source: {state.get('source', '?')}\n", file=sys.stderr) + return {"ok": True} + + +# ── Entry point ───────────────────────────────────────────────────────────── + +def run(args, stacklet, config): + opts = _parse_args(args) + iroot = _import_root(config) + + if opts.reset_cursor: + return _cmd_reset_cursor(iroot, source_path=opts.source) + + if opts.source and opts.status: + return _cmd_status(iroot, specific_source=opts.source) + + if opts.source and opts.proceed: + _cmd_source(opts.source, iroot) + return _cmd_proceed(opts.proceed, iroot, config, source_path=opts.source, + dry_run=opts.dry_run, verify=opts.verify, + album=opts.album, force=opts.force) + + if opts.source: + return _cmd_source(opts.source, iroot) + + if opts.status: + return _cmd_status(iroot) + + if opts.proceed: + return _cmd_proceed(opts.proceed, iroot, config, + dry_run=opts.dry_run, verify=opts.verify, + album=opts.album, force=opts.force) + + print(" Usage:", file=sys.stderr) + print(" stack photos import --source /Volumes/NAS/photos # scan source", file=sys.stderr) + print(" stack photos import --status # show progress", file=sys.stderr) + print(" stack photos import --proceed 100 # upload next 100", file=sys.stderr) + print(" stack photos import --proceed 10 --force # skip hash check", file=sys.stderr) + print(" stack photos import --reset-cursor # reset to start", file=sys.stderr) + print(" stack photos import --source /path --status # status for one source\n", + file=sys.stderr) + return {"error": "No action specified. Use --source, --status, --proceed, --reset-cursor."} From 7518a164c70e66624b692bd9d26be297642d4499 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 16:56:29 +0200 Subject: [PATCH 4/7] fix(photos): resolve ruff lint errors in import and album CLI Remove extraneous f-string prefixes, rename ambiguous variable 'l' to 'line', and drop undefined 'already_known' reference. --- stacklets/photos/cli/album.py | 16 ++++++++-------- stacklets/photos/cli/import.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/stacklets/photos/cli/album.py b/stacklets/photos/cli/album.py index 9d2b764..0bc8b4a 100644 --- a/stacklets/photos/cli/album.py +++ b/stacklets/photos/cli/album.py @@ -166,7 +166,7 @@ def _update_assets(client, assets, is_archived): file=sys.stderr) for name in failed: print(f" {name}", file=sys.stderr) - print(f" hint: use --api-key with that user's key or an admin key to archive these too", + print(" hint: use --api-key with that user's key or an admin key to archive these too", file=sys.stderr) return updated @@ -346,7 +346,7 @@ def _parse_args(argv): def _print_sample(assets): sample = assets[:10] - print(f"\n Sample:", file=sys.stderr) + print("\n Sample:", file=sys.stderr) for a in sample: print(f" {a.get('originalFileName', '?')}", file=sys.stderr) if len(assets) > 10: @@ -411,26 +411,26 @@ def _run_archive(client, opts): detach = is_archived and not getattr(opts, "keep_albums", False) overlaps = [] if detach: - print(f"\n Scanning albums...", file=sys.stderr) + print("\n Scanning albums...", file=sys.stderr) overlaps = _find_album_overlaps(client, ids) if overlaps: print(f" Will detach from {len(overlaps)} album(s):", file=sys.stderr) for _, name, ids_in_album in overlaps: print(f" {name} ({len(ids_in_album)} assets)", file=sys.stderr) else: - print(f" No album memberships found", file=sys.stderr) + print(" No album memberships found", file=sys.stderr) if opts.dry_run: - print(f"\n Dry run — no changes made.\n", file=sys.stderr) + print("\n Dry run — no changes made.\n", file=sys.stderr) return {"ok": True, "matched": len(matched), "would_update": len(matched)} - print(f"\n Updating...", file=sys.stderr) + print("\n Updating...", file=sys.stderr) updated = _update_assets(client, matched, is_archived) print(f" {action.title()}d {updated} assets.", file=sys.stderr) detached = 0 if overlaps and updated: - print(f" Detaching from albums...", file=sys.stderr) + print(" Detaching from albums...", file=sys.stderr) detached = _detach_from_albums(client, overlaps) print(file=sys.stderr) @@ -473,7 +473,7 @@ def _run_album(client, opts): print(f" Target album: {dst_name}", file=sys.stderr) if opts.dry_run: - print(f"\n Dry run — no changes made.\n", file=sys.stderr) + print("\n Dry run — no changes made.\n", file=sys.stderr) result = {"ok": True, "matched": len(assets)} if action in ("copy", "move"): result["would_copy"] = len(assets) diff --git a/stacklets/photos/cli/import.py b/stacklets/photos/cli/import.py index 386d770..a9b6e4e 100644 --- a/stacklets/photos/cli/import.py +++ b/stacklets/photos/cli/import.py @@ -151,7 +151,7 @@ def _scan_media(root): def _load_manifest(manifest_path): if not manifest_path.exists(): return [] - return [l.strip() for l in manifest_path.read_text().splitlines() if l.strip()] + return [line.strip() for line in manifest_path.read_text().splitlines() if line.strip()] def _save_manifest(manifest_path, files): @@ -332,7 +332,7 @@ def _cmd_source(source_path, iroot): if remaining > 0: print(f" Source already scanned — {remaining} files remaining at cursor {cursor}.", file=sys.stderr) - print(f" Use --proceed N to continue uploading.\n", file=sys.stderr) + print(" Use --proceed N to continue uploading.\n", file=sys.stderr) existing_state["last_used"] = datetime.now(timezone.utc).isoformat() _save_state(sdir / "state.json", existing_state) return {"ok": True, "resumed": True, "remaining": remaining} @@ -356,13 +356,13 @@ def _cmd_source(source_path, iroot): } _save_state(sdir / "state.json", state) - print(f"\n Manifest built:", file=sys.stderr) + print("\n Manifest built:", file=sys.stderr) print(f" Source: {source_path}", file=sys.stderr) print(f" Media files found: {len(files)}", file=sys.stderr) - print(f"\n Run 'stack photos import --proceed N' to start uploading.\n", + print("\n Run 'stack photos import --proceed N' to start uploading.\n", file=sys.stderr) - return {"ok": True, "queued": len(files), "already_imported": already_known} + return {"ok": True, "queued": len(files)} def _cmd_status(iroot, specific_source=None): @@ -407,7 +407,7 @@ def _print_source_status(state, manifest): cursor = state.get("cursor", 0) remaining = len(manifest) - cursor - print(f"\n Import status:", file=sys.stderr) + print("\n Import status:", file=sys.stderr) print(f" Source: {state.get('source', '?')}", file=sys.stderr) print(f" Total scanned: {state.get('total_scanned', '?')}", file=sys.stderr) print(f" Already imported: {state.get('already_imported', 0)}", file=sys.stderr) @@ -419,13 +419,13 @@ def _print_source_status(state, manifest): if remaining > 0 and manifest: next_files = manifest[cursor:cursor + 3] - print(f"\n Next up:", file=sys.stderr) + print("\n Next up:", file=sys.stderr) for f in next_files: print(f" {f}", file=sys.stderr) if remaining > 3: print(f" ... and {remaining - 3} more", file=sys.stderr) elif remaining == 0: - print(f"\n Import complete!", file=sys.stderr) + print("\n Import complete!", file=sys.stderr) print(file=sys.stderr) From 95e8a96fe45eaeadbc23fa4c65dbca7fae2fc71c Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 17:35:15 +0200 Subject: [PATCH 5/7] feat(photos): add --album-per-year to auto-bucket imports by EXIF date Extract year from EXIF DateTimeOriginal via batch exiftool, fall back to filename patterns (20180415_..., IMG-20180415-WA...). Files without a detectable year go into "Unknown Year". Creates one Immich album per year automatically during upload. --- docs/admin-guide.md | 3 + stacklets/photos/cli/import.py | 114 +++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 13 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 09ccf7f..f1afcc0 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -318,10 +318,13 @@ Import media from a mounted drive, NAS share, or local folder. The importer trac ./stack photos import --proceed 10 --verify # spot-check EXIF first ./stack photos import --proceed 500 --force # skip hash dedup (faster) ./stack photos import --album "Vacation 2023" # upload into a specific album +./stack photos import --proceed 500 --album-per-year # auto-bucket by EXIF year ``` Deduplication is two-layered: a local SHA-256 hash ledger skips files already imported in previous runs, and Immich's server-side check catches files uploaded via the mobile app or web UI. The hash step can bottleneck large imports — use `--force` to skip it when you know there are no duplicates (Immich still deduplicates server-side). Switching `--source` starts a new cursor; going back to a previous source picks up where you left off. +**Auto-sort by year.** For unsorted photo dumps, `--album-per-year` reads the EXIF date from each file, groups by year, and creates one album per year automatically. Falls back to filename patterns (`20180415_...`) when EXIF is missing. Files without a detectable year go into "Unknown Year". + **WhatsApp images.** Phone backups often contain hundreds of WhatsApp forwarded images (`IMG-*-WA*`) that pollute the timeline. After importing, archive them in one pass: ```bash diff --git a/stacklets/photos/cli/import.py b/stacklets/photos/cli/import.py index a9b6e4e..e007fc4 100644 --- a/stacklets/photos/cli/import.py +++ b/stacklets/photos/cli/import.py @@ -27,6 +27,7 @@ stack photos import --proceed 100 # upload next 100 files stack photos import --proceed 10 --dry-run # preview, no upload stack photos import --proceed 10 --verify # spot-check EXIF first + stack photos import --proceed 500 --album-per-year # auto-bucket by year # later — switch to the full directory stack photos import --source /Volumes/files/Bilder # new source, own cursor @@ -214,6 +215,51 @@ def _verify_metadata(source_root, rel_paths, count=5): return ok +# ── Year extraction ───────────────────────────────────────────────────────── + +_YEAR_FROM_NAME = re.compile(r'(?:^|[_\-])(\d{4})(?:\d{4}|[_\-])') + + +def _year_from_filename(name): + """Try to extract a four-digit year (1990–2039) from a filename.""" + m = _YEAR_FROM_NAME.search(name) + if m: + y = int(m.group(1)) + if 1990 <= y <= 2039: + return str(y) + return None + + +def _extract_years(source_root, rel_paths): + """Return a dict mapping rel_path -> year string. + + Uses exiftool in batch mode for speed. Falls back to filename patterns + for files without EXIF dates. + """ + years = {} + + if shutil.which("exiftool"): + full_paths = [str(Path(source_root) / rel) for rel in rel_paths] + result = subprocess.run( + ["exiftool", "-DateTimeOriginal", "-s3", "-f"] + full_paths, + capture_output=True, text=True, timeout=300, + ) + lines = result.stdout.strip().splitlines() + for rel, line in zip(rel_paths, lines): + val = line.strip() + if val and val != "-" and len(val) >= 4: + y = val[:4] + if y.isdigit() and 1990 <= int(y) <= 2039: + years[rel] = y + continue + years[rel] = _year_from_filename(Path(rel).name) + else: + for rel in rel_paths: + years[rel] = _year_from_filename(Path(rel).name) + + return years + + # ── Upload ────────────────────────────────────────────────────────────────── def _get_api_key(config): @@ -305,8 +351,11 @@ def _parse_args(argv): help="Show what would be uploaded, don't actually upload") p.add_argument("--verify", action="store_true", help="Spot-check EXIF metadata before uploading") - p.add_argument("--album", default="Imported", - help="Immich album name for uploaded files (default: Imported)") + album_group = p.add_mutually_exclusive_group() + album_group.add_argument("--album", default=None, + help="Immich album name for uploaded files (default: Imported)") + album_group.add_argument("--album-per-year", action="store_true", + help="Auto-create albums by year from EXIF date (falls back to filename)") p.add_argument("--force", action="store_true", help="Skip hash ledger check — upload even if previously imported") p.add_argument("--reset-cursor", action="store_true", @@ -430,7 +479,8 @@ def _print_source_status(state, manifest): print(file=sys.stderr) -def _cmd_proceed(n, iroot, config, source_path=None, dry_run=False, verify=False, album="Imported", force=False): +def _cmd_proceed(n, iroot, config, source_path=None, dry_run=False, verify=False, + album="Imported", force=False, album_per_year=False): """Process next N files from a source.""" # Resolve which source to use if source_path: @@ -492,14 +542,37 @@ def _cmd_proceed(n, iroot, config, source_path=None, dry_run=False, verify=False if verify and to_upload: _verify_metadata(source, to_upload) + # Group by year if requested + year_groups = None + if album_per_year and to_upload: + print("\n Extracting years...", file=sys.stderr) + years = _extract_years(source, to_upload) + year_groups = {} + unknown = [] + for rel in to_upload: + y = years.get(rel) + if y: + year_groups.setdefault(y, []).append(rel) + else: + unknown.append(rel) + if unknown: + year_groups["Unknown Year"] = unknown + for y in sorted(year_groups): + label = y if y != "Unknown Year" else "Unknown Year (no EXIF date or year in filename)" + print(f" {label}: {len(year_groups[y])} files", file=sys.stderr) + if dry_run: - print(f"\n Dry run — next {min(10, len(to_upload))} files:\n", file=sys.stderr) - for rel in to_upload[:10]: - print(f" {rel}", file=sys.stderr) - if len(to_upload) > 10: - print(f" ... and {len(to_upload) - 10} more", file=sys.stderr) - print(f"\n Would upload {len(to_upload)} files. No changes made.\n", - file=sys.stderr) + if year_groups: + print(f"\n Would upload {len(to_upload)} files into {len(year_groups)} album(s). " + "No changes made.\n", file=sys.stderr) + else: + print(f"\n Dry run — next {min(10, len(to_upload))} files:\n", file=sys.stderr) + for rel in to_upload[:10]: + print(f" {rel}", file=sys.stderr) + if len(to_upload) > 10: + print(f" ... and {len(to_upload) - 10} more", file=sys.stderr) + print(f"\n Would upload {len(to_upload)} files. No changes made.\n", + file=sys.stderr) return {"ok": True, "would_upload": len(to_upload), "dupes": dupes} if not config["is_healthy"](): @@ -523,7 +596,18 @@ def _cmd_proceed(n, iroot, config, source_path=None, dry_run=False, verify=False return {"ok": True, "uploaded": 0, "dupes": dupes} immich_version = _get_immich_version(config) - success = _upload_files(source, to_upload, immich_version, api_key, album) + + if year_groups: + success = True + for y in sorted(year_groups): + album_name = y + files = year_groups[y] + print(f"\n Album \"{album_name}\" ({len(files)} files):", file=sys.stderr) + if not _upload_files(source, files, immich_version, api_key, album_name): + success = False + break + else: + success = _upload_files(source, to_upload, immich_version, api_key, album) if success: _append_ledger(ledger_p, to_upload_hashes) @@ -582,11 +666,14 @@ def run(args, stacklet, config): if opts.source and opts.status: return _cmd_status(iroot, specific_source=opts.source) + album = opts.album or ("Imported" if not opts.album_per_year else None) + if opts.source and opts.proceed: _cmd_source(opts.source, iroot) return _cmd_proceed(opts.proceed, iroot, config, source_path=opts.source, dry_run=opts.dry_run, verify=opts.verify, - album=opts.album, force=opts.force) + album=album, force=opts.force, + album_per_year=opts.album_per_year) if opts.source: return _cmd_source(opts.source, iroot) @@ -597,7 +684,8 @@ def run(args, stacklet, config): if opts.proceed: return _cmd_proceed(opts.proceed, iroot, config, dry_run=opts.dry_run, verify=opts.verify, - album=opts.album, force=opts.force) + album=album, force=opts.force, + album_per_year=opts.album_per_year) print(" Usage:", file=sys.stderr) print(" stack photos import --source /Volumes/NAS/photos # scan source", file=sys.stderr) From ee754fca1ffb826c6cfe5b5d6ccda0c855f0fb18 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 17:46:18 +0200 Subject: [PATCH 6/7] fix(photos): parse exiftool batch output correctly for year extraction Exiftool outputs header lines (======== /path) between files. The old code zipped raw output with file paths, misaligning dates and producing a 50/50 split between detected and unknown years. Filter headers and summary lines before matching. Adds unit tests with real JPEG files containing embedded EXIF dates to catch regressions in exiftool output format. --- stacklets/photos/cli/import.py | 24 ++- tests/stacklets/test_photos_import.py | 240 ++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 tests/stacklets/test_photos_import.py diff --git a/stacklets/photos/cli/import.py b/stacklets/photos/cli/import.py index e007fc4..8621a4a 100644 --- a/stacklets/photos/cli/import.py +++ b/stacklets/photos/cli/import.py @@ -230,6 +230,25 @@ def _year_from_filename(name): return None +def _parse_exiftool_output(stdout): + """Parse exiftool batch output into a list of values, one per file. + + Exiftool outputs per file: + ======== /path/to/file.jpg + 2018:12:31 20:59:06 + With -f, missing tags show as '-'. A summary line at the end is ignored. + """ + values = [] + for line in stdout.splitlines(): + line = line.strip() + if line.startswith("========"): + continue + if line and line[0].isdigit() and "image file" in line: + continue + values.append(line) + return values + + def _extract_years(source_root, rel_paths): """Return a dict mapping rel_path -> year string. @@ -244,9 +263,8 @@ def _extract_years(source_root, rel_paths): ["exiftool", "-DateTimeOriginal", "-s3", "-f"] + full_paths, capture_output=True, text=True, timeout=300, ) - lines = result.stdout.strip().splitlines() - for rel, line in zip(rel_paths, lines): - val = line.strip() + values = _parse_exiftool_output(result.stdout) + for rel, val in zip(rel_paths, values): if val and val != "-" and len(val) >= 4: y = val[:4] if y.isdigit() and 1990 <= int(y) <= 2039: diff --git a/tests/stacklets/test_photos_import.py b/tests/stacklets/test_photos_import.py new file mode 100644 index 0000000..8fc9c68 --- /dev/null +++ b/tests/stacklets/test_photos_import.py @@ -0,0 +1,240 @@ +"""Unit tests for photos import year extraction. + +Tests year extraction from EXIF metadata and filename patterns. The +exiftool tests create real JPEG files with embedded EXIF dates to catch +regressions in exiftool output format parsing — no mocked output. +""" + +import os +import struct +import sys +import tempfile +from pathlib import Path + +import pytest + +sys.path.insert( + 0, str(Path(__file__).resolve().parent.parent.parent + / "stacklets" / "photos" / "cli"), +) + +import importlib.util +_spec = importlib.util.spec_from_file_location( + "album_import", + Path(__file__).resolve().parent.parent.parent + / "stacklets" / "photos" / "cli" / "import.py", +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +_year_from_filename = _mod._year_from_filename +_parse_exiftool_output = _mod._parse_exiftool_output +_extract_years = _mod._extract_years + +HAS_EXIFTOOL = bool(__import__("shutil").which("exiftool")) + + +def _make_jpeg(path, exif_date=None): + """Create a minimal valid JPEG file, optionally with a DateTimeOriginal tag. + + Builds the EXIF structure by hand — no PIL dependency needed. + """ + buf = bytearray() + buf += b'\xff\xd8' # SOI + + if exif_date: + date_bytes = exif_date.encode("ascii") + b'\x00' # null-terminated + + # TIFF header (little-endian) + tiff = bytearray() + tiff += b'II' # little-endian + tiff += struct.pack('H', len(exif_payload) + 2) + buf += exif_payload + + # Minimal SOS + EOI + buf += b'\xff\xda\x00\x02' + buf += b'\xff\xd9' + + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(bytes(buf)) + + +# ── Year from filename ───────────────────────────────────────────────── + + +class TestYearFromFilename: + """Extract year from common photo filename patterns.""" + + def test_android_camera(self): + assert _year_from_filename("20181231_205906.jpg") == "2018" + + def test_android_img_prefix(self): + assert _year_from_filename("IMG_20190101_120000.jpg") == "2019" + + def test_whatsapp(self): + assert _year_from_filename("IMG-20200412-WA0032.jpg") == "2020" + + def test_samsung_style(self): + assert _year_from_filename("20170815_134522_HDR.jpg") == "2017" + + def test_google_takeout(self): + assert _year_from_filename("2016-08-14.jpg") == "2016" + + def test_no_year_generic(self): + assert _year_from_filename("random.jpg") is None + + def test_no_year_dsc(self): + assert _year_from_filename("DSC0001.jpg") is None + + def test_no_year_short_number(self): + assert _year_from_filename("IMG_001.jpg") is None + + def test_year_too_old(self): + assert _year_from_filename("18901231_120000.jpg") is None + + def test_year_too_future(self): + assert _year_from_filename("20401231_120000.jpg") is None + + def test_boundary_1990(self): + assert _year_from_filename("19900101_000000.jpg") == "1990" + + def test_boundary_2039(self): + assert _year_from_filename("20390101_000000.jpg") == "2039" + + +# ── Exiftool output parsing (string-level) ───────────────────────────── + + +class TestParseExiftoolOutput: + """Parse raw exiftool -DateTimeOriginal -s3 -f batch output.""" + + def test_single_file_with_date(self): + stdout = ( + "======== /tmp/a.jpg\n" + "2018:12:31 20:59:06\n" + " 1 image files read\n" + ) + assert _parse_exiftool_output(stdout) == ["2018:12:31 20:59:06"] + + def test_single_file_no_date(self): + stdout = ( + "======== /tmp/a.jpg\n" + "-\n" + " 1 image files read\n" + ) + assert _parse_exiftool_output(stdout) == ["-"] + + def test_multiple_files(self): + stdout = ( + "======== /tmp/a.jpg\n" + "2018:12:31 20:59:06\n" + "======== /tmp/b.jpg\n" + "2019:06:15 10:30:00\n" + "======== /tmp/c.jpg\n" + "-\n" + " 3 image files read\n" + ) + assert _parse_exiftool_output(stdout) == [ + "2018:12:31 20:59:06", + "2019:06:15 10:30:00", + "-", + ] + + def test_count_matches_files(self): + files = ["a.jpg", "b.jpg", "c.jpg", "d.jpg", "e.jpg"] + lines = [] + for f in files: + lines.append(f"======== /tmp/{f}") + lines.append("2020:01:01 00:00:00") + lines.append(f" {len(files)} image files read") + values = _parse_exiftool_output("\n".join(lines) + "\n") + assert len(values) == len(files) + + +# ── End-to-end year extraction with real EXIF files ──────────────────── + + +@pytest.mark.skipif(not HAS_EXIFTOOL, reason="exiftool not installed") +class TestExtractYearsWithExiftool: + """Create real JPEG files with EXIF dates and verify _extract_years + reads them correctly via exiftool.""" + + @pytest.fixture + def source_dir(self, tmp_path): + return tmp_path / "photos" + + def test_single_file_with_exif(self, source_dir): + _make_jpeg(source_dir / "photo.jpg", "2018:12:31 20:59:06") + years = _extract_years(str(source_dir), ["photo.jpg"]) + assert years["photo.jpg"] == "2018" + + def test_multiple_years(self, source_dir): + _make_jpeg(source_dir / "a.jpg", "2015:03:20 14:22:01") + _make_jpeg(source_dir / "b.jpg", "2019:08:10 09:00:00") + _make_jpeg(source_dir / "c.jpg", "2022:01:01 00:00:00") + years = _extract_years(str(source_dir), ["a.jpg", "b.jpg", "c.jpg"]) + assert years == {"a.jpg": "2015", "b.jpg": "2019", "c.jpg": "2022"} + + def test_no_exif_falls_back_to_filename(self, source_dir): + _make_jpeg(source_dir / "20180415_120000.jpg") + years = _extract_years(str(source_dir), ["20180415_120000.jpg"]) + assert years["20180415_120000.jpg"] == "2018" + + def test_no_exif_no_year_in_name(self, source_dir): + _make_jpeg(source_dir / "random.jpg") + years = _extract_years(str(source_dir), ["random.jpg"]) + assert years["random.jpg"] is None + + def test_mixed_exif_and_no_exif(self, source_dir): + _make_jpeg(source_dir / "with_exif.jpg", "2017:06:15 10:30:00") + _make_jpeg(source_dir / "20190101_120000.jpg") + _make_jpeg(source_dir / "mystery.jpg") + rel_paths = ["with_exif.jpg", "20190101_120000.jpg", "mystery.jpg"] + years = _extract_years(str(source_dir), rel_paths) + assert years["with_exif.jpg"] == "2017" + assert years["20190101_120000.jpg"] == "2019" + assert years["mystery.jpg"] is None + + def test_count_matches_input(self, source_dir): + """Every input file must have a year entry (even if None).""" + for i in range(10): + date = f"20{15 + i % 5}:01:01 00:00:00" if i % 2 == 0 else None + _make_jpeg(source_dir / f"img_{i:03d}.jpg", date) + rel_paths = [f"img_{i:03d}.jpg" for i in range(10)] + years = _extract_years(str(source_dir), rel_paths) + assert len(years) == 10 + assert set(years.keys()) == set(rel_paths) + + def test_subdirectory_paths(self, source_dir): + _make_jpeg(source_dir / "2018" / "vacation" / "photo.jpg", + "2018:07:20 15:00:00") + years = _extract_years(str(source_dir), + [str(Path("2018") / "vacation" / "photo.jpg")]) + assert years[str(Path("2018") / "vacation" / "photo.jpg")] == "2018" From 168010162140e0f139e28c5e8def6bbec2d44779 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 18:14:04 +0200 Subject: [PATCH 7/7] fix: remove unused imports in photos import tests --- tests/stacklets/test_photos_import.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/stacklets/test_photos_import.py b/tests/stacklets/test_photos_import.py index 8fc9c68..3438790 100644 --- a/tests/stacklets/test_photos_import.py +++ b/tests/stacklets/test_photos_import.py @@ -5,10 +5,8 @@ regressions in exiftool output format parsing — no mocked output. """ -import os import struct import sys -import tempfile from pathlib import Path import pytest @@ -52,7 +50,6 @@ def _make_jpeg(path, exif_date=None): tiff += struct.pack('