diff --git a/clusters/base/apps/immich/configmap-rsync.yaml b/clusters/base/apps/immich/configmap-rsync.yaml index d8ceba7..09b1045 100644 --- a/clusters/base/apps/immich/configmap-rsync.yaml +++ b/clusters/base/apps/immich/configmap-rsync.yaml @@ -13,23 +13,34 @@ data: import json from datetime import datetime, timezone - SRC = "/source/" - DST = "/data/" + # source mirrors immich's own folder layout 1:1 (library, backups, + # encoded-video, profile, thumbs, upload all sit directly under this). + SRC_ROOT = "/source" + LIBRARY_RECENT_DST = "/disks/library-recent" + LIBRARY_ARCHIVE_DST = "/disks/library-archive" + OTHER_DST = "/data" OUT_DIR = "/www" + FILELIST_DIR = "/tmp/filelists" + CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp() + + PHASES = ["listing", "library (recent)", "library (archive)", "other folders"] state_lock = threading.Lock() state = { "status": "starting", + "phase": "", + "phases_done": [], "percent": 0.0, "processed": 0, "total": 0, "rate": "", "eta": "", "bytes_transferred": "", + "recent_files": 0, + "archive_files": 0, "started_at": None, "finished_at": None, "error": None, - "returncode": None, } # matches lines from `rsync --info=progress2`, e.g.: @@ -44,15 +55,20 @@ data: def render_html(s): progress_html = "" - if s["status"] == "scanning": - progress_html = "

Building file list...

" - elif s["status"] in ("syncing", "done", "error"): + if s["status"] in ("syncing", "done", "error") and s["phase"]: progress_html = f""" +

Phase: {s['phase']}

Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)

Transferred: {s['bytes_transferred']} bytes · {s['rate']} · ETA {s['eta']}

""" + elif s["status"] == "listing": + progress_html = "

Scanning source and splitting library by date...

" error_html = f"

Error: {s['error']}

" if s["error"] else "" + phases_html = "".join( + f"
  • {'✓' if p in s['phases_done'] else ('→' if p == s['phase'] else '·')} {p}
    • " + for p in PHASES + ) return f""" Immich NAS rsync @@ -60,19 +76,22 @@ data: body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }} table {{ border-collapse: collapse; }} td {{ padding: .25rem 1.5rem .25rem 0; }} + ul {{ list-style: none; padding: 0; }} .bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }} .fill {{ background: #4caf50; height: 100%; }} .err {{ color: #f66; }}

    Immich NAS rsync

    -

    {SRC} → {DST}

    +

    {SRC_ROOT} → library (split by date) + everything else

    Status: {s['status']}

    + {progress_html} {error_html} + + -
    Library files ≥ 2025-07-01{s['recent_files']}
    Library files < 2025-07-01{s['archive_files']}
    Started{s['started_at'] or '-'}
    Finished{s['finished_at'] or '-'}
    Exit code{s['returncode'] if s['returncode'] is not None else '-'}
    """ @@ -93,21 +112,32 @@ data: os.replace(tmp, os.path.join(OUT_DIR, "stats.json")) - def run_rsync(): + def split_library_by_date(): + """Walk source/library once, bucket relative file paths by mtime.""" + lib_root = os.path.join(SRC_ROOT, "library") + recent, archive = [], [] + if os.path.isdir(lib_root): + for dirpath, _dirs, files in os.walk(lib_root): + for name in files: + fp = os.path.join(dirpath, name) + try: + mtime = os.stat(fp).st_mtime + except OSError: + continue + # rsync --files-from always wants "/"-separated paths + rel = os.path.relpath(fp, lib_root).replace(os.sep, "/") + (recent if mtime >= CUTOFF else archive).append(rel) + return recent, archive + + + def run_rsync(args, phase_label): with state_lock: - state.update(status="scanning", started_at=datetime.now(timezone.utc).isoformat()) + state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0, + bytes_transferred="", rate="", eta="") write_output() proc = subprocess.Popen( - [ - "rsync", - "-a", - "--info=progress2", - "--no-inc-recursive", - "--outbuf=L", - SRC, - DST, - ], + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -124,7 +154,6 @@ data: if m: bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups() with state_lock: - state["status"] = "syncing" state["bytes_transferred"] = bytes_transferred state["percent"] = float(percent) state["rate"] = rate @@ -141,23 +170,78 @@ data: last_write = now returncode = proc.wait() + if returncode != 0: + raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}") + + def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label): + run_rsync( + [ + "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L", + f"--files-from={filelist_path}", src_dir, dst_dir, + ], + phase_label, + ) + + + def run_other_folders(): + run_rsync( + [ + "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L", + "--exclude=library", SRC_ROOT + "/", OTHER_DST + "/", + ], + "other folders", + ) + + + def main_flow(): with state_lock: - state["returncode"] = returncode - state["finished_at"] = datetime.now(timezone.utc).isoformat() - if returncode == 0: - state["status"] = "done" - state["percent"] = 100.0 - else: - state["status"] = "error" - state["error"] = "".join(tail[-5:]).strip() or f"rsync exited {returncode}" + state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat()) + write_output() + + recent, archive = split_library_by_date() + with state_lock: + state["recent_files"] = len(recent) + state["archive_files"] = len(archive) + state["phases_done"].append("listing") + write_output() + + os.makedirs(FILELIST_DIR, exist_ok=True) + recent_list = os.path.join(FILELIST_DIR, "recent.txt") + archive_list = os.path.join(FILELIST_DIR, "archive.txt") + with open(recent_list, "w") as f: + f.write("\n".join(recent)) + with open(archive_list, "w") as f: + f.write("\n".join(archive)) + + lib_src = os.path.join(SRC_ROOT, "library") + "/" + run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)") + with state_lock: + state["phases_done"].append("library (recent)") + write_output() + + run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)") + with state_lock: + state["phases_done"].append("library (archive)") + write_output() + + run_other_folders() + with state_lock: + state["phases_done"].append("other folders") write_output() def main(): os.makedirs(OUT_DIR, exist_ok=True) write_output() - run_rsync() + try: + main_flow() + with state_lock: + state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat()) + except Exception as e: + with state_lock: + state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat()) + write_output() # stay alive so the pod doesn't restart-loop after a one-shot sync; # the status page keeps showing the final result. while True: diff --git a/clusters/base/apps/immich/helmrelease-server.yaml b/clusters/base/apps/immich/helmrelease-server.yaml index d3a10c5..7a87d2e 100644 --- a/clusters/base/apps/immich/helmrelease-server.yaml +++ b/clusters/base/apps/immich/helmrelease-server.yaml @@ -13,6 +13,66 @@ spec: values: controllers: server: + strategy: Recreate + podDisruptionBudget: + maxUnavailable: "0" + initContainers: + fix-permissions: + image: + repository: hectorm/nfs-ganesha + tag: v10 + command: + - sh + - -c + - chmod -R a+rwX /disks/library-recent /disks/library-archive + # native sidecar (restartPolicy: Always): mounts the two library + # disks and exposes their union at /disks/merged-library via + # FUSE. It must be up and actually mounted *before* the "app" + # container is created, since HostToContainer propagation only + # relays mounts made *after* the receiving container's own mount + # exists - it won't retroactively pick up a FUSE mount that's + # already there. Running it as a restartPolicy:Always init + # container (started, gated on the startupProbe below, before + # regular containers) guarantees that ordering; a regular sidecar + # container racing "app" on every pod start does not. Same + # pattern as clusters/base/apps/nfs-photo. + mergerfs: + dependsOn: + - fix-permissions + restartPolicy: Always + image: + repository: ghcr.io/hotio/mergerfs + tag: release-v2.42.0 + args: + - "-o" + # library-recent listed first: category.create=ff means new + # files created through the merged mount (i.e. new immich + # uploads going forward) land on library-recent by default, + # which is exactly what's wanted for freshly-uploaded photos. + - "allow_other,use_ino,func.create=ff,category.action=all,category.create=ff,category.search=newest" + - "/disks/library-recent:/disks/library-archive" + - "/disks/merged-library" + securityContext: + privileged: true + probes: + startup: + enabled: true + custom: true + spec: + exec: + command: + - sh + - -c + - grep -q fuse.mergerfs /proc/mounts + periodSeconds: 1 + timeoutSeconds: 2 + failureThreshold: 30 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 512Mi containers: app: image: @@ -54,10 +114,12 @@ spec: limits: memory: 4Gi # one-shot migration copy from the old NAS-hosted docker install - # into this install's own library volume (see "library" below). + # into this install's own volumes (see "persistence" below). # rsync is idempotent, so a pod restart mid-copy just resumes # cleanly; once done it sleeps forever instead of exiting, so the # container doesn't restart-loop and the status page stays up. + # (starts after all initContainers - fix-permissions and the + # mergerfs sidecar's startupProbe - complete, same as "app") rsync: image: repository: python @@ -68,10 +130,10 @@ spec: - apk add --no-cache rsync >/dev/null && python3 /app/rsync.py resources: requests: - cpu: 50m - memory: 64Mi - limits: + cpu: 100m memory: 256Mi + limits: + memory: 1Gi rsync-web: image: repository: nginx @@ -141,16 +203,71 @@ spec: - rsync.photo.aggtaa.com persistence: - # brand new, dedicated library - deliberately not reusing the old - # docker-compose install's NAS data, and not nfs-photo either. - library: + # Immich's "library" folder, mergerfs-merged from two disks: recent + # files (per the rsync migration's cutoff) and everything else. + # New uploads going forward land on library-recent (see mergerfs + # category.create above). Not reusing the old single 800Gi library + # PVC/PV - this replaces it (that PVC becomes orphaned, retained, + # and unused; clean it up separately if no longer wanted). + library-recent: type: persistentVolumeClaim accessMode: ReadWriteOnce - size: 800Gi + size: 200Gi storageClass: longhorn-single-replica retain: true - globalMounts: - - path: /data + advancedMounts: + server: + mergerfs: + - path: /disks/library-recent + fix-permissions: + - path: /disks/library-recent + rsync: + - path: /disks/library-recent + library-archive: + type: persistentVolumeClaim + accessMode: ReadWriteOnce + size: 500Gi + storageClass: longhorn-single-replica + retain: true + advancedMounts: + server: + mergerfs: + - path: /disks/library-archive + fix-permissions: + - path: /disks/library-archive + rsync: + - path: /disks/library-archive + merged-library: + type: emptyDir + advancedMounts: + server: + mergerfs: + - path: /disks/merged-library + mountPropagation: Bidirectional + app: + - path: /data/library + mountPropagation: HostToContainer + dev-fuse: + type: hostPath + hostPath: /dev/fuse + advancedMounts: + server: + mergerfs: + - path: /dev/fuse + # everything immich keeps outside "library" (backups, encoded-video, + # profile, thumbs, upload) - plain volume, no mergerfs involved. + other: + type: persistentVolumeClaim + accessMode: ReadWriteOnce + size: 100Gi + storageClass: longhorn-single-replica + retain: true + advancedMounts: + server: + app: + - path: /data + rsync: + - path: /data nas-source: type: persistentVolumeClaim existingClaim: immich-nas-source diff --git a/clusters/base/apps/immich/pv-pvc-nas-source.yaml b/clusters/base/apps/immich/pv-pvc-nas-source.yaml index 8a26b96..959449f 100644 --- a/clusters/base/apps/immich/pv-pvc-nas-source.yaml +++ b/clusters/base/apps/immich/pv-pvc-nas-source.yaml @@ -14,7 +14,11 @@ spec: - nolock nfs: server: 10.0.64.80 - path: /mnt/sdb2/immich + # .../uploads specifically, not just .../immich: this aligns 1:1 with + # immich's own folder names (library, backups, encoded-video, profile, + # thumbs, upload) sitting directly under /source, matching what + # rsync.py expects. + path: /mnt/sdb2/immich/uploads readOnly: true claimRef: namespace: photo