immich mergerfs

clusters/base/apps/immich/configmap-rsync.yaml

@@ -13,23 +13,34 @@ data:
     import json
     from datetime import datetime, timezone
 
-    SRC = "/source/"
+    # source mirrors immich's own folder layout 1:1 (library, backups,
-    DST = "/data/"
+    # encoded-video, profile, thumbs, upload all sit directly under this).
+    SRC_ROOT = "/source"
+    LIBRARY_RECENT_DST = "/disks/library-recent"
+    LIBRARY_ARCHIVE_DST = "/disks/library-archive"
+    OTHER_DST = "/data"
     OUT_DIR = "/www"
+    FILELIST_DIR = "/tmp/filelists"
+    CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp()
+
+    PHASES = ["listing", "library (recent)", "library (archive)", "other folders"]
 
     state_lock = threading.Lock()
     state = {
         "status": "starting",
+        "phase": "",
+        "phases_done": [],
         "percent": 0.0,
         "processed": 0,
         "total": 0,
         "rate": "",
         "eta": "",
         "bytes_transferred": "",
+        "recent_files": 0,
+        "archive_files": 0,
         "started_at": None,
         "finished_at": None,
         "error": None,
-        "returncode": None,
     }
 
     # matches lines from `rsync --info=progress2`, e.g.:
@@ -44,15 +55,20 @@ data:
 
     def render_html(s):
         progress_html = ""
-        if s["status"] == "scanning":
+        if s["status"] in ("syncing", "done", "error") and s["phase"]:
-            progress_html = "<p>Building file list...</p>"
-        elif s["status"] in ("syncing", "done", "error"):
             progress_html = f"""
+    <p>Phase: <b>{s['phase']}</b></p>
     <p>Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)</p>
     <div class="bar"><div class="fill" style="width:{s['percent']:.1f}%"></div></div>
     <p>Transferred: {s['bytes_transferred']} bytes &middot; {s['rate']} &middot; ETA {s['eta']}</p>
     """
+        elif s["status"] == "listing":
+            progress_html = "<p>Scanning source and splitting library by date...</p>"
         error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
+        phases_html = "".join(
+            f"<li>{'&#10003;' if p in s['phases_done'] else ('&rarr;' if p == s['phase'] else '&middot;')} {p}</li>"
+            for p in PHASES
+        )
         return f"""<!doctype html>
     <html><head><meta charset="utf-8"><title>Immich NAS rsync</title>
     <meta http-equiv="refresh" content="5">
@@ -60,19 +76,22 @@ data:
     body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
     table {{ border-collapse: collapse; }}
     td {{ padding: .25rem 1.5rem .25rem 0; }}
+    ul {{ list-style: none; padding: 0; }}
     .bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
     .fill {{ background: #4caf50; height: 100%; }}
     .err {{ color: #f66; }}
     </style></head><body>
     <h1>Immich NAS rsync</h1>
-    <p>{SRC} &rarr; {DST}</p>
+    <p>{SRC_ROOT} &rarr; library (split by date) + everything else</p>
     <p>Status: <b>{s['status']}</b></p>
+    <ul>{phases_html}</ul>
     {progress_html}
     {error_html}
     <table>
+    <tr><td>Library files &ge; 2025-07-01</td><td>{s['recent_files']}</td></tr>
+    <tr><td>Library files &lt; 2025-07-01</td><td>{s['archive_files']}</td></tr>
     <tr><td>Started</td><td>{s['started_at'] or '-'}</td></tr>
     <tr><td>Finished</td><td>{s['finished_at'] or '-'}</td></tr>
-    <tr><td>Exit code</td><td>{s['returncode'] if s['returncode'] is not None else '-'}</td></tr>
     </table>
     </body></html>"""
 
@@ -93,21 +112,32 @@ data:
         os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
 
 
-    def run_rsync():
+    def split_library_by_date():
+        """Walk source/library once, bucket relative file paths by mtime."""
+        lib_root = os.path.join(SRC_ROOT, "library")
+        recent, archive = [], []
+        if os.path.isdir(lib_root):
+            for dirpath, _dirs, files in os.walk(lib_root):
+                for name in files:
+                    fp = os.path.join(dirpath, name)
+                    try:
+                        mtime = os.stat(fp).st_mtime
+                    except OSError:
+                        continue
+                    # rsync --files-from always wants "/"-separated paths
+                    rel = os.path.relpath(fp, lib_root).replace(os.sep, "/")
+                    (recent if mtime >= CUTOFF else archive).append(rel)
+        return recent, archive
+
+
+    def run_rsync(args, phase_label):
         with state_lock:
-            state.update(status="scanning", started_at=datetime.now(timezone.utc).isoformat())
+            state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0,
+                         bytes_transferred="", rate="", eta="")
         write_output()
 
         proc = subprocess.Popen(
-            [
+            args,
-                "rsync",
-                "-a",
-                "--info=progress2",
-                "--no-inc-recursive",
-                "--outbuf=L",
-                SRC,
-                DST,
-            ],
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             text=True,
@@ -124,7 +154,6 @@ data:
             if m:
                 bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups()
                 with state_lock:
-                    state["status"] = "syncing"
                     state["bytes_transferred"] = bytes_transferred
                     state["percent"] = float(percent)
                     state["rate"] = rate
@@ -141,23 +170,78 @@ data:
                 last_write = now
 
         returncode = proc.wait()
+        if returncode != 0:
+            raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}")
 
+
+    def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label):
+        run_rsync(
+            [
+                "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
+                f"--files-from={filelist_path}", src_dir, dst_dir,
+            ],
+            phase_label,
+        )
+
+
+    def run_other_folders():
+        run_rsync(
+            [
+                "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
+                "--exclude=library", SRC_ROOT + "/", OTHER_DST + "/",
+            ],
+            "other folders",
+        )
+
+
+    def main_flow():
         with state_lock:
-            state["returncode"] = returncode
+            state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat())
-            state["finished_at"] = datetime.now(timezone.utc).isoformat()
+        write_output()
-            if returncode == 0:
+
-                state["status"] = "done"
+        recent, archive = split_library_by_date()
-                state["percent"] = 100.0
+        with state_lock:
-            else:
+            state["recent_files"] = len(recent)
-                state["status"] = "error"
+            state["archive_files"] = len(archive)
-                state["error"] = "".join(tail[-5:]).strip() or f"rsync exited {returncode}"
+            state["phases_done"].append("listing")
+        write_output()
+
+        os.makedirs(FILELIST_DIR, exist_ok=True)
+        recent_list = os.path.join(FILELIST_DIR, "recent.txt")
+        archive_list = os.path.join(FILELIST_DIR, "archive.txt")
+        with open(recent_list, "w") as f:
+            f.write("\n".join(recent))
+        with open(archive_list, "w") as f:
+            f.write("\n".join(archive))
+
+        lib_src = os.path.join(SRC_ROOT, "library") + "/"
+        run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)")
+        with state_lock:
+            state["phases_done"].append("library (recent)")
+        write_output()
+
+        run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)")
+        with state_lock:
+            state["phases_done"].append("library (archive)")
+        write_output()
+
+        run_other_folders()
+        with state_lock:
+            state["phases_done"].append("other folders")
         write_output()
 
 
     def main():
         os.makedirs(OUT_DIR, exist_ok=True)
         write_output()
-        run_rsync()
+        try:
+            main_flow()
+            with state_lock:
+                state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat())
+        except Exception as e:
+            with state_lock:
+                state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat())
+        write_output()
         # stay alive so the pod doesn't restart-loop after a one-shot sync;
         # the status page keeps showing the final result.
         while True:

clusters/base/apps/immich/helmrelease-server.yaml

@@ -13,6 +13,66 @@ spec:
   values:
     controllers:
       server:
+        strategy: Recreate
+        podDisruptionBudget:
+          maxUnavailable: "0"
+        initContainers:
+          fix-permissions:
+            image:
+              repository: hectorm/nfs-ganesha
+              tag: v10
+            command:
+              - sh
+              - -c
+              - chmod -R a+rwX /disks/library-recent /disks/library-archive
+          # native sidecar (restartPolicy: Always): mounts the two library
+          # disks and exposes their union at /disks/merged-library via
+          # FUSE. It must be up and actually mounted *before* the "app"
+          # container is created, since HostToContainer propagation only
+          # relays mounts made *after* the receiving container's own mount
+          # exists - it won't retroactively pick up a FUSE mount that's
+          # already there. Running it as a restartPolicy:Always init
+          # container (started, gated on the startupProbe below, before
+          # regular containers) guarantees that ordering; a regular sidecar
+          # container racing "app" on every pod start does not. Same
+          # pattern as clusters/base/apps/nfs-photo.
+          mergerfs:
+            dependsOn:
+              - fix-permissions
+            restartPolicy: Always
+            image:
+              repository: ghcr.io/hotio/mergerfs
+              tag: release-v2.42.0
+            args:
+              - "-o"
+              # library-recent listed first: category.create=ff means new
+              # files created through the merged mount (i.e. new immich
+              # uploads going forward) land on library-recent by default,
+              # which is exactly what's wanted for freshly-uploaded photos.
+              - "allow_other,use_ino,func.create=ff,category.action=all,category.create=ff,category.search=newest"
+              - "/disks/library-recent:/disks/library-archive"
+              - "/disks/merged-library"
+            securityContext:
+              privileged: true
+            probes:
+              startup:
+                enabled: true
+                custom: true
+                spec:
+                  exec:
+                    command:
+                      - sh
+                      - -c
+                      - grep -q fuse.mergerfs /proc/mounts
+                  periodSeconds: 1
+                  timeoutSeconds: 2
+                  failureThreshold: 30
+            resources:
+              requests:
+                cpu: 50m
+                memory: 64Mi
+              limits:
+                memory: 512Mi
         containers:
           app:
             image:
@@ -54,10 +114,12 @@ spec:
               limits:
                 memory: 4Gi
           # one-shot migration copy from the old NAS-hosted docker install
-          # into this install's own library volume (see "library" below).
+          # into this install's own volumes (see "persistence" below).
           # rsync is idempotent, so a pod restart mid-copy just resumes
           # cleanly; once done it sleeps forever instead of exiting, so the
           # container doesn't restart-loop and the status page stays up.
+          # (starts after all initContainers - fix-permissions and the
+          # mergerfs sidecar's startupProbe - complete, same as "app")
           rsync:
             image:
               repository: python
@@ -68,10 +130,10 @@ spec:
               - apk add --no-cache rsync >/dev/null && python3 /app/rsync.py
             resources:
               requests:
-                cpu: 50m
+                cpu: 100m
-                memory: 64Mi
-              limits:
                 memory: 256Mi
+              limits:
+                memory: 1Gi
           rsync-web:
             image:
               repository: nginx
@@ -141,16 +203,71 @@ spec:
               - rsync.photo.aggtaa.com
 
     persistence:
-      # brand new, dedicated library - deliberately not reusing the old
+      # Immich's "library" folder, mergerfs-merged from two disks: recent
-      # docker-compose install's NAS data, and not nfs-photo either.
+      # files (per the rsync migration's cutoff) and everything else.
-      library:
+      # New uploads going forward land on library-recent (see mergerfs
+      # category.create above). Not reusing the old single 800Gi library
+      # PVC/PV - this replaces it (that PVC becomes orphaned, retained,
+      # and unused; clean it up separately if no longer wanted).
+      library-recent:
         type: persistentVolumeClaim
         accessMode: ReadWriteOnce
-        size: 800Gi
+        size: 200Gi
         storageClass: longhorn-single-replica
         retain: true
-        globalMounts:
+        advancedMounts:
-          - path: /data
+          server:
+            mergerfs:
+              - path: /disks/library-recent
+            fix-permissions:
+              - path: /disks/library-recent
+            rsync:
+              - path: /disks/library-recent
+      library-archive:
+        type: persistentVolumeClaim
+        accessMode: ReadWriteOnce
+        size: 500Gi
+        storageClass: longhorn-single-replica
+        retain: true
+        advancedMounts:
+          server:
+            mergerfs:
+              - path: /disks/library-archive
+            fix-permissions:
+              - path: /disks/library-archive
+            rsync:
+              - path: /disks/library-archive
+      merged-library:
+        type: emptyDir
+        advancedMounts:
+          server:
+            mergerfs:
+              - path: /disks/merged-library
+                mountPropagation: Bidirectional
+            app:
+              - path: /data/library
+                mountPropagation: HostToContainer
+      dev-fuse:
+        type: hostPath
+        hostPath: /dev/fuse
+        advancedMounts:
+          server:
+            mergerfs:
+              - path: /dev/fuse
+      # everything immich keeps outside "library" (backups, encoded-video,
+      # profile, thumbs, upload) - plain volume, no mergerfs involved.
+      other:
+        type: persistentVolumeClaim
+        accessMode: ReadWriteOnce
+        size: 100Gi
+        storageClass: longhorn-single-replica
+        retain: true
+        advancedMounts:
+          server:
+            app:
+              - path: /data
+            rsync:
+              - path: /data
       nas-source:
         type: persistentVolumeClaim
         existingClaim: immich-nas-source

clusters/base/apps/immich/pv-pvc-nas-source.yaml

@@ -14,7 +14,11 @@ spec:
     - nolock
   nfs:
     server: 10.0.64.80
-    path: /mnt/sdb2/immich
+    # .../uploads specifically, not just .../immich: this aligns 1:1 with
+    # immich's own folder names (library, backups, encoded-video, profile,
+    # thumbs, upload) sitting directly under /source, matching what
+    # rsync.py expects.
+    path: /mnt/sdb2/immich/uploads
     readOnly: true
   claimRef:
     namespace: photo