rsync from nas for immich

clusters/base/apps/immich/configmap-rsync.yaml

@@ -0,0 +1,168 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: immich-rsync-script
+  namespace: photo
+data:
+  rsync.py: |
+    import os
+    import re
+    import subprocess
+    import threading
+    import time
+    import json
+    from datetime import datetime, timezone
+
+    SRC = "/source/"
+    DST = "/data/"
+    OUT_DIR = "/www"
+
+    state_lock = threading.Lock()
+    state = {
+        "status": "starting",
+        "percent": 0.0,
+        "processed": 0,
+        "total": 0,
+        "rate": "",
+        "eta": "",
+        "bytes_transferred": "",
+        "started_at": None,
+        "finished_at": None,
+        "error": None,
+        "returncode": None,
+    }
+
+    # matches lines from `rsync --info=progress2`, e.g.:
+    #   1,234,567  43%   12.34MB/s    0:00:12 (xfr#12, to-chk=345/400)
+    # the "(xfr#N, to-chk=X/Y)" suffix only appears on some lines (once a
+    # file completes), so processed/total only update when it's present.
+    LINE_RE = re.compile(
+        r"^\s*([\d,]+)\s+(\d+)%\s+(\S+/s)\s+([\d:]+)"
+        r"(?:\s+\(xfr#(\d+),\s+(?:to-chk|ir-chk)=(\d+)/(\d+)\))?\s*$"
+    )
+
+
+    def render_html(s):
+        progress_html = ""
+        if s["status"] == "scanning":
+            progress_html = "<p>Building file list...</p>"
+        elif s["status"] in ("syncing", "done", "error"):
+            progress_html = f"""
+    <p>Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)</p>
+    <div class="bar"><div class="fill" style="width:{s['percent']:.1f}%"></div></div>
+    <p>Transferred: {s['bytes_transferred']} bytes &middot; {s['rate']} &middot; ETA {s['eta']}</p>
+    """
+        error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
+        return f"""<!doctype html>
+    <html><head><meta charset="utf-8"><title>Immich NAS rsync</title>
+    <meta http-equiv="refresh" content="5">
+    <style>
+    body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
+    table {{ border-collapse: collapse; }}
+    td {{ padding: .25rem 1.5rem .25rem 0; }}
+    .bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
+    .fill {{ background: #4caf50; height: 100%; }}
+    .err {{ color: #f66; }}
+    </style></head><body>
+    <h1>Immich NAS rsync</h1>
+    <p>{SRC} &rarr; {DST}</p>
+    <p>Status: <b>{s['status']}</b></p>
+    {progress_html}
+    {error_html}
+    <table>
+    <tr><td>Started</td><td>{s['started_at'] or '-'}</td></tr>
+    <tr><td>Finished</td><td>{s['finished_at'] or '-'}</td></tr>
+    <tr><td>Exit code</td><td>{s['returncode'] if s['returncode'] is not None else '-'}</td></tr>
+    </table>
+    </body></html>"""
+
+
+    def write_output():
+        with state_lock:
+            s = dict(state)
+        html = render_html(s)
+
+        tmp = os.path.join(OUT_DIR, "index.html.tmp")
+        with open(tmp, "w") as f:
+            f.write(html)
+        os.replace(tmp, os.path.join(OUT_DIR, "index.html"))
+
+        tmp = os.path.join(OUT_DIR, "stats.json.tmp")
+        with open(tmp, "w") as f:
+            json.dump(s, f, indent=2)
+        os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
+
+
+    def run_rsync():
+        with state_lock:
+            state.update(status="scanning", started_at=datetime.now(timezone.utc).isoformat())
+        write_output()
+
+        proc = subprocess.Popen(
+            [
+                "rsync",
+                "-a",
+                "--info=progress2",
+                "--no-inc-recursive",
+                "--outbuf=L",
+                SRC,
+                DST,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+
+        last_write = 0.0
+        tail = []
+        for line in proc.stdout:
+            tail.append(line)
+            tail[:] = tail[-20:]
+
+            m = LINE_RE.match(line)
+            if m:
+                bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups()
+                with state_lock:
+                    state["status"] = "syncing"
+                    state["bytes_transferred"] = bytes_transferred
+                    state["percent"] = float(percent)
+                    state["rate"] = rate
+                    state["eta"] = eta
+                    if to_chk_total is not None:
+                        total = int(to_chk_total)
+                        left = int(to_chk_left)
+                        state["total"] = total
+                        state["processed"] = total - left
+
+            now = time.time()
+            if now - last_write > 2:
+                write_output()
+                last_write = now
+
+        returncode = proc.wait()
+
+        with state_lock:
+            state["returncode"] = returncode
+            state["finished_at"] = datetime.now(timezone.utc).isoformat()
+            if returncode == 0:
+                state["status"] = "done"
+                state["percent"] = 100.0
+            else:
+                state["status"] = "error"
+                state["error"] = "".join(tail[-5:]).strip() or f"rsync exited {returncode}"
+        write_output()
+
+
+    def main():
+        os.makedirs(OUT_DIR, exist_ok=True)
+        write_output()
+        run_rsync()
+        # stay alive so the pod doesn't restart-loop after a one-shot sync;
+        # the status page keeps showing the final result.
+        while True:
+            time.sleep(3600)
+
+
+    if __name__ == "__main__":
+        main()

clusters/base/apps/immich/helmrelease-server.yaml

@@ -53,6 +53,46 @@ spec:
                 memory: 512Mi
               limits:
                 memory: 4Gi
+          # one-shot migration copy from the old NAS-hosted docker install
+          # into this install's own library volume (see "library" below).
+          # rsync is idempotent, so a pod restart mid-copy just resumes
+          # cleanly; once done it sleeps forever instead of exiting, so the
+          # container doesn't restart-loop and the status page stays up.
+          rsync:
+            image:
+              repository: python
+              tag: 3.13-alpine
+            command:
+              - sh
+              - -c
+              - apk add --no-cache rsync >/dev/null && python3 /app/rsync.py
+            resources:
+              requests:
+                cpu: 50m
+                memory: 64Mi
+              limits:
+                memory: 256Mi
+          rsync-web:
+            image:
+              repository: nginx
+              tag: 1.27-alpine
+            probes:
+              # explicit port: this controller has two services (app:2283,
+              # rsync-web:80) and probe auto-detection picks the
+              # controller's primary service port rather than this
+              # container's own - without this it checks the wrong port.
+              liveness:
+                enabled: true
+                port: 80
+              readiness:
+                enabled: true
+                port: 80
+            resources:
+              requests:
+                cpu: 10m
+                memory: 16Mi
+              limits:
+                memory: 64Mi
 
     service:
       app:
@@ -60,6 +100,11 @@ spec:
         ports:
           http:
             port: 2283
+      rsync-web:
+        controller: server
+        ports:
+          http:
+            port: 80
 
     ingress:
       app:
@@ -78,6 +123,22 @@ spec:
           - secretName: immich-tls
             hosts:
               - photo.aggtaa.com
+      rsync-web:
+        className: traefik
+        annotations:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+        hosts:
+          - host: rsync.photo.aggtaa.com
+            paths:
+              - path: /
+                pathType: Prefix
+                service:
+                  identifier: rsync-web
+                  port: http
+        tls:
+          - secretName: immich-rsync-tls
+            hosts:
+              - rsync.photo.aggtaa.com
 
     persistence:
       # brand new, dedicated library - deliberately not reusing the old
@@ -90,3 +151,28 @@ spec:
         retain: true
         globalMounts:
           - path: /data
+      nas-source:
+        type: persistentVolumeClaim
+        existingClaim: immich-nas-source
+        advancedMounts:
+          server:
+            rsync:
+              - path: /source
+                readOnly: true
+      rsync-script:
+        type: configMap
+        name: immich-rsync-script
+        advancedMounts:
+          server:
+            rsync:
+              - path: /app/rsync.py
+                subPath: rsync.py
+                readOnly: true
+      rsync-www:
+        type: emptyDir
+        advancedMounts:
+          server:
+            rsync:
+              - path: /www
+            rsync-web:
+              - path: /usr/share/nginx/html

clusters/base/apps/immich/kustomization.yaml

@@ -3,6 +3,8 @@ kind: Kustomization
 resources:
   - helmrepository.yaml
   - secret.enc.yaml
+  - configmap-rsync.yaml
+  - pv-pvc-nas-source.yaml
   - helmrelease-postgres.yaml
   - helmrelease-redis.yaml
   - helmrelease-ml.yaml

clusters/base/apps/immich/pv-pvc-nas-source.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: immich-nas-source-nfs
+spec:
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadOnlyMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  mountOptions:
+    - nfsvers=3
+    - nolock
+  nfs:
+    server: 10.0.64.80
+    path: /mnt/sdb2/immich
+    readOnly: true
+  claimRef:
+    namespace: photo
+    name: immich-nas-source
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-nas-source
+  namespace: photo
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  accessModes:
+    - ReadOnlyMany
+  storageClassName: ""
+  volumeName: immich-nas-source-nfs
+  resources:
+    requests:
+      storage: 1Ti