diff --git a/clusters/base/apps/immich/configmap-rsync.yaml b/clusters/base/apps/immich/configmap-rsync.yaml deleted file mode 100644 index 09b1045..0000000 --- a/clusters/base/apps/immich/configmap-rsync.yaml +++ /dev/null @@ -1,252 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: immich-rsync-script - namespace: photo -data: - rsync.py: | - import os - import re - import subprocess - import threading - import time - import json - from datetime import datetime, timezone - - # source mirrors immich's own folder layout 1:1 (library, backups, - # encoded-video, profile, thumbs, upload all sit directly under this). - SRC_ROOT = "/source" - LIBRARY_RECENT_DST = "/disks/library-recent" - LIBRARY_ARCHIVE_DST = "/disks/library-archive" - OTHER_DST = "/data" - OUT_DIR = "/www" - FILELIST_DIR = "/tmp/filelists" - CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp() - - PHASES = ["listing", "library (recent)", "library (archive)", "other folders"] - - state_lock = threading.Lock() - state = { - "status": "starting", - "phase": "", - "phases_done": [], - "percent": 0.0, - "processed": 0, - "total": 0, - "rate": "", - "eta": "", - "bytes_transferred": "", - "recent_files": 0, - "archive_files": 0, - "started_at": None, - "finished_at": None, - "error": None, - } - - # matches lines from `rsync --info=progress2`, e.g.: - # 1,234,567 43% 12.34MB/s 0:00:12 (xfr#12, to-chk=345/400) - # the "(xfr#N, to-chk=X/Y)" suffix only appears on some lines (once a - # file completes), so processed/total only update when it's present. - LINE_RE = re.compile( - r"^\s*([\d,]+)\s+(\d+)%\s+(\S+/s)\s+([\d:]+)" - r"(?:\s+\(xfr#(\d+),\s+(?:to-chk|ir-chk)=(\d+)/(\d+)\))?\s*$" - ) - - - def render_html(s): - progress_html = "" - if s["status"] in ("syncing", "done", "error") and s["phase"]: - progress_html = f""" -

Phase: {s['phase']}

-

Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)

-
-

Transferred: {s['bytes_transferred']} bytes · {s['rate']} · ETA {s['eta']}

- """ - elif s["status"] == "listing": - progress_html = "

Scanning source and splitting library by date...

" - error_html = f"

Error: {s['error']}

" if s["error"] else "" - phases_html = "".join( - f"
  • {'✓' if p in s['phases_done'] else ('→' if p == s['phase'] else '·')} {p}
    • " - for p in PHASES - ) - return f""" - Immich NAS rsync - - -

    Immich NAS rsync

    -

    {SRC_ROOT} → library (split by date) + everything else

    -

    Status: {s['status']}

    - - {progress_html} - {error_html} - - - - - -
    Library files ≥ 2025-07-01{s['recent_files']}
    Library files < 2025-07-01{s['archive_files']}
    Started{s['started_at'] or '-'}
    Finished{s['finished_at'] or '-'}
    - """ - - - def write_output(): - with state_lock: - s = dict(state) - html = render_html(s) - - tmp = os.path.join(OUT_DIR, "index.html.tmp") - with open(tmp, "w") as f: - f.write(html) - os.replace(tmp, os.path.join(OUT_DIR, "index.html")) - - tmp = os.path.join(OUT_DIR, "stats.json.tmp") - with open(tmp, "w") as f: - json.dump(s, f, indent=2) - os.replace(tmp, os.path.join(OUT_DIR, "stats.json")) - - - def split_library_by_date(): - """Walk source/library once, bucket relative file paths by mtime.""" - lib_root = os.path.join(SRC_ROOT, "library") - recent, archive = [], [] - if os.path.isdir(lib_root): - for dirpath, _dirs, files in os.walk(lib_root): - for name in files: - fp = os.path.join(dirpath, name) - try: - mtime = os.stat(fp).st_mtime - except OSError: - continue - # rsync --files-from always wants "/"-separated paths - rel = os.path.relpath(fp, lib_root).replace(os.sep, "/") - (recent if mtime >= CUTOFF else archive).append(rel) - return recent, archive - - - def run_rsync(args, phase_label): - with state_lock: - state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0, - bytes_transferred="", rate="", eta="") - write_output() - - proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - - last_write = 0.0 - tail = [] - for line in proc.stdout: - tail.append(line) - tail[:] = tail[-20:] - - m = LINE_RE.match(line) - if m: - bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups() - with state_lock: - state["bytes_transferred"] = bytes_transferred - state["percent"] = float(percent) - state["rate"] = rate - state["eta"] = eta - if to_chk_total is not None: - total = int(to_chk_total) - left = int(to_chk_left) - state["total"] = total - state["processed"] = total - left - - now = time.time() - if now - last_write > 2: - write_output() - last_write = now - - returncode = proc.wait() - if returncode != 0: - raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}") - - - def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label): - run_rsync( - [ - "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L", - f"--files-from={filelist_path}", src_dir, dst_dir, - ], - phase_label, - ) - - - def run_other_folders(): - run_rsync( - [ - "rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L", - "--exclude=library", SRC_ROOT + "/", OTHER_DST + "/", - ], - "other folders", - ) - - - def main_flow(): - with state_lock: - state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat()) - write_output() - - recent, archive = split_library_by_date() - with state_lock: - state["recent_files"] = len(recent) - state["archive_files"] = len(archive) - state["phases_done"].append("listing") - write_output() - - os.makedirs(FILELIST_DIR, exist_ok=True) - recent_list = os.path.join(FILELIST_DIR, "recent.txt") - archive_list = os.path.join(FILELIST_DIR, "archive.txt") - with open(recent_list, "w") as f: - f.write("\n".join(recent)) - with open(archive_list, "w") as f: - f.write("\n".join(archive)) - - lib_src = os.path.join(SRC_ROOT, "library") + "/" - run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)") - with state_lock: - state["phases_done"].append("library (recent)") - write_output() - - run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)") - with state_lock: - state["phases_done"].append("library (archive)") - write_output() - - run_other_folders() - with state_lock: - state["phases_done"].append("other folders") - write_output() - - - def main(): - os.makedirs(OUT_DIR, exist_ok=True) - write_output() - try: - main_flow() - with state_lock: - state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat()) - except Exception as e: - with state_lock: - state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat()) - write_output() - # stay alive so the pod doesn't restart-loop after a one-shot sync; - # the status page keeps showing the final result. - while True: - time.sleep(3600) - - - if __name__ == "__main__": - main() diff --git a/clusters/base/apps/immich/helmrelease-server.yaml b/clusters/base/apps/immich/helmrelease-server.yaml index 3b9a913..eaf7269 100644 --- a/clusters/base/apps/immich/helmrelease-server.yaml +++ b/clusters/base/apps/immich/helmrelease-server.yaml @@ -121,48 +121,6 @@ spec: memory: 512Mi limits: memory: 4Gi - # one-shot migration copy from the old NAS-hosted docker install - # into this install's own volumes (see "persistence" below). - # rsync is idempotent, so a pod restart mid-copy just resumes - # cleanly; once done it sleeps forever instead of exiting, so the - # container doesn't restart-loop and the status page stays up. - # (starts after all initContainers - fix-permissions and the - # mergerfs sidecar's startupProbe - complete, same as "app") - rsync: - image: - repository: python - tag: 3.13-alpine - command: - - sh - - -c - - apk add --no-cache rsync >/dev/null && python3 /app/rsync.py - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi - rsync-web: - image: - repository: nginx - tag: 1.27-alpine - probes: - # explicit port: this controller has two services (app:2283, - # rsync-web:80) and probe auto-detection picks the - # controller's primary service port rather than this - # container's own - without this it checks the wrong port. - liveness: - enabled: true - port: 80 - readiness: - enabled: true - port: 80 - resources: - requests: - cpu: 10m - memory: 16Mi - limits: - memory: 64Mi service: app: @@ -170,11 +128,6 @@ spec: ports: http: port: 2283 - rsync-web: - controller: server - ports: - http: - port: 80 ingress: app: @@ -193,22 +146,6 @@ spec: - secretName: immich-tls hosts: - photo.aggtaa.com - rsync-web: - className: traefik - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - hosts: - - host: rsync.photo.aggtaa.com - paths: - - path: / - pathType: Prefix - service: - identifier: rsync-web - port: http - tls: - - secretName: immich-rsync-tls - hosts: - - rsync.photo.aggtaa.com persistence: # Immich's "library" folder, mergerfs-merged from two disks: recent @@ -229,8 +166,6 @@ spec: - path: /disks/library-recent fix-permissions: - path: /disks/library-recent - rsync: - - path: /disks/library-recent library-archive: type: persistentVolumeClaim accessMode: ReadWriteOnce @@ -243,8 +178,6 @@ spec: - path: /disks/library-archive fix-permissions: - path: /disks/library-archive - rsync: - - path: /disks/library-archive merged-library: type: emptyDir advancedMounts: @@ -274,30 +207,3 @@ spec: server: app: - path: /data - rsync: - - path: /data - nas-source: - type: persistentVolumeClaim - existingClaim: immich-nas-source - advancedMounts: - server: - rsync: - - path: /source - readOnly: true - rsync-script: - type: configMap - name: immich-rsync-script - advancedMounts: - server: - rsync: - - path: /app/rsync.py - subPath: rsync.py - readOnly: true - rsync-www: - type: emptyDir - advancedMounts: - server: - rsync: - - path: /www - rsync-web: - - path: /usr/share/nginx/html diff --git a/clusters/base/apps/immich/kustomization.yaml b/clusters/base/apps/immich/kustomization.yaml index aad1359..d205c70 100644 --- a/clusters/base/apps/immich/kustomization.yaml +++ b/clusters/base/apps/immich/kustomization.yaml @@ -3,8 +3,6 @@ kind: Kustomization resources: - helmrepository.yaml - secret.enc.yaml - - configmap-rsync.yaml - - pv-pvc-nas-source.yaml - helmrelease-postgres.yaml - helmrelease-redis.yaml - helmrelease-ml.yaml diff --git a/clusters/base/apps/immich/pv-pvc-nas-source.yaml b/clusters/base/apps/immich/pv-pvc-nas-source.yaml deleted file mode 100644 index 959449f..0000000 --- a/clusters/base/apps/immich/pv-pvc-nas-source.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: immich-nas-source-nfs -spec: - capacity: - storage: 1Ti - accessModes: - - ReadOnlyMany - persistentVolumeReclaimPolicy: Retain - storageClassName: "" - mountOptions: - - nfsvers=3 - - nolock - nfs: - server: 10.0.64.80 - # .../uploads specifically, not just .../immich: this aligns 1:1 with - # immich's own folder names (library, backups, encoded-video, profile, - # thumbs, upload) sitting directly under /source, matching what - # rsync.py expects. - path: /mnt/sdb2/immich/uploads - readOnly: true - claimRef: - namespace: photo - name: immich-nas-source ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: immich-nas-source - namespace: photo - annotations: - helm.sh/resource-policy: keep -spec: - accessModes: - - ReadOnlyMany - storageClassName: "" - volumeName: immich-nas-source-nfs - resources: - requests: - storage: 1Ti