immich mergerfs
This commit is contained in:
azykov@mail.ru
@@ -13,23 +13,34 @@ data:
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SRC = "/source/"
|
||||
DST = "/data/"
|
||||
# source mirrors immich's own folder layout 1:1 (library, backups,
|
||||
# encoded-video, profile, thumbs, upload all sit directly under this).
|
||||
SRC_ROOT = "/source"
|
||||
LIBRARY_RECENT_DST = "/disks/library-recent"
|
||||
LIBRARY_ARCHIVE_DST = "/disks/library-archive"
|
||||
OTHER_DST = "/data"
|
||||
OUT_DIR = "/www"
|
||||
FILELIST_DIR = "/tmp/filelists"
|
||||
CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp()
|
||||
|
||||
PHASES = ["listing", "library (recent)", "library (archive)", "other folders"]
|
||||
|
||||
state_lock = threading.Lock()
|
||||
state = {
|
||||
"status": "starting",
|
||||
"phase": "",
|
||||
"phases_done": [],
|
||||
"percent": 0.0,
|
||||
"processed": 0,
|
||||
"total": 0,
|
||||
"rate": "",
|
||||
"eta": "",
|
||||
"bytes_transferred": "",
|
||||
"recent_files": 0,
|
||||
"archive_files": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"error": None,
|
||||
"returncode": None,
|
||||
}
|
||||
|
||||
# matches lines from `rsync --info=progress2`, e.g.:
|
||||
@@ -44,15 +55,20 @@ data:
|
||||
|
||||
def render_html(s):
|
||||
progress_html = ""
|
||||
if s["status"] == "scanning":
|
||||
progress_html = "<p>Building file list...</p>"
|
||||
elif s["status"] in ("syncing", "done", "error"):
|
||||
if s["status"] in ("syncing", "done", "error") and s["phase"]:
|
||||
progress_html = f"""
|
||||
<p>Phase: <b>{s['phase']}</b></p>
|
||||
<p>Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)</p>
|
||||
<div class="bar"><div class="fill" style="width:{s['percent']:.1f}%"></div></div>
|
||||
<p>Transferred: {s['bytes_transferred']} bytes · {s['rate']} · ETA {s['eta']}</p>
|
||||
"""
|
||||
elif s["status"] == "listing":
|
||||
progress_html = "<p>Scanning source and splitting library by date...</p>"
|
||||
error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
|
||||
phases_html = "".join(
|
||||
f"<li>{'✓' if p in s['phases_done'] else ('→' if p == s['phase'] else '·')} {p}</li>"
|
||||
for p in PHASES
|
||||
)
|
||||
return f"""<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>Immich NAS rsync</title>
|
||||
<meta http-equiv="refresh" content="5">
|
||||
@@ -60,19 +76,22 @@ data:
|
||||
body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
|
||||
table {{ border-collapse: collapse; }}
|
||||
td {{ padding: .25rem 1.5rem .25rem 0; }}
|
||||
ul {{ list-style: none; padding: 0; }}
|
||||
.bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
|
||||
.fill {{ background: #4caf50; height: 100%; }}
|
||||
.err {{ color: #f66; }}
|
||||
</style></head><body>
|
||||
<h1>Immich NAS rsync</h1>
|
||||
<p>{SRC} → {DST}</p>
|
||||
<p>{SRC_ROOT} → library (split by date) + everything else</p>
|
||||
<p>Status: <b>{s['status']}</b></p>
|
||||
<ul>{phases_html}</ul>
|
||||
{progress_html}
|
||||
{error_html}
|
||||
<table>
|
||||
<tr><td>Library files ≥ 2025-07-01</td><td>{s['recent_files']}</td></tr>
|
||||
<tr><td>Library files < 2025-07-01</td><td>{s['archive_files']}</td></tr>
|
||||
<tr><td>Started</td><td>{s['started_at'] or '-'}</td></tr>
|
||||
<tr><td>Finished</td><td>{s['finished_at'] or '-'}</td></tr>
|
||||
<tr><td>Exit code</td><td>{s['returncode'] if s['returncode'] is not None else '-'}</td></tr>
|
||||
</table>
|
||||
</body></html>"""
|
||||
|
||||
@@ -93,21 +112,32 @@ data:
|
||||
os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
|
||||
|
||||
|
||||
def run_rsync():
|
||||
def split_library_by_date():
|
||||
"""Walk source/library once, bucket relative file paths by mtime."""
|
||||
lib_root = os.path.join(SRC_ROOT, "library")
|
||||
recent, archive = [], []
|
||||
if os.path.isdir(lib_root):
|
||||
for dirpath, _dirs, files in os.walk(lib_root):
|
||||
for name in files:
|
||||
fp = os.path.join(dirpath, name)
|
||||
try:
|
||||
mtime = os.stat(fp).st_mtime
|
||||
except OSError:
|
||||
continue
|
||||
# rsync --files-from always wants "/"-separated paths
|
||||
rel = os.path.relpath(fp, lib_root).replace(os.sep, "/")
|
||||
(recent if mtime >= CUTOFF else archive).append(rel)
|
||||
return recent, archive
|
||||
|
||||
|
||||
def run_rsync(args, phase_label):
|
||||
with state_lock:
|
||||
state.update(status="scanning", started_at=datetime.now(timezone.utc).isoformat())
|
||||
state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0,
|
||||
bytes_transferred="", rate="", eta="")
|
||||
write_output()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"rsync",
|
||||
"-a",
|
||||
"--info=progress2",
|
||||
"--no-inc-recursive",
|
||||
"--outbuf=L",
|
||||
SRC,
|
||||
DST,
|
||||
],
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
@@ -124,7 +154,6 @@ data:
|
||||
if m:
|
||||
bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups()
|
||||
with state_lock:
|
||||
state["status"] = "syncing"
|
||||
state["bytes_transferred"] = bytes_transferred
|
||||
state["percent"] = float(percent)
|
||||
state["rate"] = rate
|
||||
@@ -141,23 +170,78 @@ data:
|
||||
last_write = now
|
||||
|
||||
returncode = proc.wait()
|
||||
if returncode != 0:
|
||||
raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}")
|
||||
|
||||
|
||||
def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label):
|
||||
run_rsync(
|
||||
[
|
||||
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
||||
f"--files-from={filelist_path}", src_dir, dst_dir,
|
||||
],
|
||||
phase_label,
|
||||
)
|
||||
|
||||
|
||||
def run_other_folders():
|
||||
run_rsync(
|
||||
[
|
||||
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
||||
"--exclude=library", SRC_ROOT + "/", OTHER_DST + "/",
|
||||
],
|
||||
"other folders",
|
||||
)
|
||||
|
||||
|
||||
def main_flow():
|
||||
with state_lock:
|
||||
state["returncode"] = returncode
|
||||
state["finished_at"] = datetime.now(timezone.utc).isoformat()
|
||||
if returncode == 0:
|
||||
state["status"] = "done"
|
||||
state["percent"] = 100.0
|
||||
else:
|
||||
state["status"] = "error"
|
||||
state["error"] = "".join(tail[-5:]).strip() or f"rsync exited {returncode}"
|
||||
state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat())
|
||||
write_output()
|
||||
|
||||
recent, archive = split_library_by_date()
|
||||
with state_lock:
|
||||
state["recent_files"] = len(recent)
|
||||
state["archive_files"] = len(archive)
|
||||
state["phases_done"].append("listing")
|
||||
write_output()
|
||||
|
||||
os.makedirs(FILELIST_DIR, exist_ok=True)
|
||||
recent_list = os.path.join(FILELIST_DIR, "recent.txt")
|
||||
archive_list = os.path.join(FILELIST_DIR, "archive.txt")
|
||||
with open(recent_list, "w") as f:
|
||||
f.write("\n".join(recent))
|
||||
with open(archive_list, "w") as f:
|
||||
f.write("\n".join(archive))
|
||||
|
||||
lib_src = os.path.join(SRC_ROOT, "library") + "/"
|
||||
run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)")
|
||||
with state_lock:
|
||||
state["phases_done"].append("library (recent)")
|
||||
write_output()
|
||||
|
||||
run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)")
|
||||
with state_lock:
|
||||
state["phases_done"].append("library (archive)")
|
||||
write_output()
|
||||
|
||||
run_other_folders()
|
||||
with state_lock:
|
||||
state["phases_done"].append("other folders")
|
||||
write_output()
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
write_output()
|
||||
run_rsync()
|
||||
try:
|
||||
main_flow()
|
||||
with state_lock:
|
||||
state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat())
|
||||
except Exception as e:
|
||||
with state_lock:
|
||||
state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat())
|
||||
write_output()
|
||||
# stay alive so the pod doesn't restart-loop after a one-shot sync;
|
||||
# the status page keeps showing the final result.
|
||||
while True:
|
||||
|
||||
@@ -13,6 +13,66 @@ spec:
|
||||
values:
|
||||
controllers:
|
||||
server:
|
||||
strategy: Recreate
|
||||
podDisruptionBudget:
|
||||
maxUnavailable: "0"
|
||||
initContainers:
|
||||
fix-permissions:
|
||||
image:
|
||||
repository: hectorm/nfs-ganesha
|
||||
tag: v10
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chmod -R a+rwX /disks/library-recent /disks/library-archive
|
||||
# native sidecar (restartPolicy: Always): mounts the two library
|
||||
# disks and exposes their union at /disks/merged-library via
|
||||
# FUSE. It must be up and actually mounted *before* the "app"
|
||||
# container is created, since HostToContainer propagation only
|
||||
# relays mounts made *after* the receiving container's own mount
|
||||
# exists - it won't retroactively pick up a FUSE mount that's
|
||||
# already there. Running it as a restartPolicy:Always init
|
||||
# container (started, gated on the startupProbe below, before
|
||||
# regular containers) guarantees that ordering; a regular sidecar
|
||||
# container racing "app" on every pod start does not. Same
|
||||
# pattern as clusters/base/apps/nfs-photo.
|
||||
mergerfs:
|
||||
dependsOn:
|
||||
- fix-permissions
|
||||
restartPolicy: Always
|
||||
image:
|
||||
repository: ghcr.io/hotio/mergerfs
|
||||
tag: release-v2.42.0
|
||||
args:
|
||||
- "-o"
|
||||
# library-recent listed first: category.create=ff means new
|
||||
# files created through the merged mount (i.e. new immich
|
||||
# uploads going forward) land on library-recent by default,
|
||||
# which is exactly what's wanted for freshly-uploaded photos.
|
||||
- "allow_other,use_ino,func.create=ff,category.action=all,category.create=ff,category.search=newest"
|
||||
- "/disks/library-recent:/disks/library-archive"
|
||||
- "/disks/merged-library"
|
||||
securityContext:
|
||||
privileged: true
|
||||
probes:
|
||||
startup:
|
||||
enabled: true
|
||||
custom: true
|
||||
spec:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- grep -q fuse.mergerfs /proc/mounts
|
||||
periodSeconds: 1
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
containers:
|
||||
app:
|
||||
image:
|
||||
@@ -54,10 +114,12 @@ spec:
|
||||
limits:
|
||||
memory: 4Gi
|
||||
# one-shot migration copy from the old NAS-hosted docker install
|
||||
# into this install's own library volume (see "library" below).
|
||||
# into this install's own volumes (see "persistence" below).
|
||||
# rsync is idempotent, so a pod restart mid-copy just resumes
|
||||
# cleanly; once done it sleeps forever instead of exiting, so the
|
||||
# container doesn't restart-loop and the status page stays up.
|
||||
# (starts after all initContainers - fix-permissions and the
|
||||
# mergerfs sidecar's startupProbe - complete, same as "app")
|
||||
rsync:
|
||||
image:
|
||||
repository: python
|
||||
@@ -68,10 +130,10 @@ spec:
|
||||
- apk add --no-cache rsync >/dev/null && python3 /app/rsync.py
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
rsync-web:
|
||||
image:
|
||||
repository: nginx
|
||||
@@ -141,16 +203,71 @@ spec:
|
||||
- rsync.photo.aggtaa.com
|
||||
|
||||
persistence:
|
||||
# brand new, dedicated library - deliberately not reusing the old
|
||||
# docker-compose install's NAS data, and not nfs-photo either.
|
||||
library:
|
||||
# Immich's "library" folder, mergerfs-merged from two disks: recent
|
||||
# files (per the rsync migration's cutoff) and everything else.
|
||||
# New uploads going forward land on library-recent (see mergerfs
|
||||
# category.create above). Not reusing the old single 800Gi library
|
||||
# PVC/PV - this replaces it (that PVC becomes orphaned, retained,
|
||||
# and unused; clean it up separately if no longer wanted).
|
||||
library-recent:
|
||||
type: persistentVolumeClaim
|
||||
accessMode: ReadWriteOnce
|
||||
size: 800Gi
|
||||
size: 200Gi
|
||||
storageClass: longhorn-single-replica
|
||||
retain: true
|
||||
globalMounts:
|
||||
- path: /data
|
||||
advancedMounts:
|
||||
server:
|
||||
mergerfs:
|
||||
- path: /disks/library-recent
|
||||
fix-permissions:
|
||||
- path: /disks/library-recent
|
||||
rsync:
|
||||
- path: /disks/library-recent
|
||||
library-archive:
|
||||
type: persistentVolumeClaim
|
||||
accessMode: ReadWriteOnce
|
||||
size: 500Gi
|
||||
storageClass: longhorn-single-replica
|
||||
retain: true
|
||||
advancedMounts:
|
||||
server:
|
||||
mergerfs:
|
||||
- path: /disks/library-archive
|
||||
fix-permissions:
|
||||
- path: /disks/library-archive
|
||||
rsync:
|
||||
- path: /disks/library-archive
|
||||
merged-library:
|
||||
type: emptyDir
|
||||
advancedMounts:
|
||||
server:
|
||||
mergerfs:
|
||||
- path: /disks/merged-library
|
||||
mountPropagation: Bidirectional
|
||||
app:
|
||||
- path: /data/library
|
||||
mountPropagation: HostToContainer
|
||||
dev-fuse:
|
||||
type: hostPath
|
||||
hostPath: /dev/fuse
|
||||
advancedMounts:
|
||||
server:
|
||||
mergerfs:
|
||||
- path: /dev/fuse
|
||||
# everything immich keeps outside "library" (backups, encoded-video,
|
||||
# profile, thumbs, upload) - plain volume, no mergerfs involved.
|
||||
other:
|
||||
type: persistentVolumeClaim
|
||||
accessMode: ReadWriteOnce
|
||||
size: 100Gi
|
||||
storageClass: longhorn-single-replica
|
||||
retain: true
|
||||
advancedMounts:
|
||||
server:
|
||||
app:
|
||||
- path: /data
|
||||
rsync:
|
||||
- path: /data
|
||||
nas-source:
|
||||
type: persistentVolumeClaim
|
||||
existingClaim: immich-nas-source
|
||||
|
||||
@@ -14,7 +14,11 @@ spec:
|
||||
- nolock
|
||||
nfs:
|
||||
server: 10.0.64.80
|
||||
path: /mnt/sdb2/immich
|
||||
# .../uploads specifically, not just .../immich: this aligns 1:1 with
|
||||
# immich's own folder names (library, backups, encoded-video, profile,
|
||||
# thumbs, upload) sitting directly under /source, matching what
|
||||
# rsync.py expects.
|
||||
path: /mnt/sdb2/immich/uploads
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: photo
|
||||
|
||||
Reference in New Issue
Block a user