253 lines
8.8 KiB
YAML
253 lines
8.8 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: immich-rsync-script
|
|
namespace: photo
|
|
data:
|
|
rsync.py: |
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import json
|
|
from datetime import datetime, timezone
|
|
|
|
# source mirrors immich's own folder layout 1:1 (library, backups,
|
|
# encoded-video, profile, thumbs, upload all sit directly under this).
|
|
SRC_ROOT = "/source"
|
|
LIBRARY_RECENT_DST = "/disks/library-recent"
|
|
LIBRARY_ARCHIVE_DST = "/disks/library-archive"
|
|
OTHER_DST = "/data"
|
|
OUT_DIR = "/www"
|
|
FILELIST_DIR = "/tmp/filelists"
|
|
CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp()
|
|
|
|
PHASES = ["listing", "library (recent)", "library (archive)", "other folders"]
|
|
|
|
state_lock = threading.Lock()
|
|
state = {
|
|
"status": "starting",
|
|
"phase": "",
|
|
"phases_done": [],
|
|
"percent": 0.0,
|
|
"processed": 0,
|
|
"total": 0,
|
|
"rate": "",
|
|
"eta": "",
|
|
"bytes_transferred": "",
|
|
"recent_files": 0,
|
|
"archive_files": 0,
|
|
"started_at": None,
|
|
"finished_at": None,
|
|
"error": None,
|
|
}
|
|
|
|
# matches lines from `rsync --info=progress2`, e.g.:
|
|
# 1,234,567 43% 12.34MB/s 0:00:12 (xfr#12, to-chk=345/400)
|
|
# the "(xfr#N, to-chk=X/Y)" suffix only appears on some lines (once a
|
|
# file completes), so processed/total only update when it's present.
|
|
LINE_RE = re.compile(
|
|
r"^\s*([\d,]+)\s+(\d+)%\s+(\S+/s)\s+([\d:]+)"
|
|
r"(?:\s+\(xfr#(\d+),\s+(?:to-chk|ir-chk)=(\d+)/(\d+)\))?\s*$"
|
|
)
|
|
|
|
|
|
def render_html(s):
|
|
progress_html = ""
|
|
if s["status"] in ("syncing", "done", "error") and s["phase"]:
|
|
progress_html = f"""
|
|
<p>Phase: <b>{s['phase']}</b></p>
|
|
<p>Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)</p>
|
|
<div class="bar"><div class="fill" style="width:{s['percent']:.1f}%"></div></div>
|
|
<p>Transferred: {s['bytes_transferred']} bytes · {s['rate']} · ETA {s['eta']}</p>
|
|
"""
|
|
elif s["status"] == "listing":
|
|
progress_html = "<p>Scanning source and splitting library by date...</p>"
|
|
error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
|
|
phases_html = "".join(
|
|
f"<li>{'✓' if p in s['phases_done'] else ('→' if p == s['phase'] else '·')} {p}</li>"
|
|
for p in PHASES
|
|
)
|
|
return f"""<!doctype html>
|
|
<html><head><meta charset="utf-8"><title>Immich NAS rsync</title>
|
|
<meta http-equiv="refresh" content="5">
|
|
<style>
|
|
body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
|
|
table {{ border-collapse: collapse; }}
|
|
td {{ padding: .25rem 1.5rem .25rem 0; }}
|
|
ul {{ list-style: none; padding: 0; }}
|
|
.bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
|
|
.fill {{ background: #4caf50; height: 100%; }}
|
|
.err {{ color: #f66; }}
|
|
</style></head><body>
|
|
<h1>Immich NAS rsync</h1>
|
|
<p>{SRC_ROOT} → library (split by date) + everything else</p>
|
|
<p>Status: <b>{s['status']}</b></p>
|
|
<ul>{phases_html}</ul>
|
|
{progress_html}
|
|
{error_html}
|
|
<table>
|
|
<tr><td>Library files ≥ 2025-07-01</td><td>{s['recent_files']}</td></tr>
|
|
<tr><td>Library files < 2025-07-01</td><td>{s['archive_files']}</td></tr>
|
|
<tr><td>Started</td><td>{s['started_at'] or '-'}</td></tr>
|
|
<tr><td>Finished</td><td>{s['finished_at'] or '-'}</td></tr>
|
|
</table>
|
|
</body></html>"""
|
|
|
|
|
|
def write_output():
|
|
with state_lock:
|
|
s = dict(state)
|
|
html = render_html(s)
|
|
|
|
tmp = os.path.join(OUT_DIR, "index.html.tmp")
|
|
with open(tmp, "w") as f:
|
|
f.write(html)
|
|
os.replace(tmp, os.path.join(OUT_DIR, "index.html"))
|
|
|
|
tmp = os.path.join(OUT_DIR, "stats.json.tmp")
|
|
with open(tmp, "w") as f:
|
|
json.dump(s, f, indent=2)
|
|
os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
|
|
|
|
|
|
def split_library_by_date():
|
|
"""Walk source/library once, bucket relative file paths by mtime."""
|
|
lib_root = os.path.join(SRC_ROOT, "library")
|
|
recent, archive = [], []
|
|
if os.path.isdir(lib_root):
|
|
for dirpath, _dirs, files in os.walk(lib_root):
|
|
for name in files:
|
|
fp = os.path.join(dirpath, name)
|
|
try:
|
|
mtime = os.stat(fp).st_mtime
|
|
except OSError:
|
|
continue
|
|
# rsync --files-from always wants "/"-separated paths
|
|
rel = os.path.relpath(fp, lib_root).replace(os.sep, "/")
|
|
(recent if mtime >= CUTOFF else archive).append(rel)
|
|
return recent, archive
|
|
|
|
|
|
def run_rsync(args, phase_label):
|
|
with state_lock:
|
|
state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0,
|
|
bytes_transferred="", rate="", eta="")
|
|
write_output()
|
|
|
|
proc = subprocess.Popen(
|
|
args,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
bufsize=1,
|
|
)
|
|
|
|
last_write = 0.0
|
|
tail = []
|
|
for line in proc.stdout:
|
|
tail.append(line)
|
|
tail[:] = tail[-20:]
|
|
|
|
m = LINE_RE.match(line)
|
|
if m:
|
|
bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups()
|
|
with state_lock:
|
|
state["bytes_transferred"] = bytes_transferred
|
|
state["percent"] = float(percent)
|
|
state["rate"] = rate
|
|
state["eta"] = eta
|
|
if to_chk_total is not None:
|
|
total = int(to_chk_total)
|
|
left = int(to_chk_left)
|
|
state["total"] = total
|
|
state["processed"] = total - left
|
|
|
|
now = time.time()
|
|
if now - last_write > 2:
|
|
write_output()
|
|
last_write = now
|
|
|
|
returncode = proc.wait()
|
|
if returncode != 0:
|
|
raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}")
|
|
|
|
|
|
def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label):
|
|
run_rsync(
|
|
[
|
|
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
|
f"--files-from={filelist_path}", src_dir, dst_dir,
|
|
],
|
|
phase_label,
|
|
)
|
|
|
|
|
|
def run_other_folders():
|
|
run_rsync(
|
|
[
|
|
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
|
"--exclude=library", SRC_ROOT + "/", OTHER_DST + "/",
|
|
],
|
|
"other folders",
|
|
)
|
|
|
|
|
|
def main_flow():
|
|
with state_lock:
|
|
state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat())
|
|
write_output()
|
|
|
|
recent, archive = split_library_by_date()
|
|
with state_lock:
|
|
state["recent_files"] = len(recent)
|
|
state["archive_files"] = len(archive)
|
|
state["phases_done"].append("listing")
|
|
write_output()
|
|
|
|
os.makedirs(FILELIST_DIR, exist_ok=True)
|
|
recent_list = os.path.join(FILELIST_DIR, "recent.txt")
|
|
archive_list = os.path.join(FILELIST_DIR, "archive.txt")
|
|
with open(recent_list, "w") as f:
|
|
f.write("\n".join(recent))
|
|
with open(archive_list, "w") as f:
|
|
f.write("\n".join(archive))
|
|
|
|
lib_src = os.path.join(SRC_ROOT, "library") + "/"
|
|
run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)")
|
|
with state_lock:
|
|
state["phases_done"].append("library (recent)")
|
|
write_output()
|
|
|
|
run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)")
|
|
with state_lock:
|
|
state["phases_done"].append("library (archive)")
|
|
write_output()
|
|
|
|
run_other_folders()
|
|
with state_lock:
|
|
state["phases_done"].append("other folders")
|
|
write_output()
|
|
|
|
|
|
def main():
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
write_output()
|
|
try:
|
|
main_flow()
|
|
with state_lock:
|
|
state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat())
|
|
except Exception as e:
|
|
with state_lock:
|
|
state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat())
|
|
write_output()
|
|
# stay alive so the pod doesn't restart-loop after a one-shot sync;
|
|
# the status page keeps showing the final result.
|
|
while True:
|
|
time.sleep(3600)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|