removed rsync & rsync-web from immich-server
This commit is contained in:
azykov@mail.ru
@@ -1,252 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: immich-rsync-script
|
||||
namespace: photo
|
||||
data:
|
||||
rsync.py: |
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# source mirrors immich's own folder layout 1:1 (library, backups,
|
||||
# encoded-video, profile, thumbs, upload all sit directly under this).
|
||||
SRC_ROOT = "/source"
|
||||
LIBRARY_RECENT_DST = "/disks/library-recent"
|
||||
LIBRARY_ARCHIVE_DST = "/disks/library-archive"
|
||||
OTHER_DST = "/data"
|
||||
OUT_DIR = "/www"
|
||||
FILELIST_DIR = "/tmp/filelists"
|
||||
CUTOFF = datetime(2025, 7, 1, tzinfo=timezone.utc).timestamp()
|
||||
|
||||
PHASES = ["listing", "library (recent)", "library (archive)", "other folders"]
|
||||
|
||||
state_lock = threading.Lock()
|
||||
state = {
|
||||
"status": "starting",
|
||||
"phase": "",
|
||||
"phases_done": [],
|
||||
"percent": 0.0,
|
||||
"processed": 0,
|
||||
"total": 0,
|
||||
"rate": "",
|
||||
"eta": "",
|
||||
"bytes_transferred": "",
|
||||
"recent_files": 0,
|
||||
"archive_files": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# matches lines from `rsync --info=progress2`, e.g.:
|
||||
# 1,234,567 43% 12.34MB/s 0:00:12 (xfr#12, to-chk=345/400)
|
||||
# the "(xfr#N, to-chk=X/Y)" suffix only appears on some lines (once a
|
||||
# file completes), so processed/total only update when it's present.
|
||||
LINE_RE = re.compile(
|
||||
r"^\s*([\d,]+)\s+(\d+)%\s+(\S+/s)\s+([\d:]+)"
|
||||
r"(?:\s+\(xfr#(\d+),\s+(?:to-chk|ir-chk)=(\d+)/(\d+)\))?\s*$"
|
||||
)
|
||||
|
||||
|
||||
def render_html(s):
|
||||
progress_html = ""
|
||||
if s["status"] in ("syncing", "done", "error") and s["phase"]:
|
||||
progress_html = f"""
|
||||
<p>Phase: <b>{s['phase']}</b></p>
|
||||
<p>Progress: {s['processed']} / {s['total']} files ({s['percent']:.1f}%)</p>
|
||||
<div class="bar"><div class="fill" style="width:{s['percent']:.1f}%"></div></div>
|
||||
<p>Transferred: {s['bytes_transferred']} bytes · {s['rate']} · ETA {s['eta']}</p>
|
||||
"""
|
||||
elif s["status"] == "listing":
|
||||
progress_html = "<p>Scanning source and splitting library by date...</p>"
|
||||
error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
|
||||
phases_html = "".join(
|
||||
f"<li>{'✓' if p in s['phases_done'] else ('→' if p == s['phase'] else '·')} {p}</li>"
|
||||
for p in PHASES
|
||||
)
|
||||
return f"""<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>Immich NAS rsync</title>
|
||||
<meta http-equiv="refresh" content="5">
|
||||
<style>
|
||||
body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
|
||||
table {{ border-collapse: collapse; }}
|
||||
td {{ padding: .25rem 1.5rem .25rem 0; }}
|
||||
ul {{ list-style: none; padding: 0; }}
|
||||
.bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
|
||||
.fill {{ background: #4caf50; height: 100%; }}
|
||||
.err {{ color: #f66; }}
|
||||
</style></head><body>
|
||||
<h1>Immich NAS rsync</h1>
|
||||
<p>{SRC_ROOT} → library (split by date) + everything else</p>
|
||||
<p>Status: <b>{s['status']}</b></p>
|
||||
<ul>{phases_html}</ul>
|
||||
{progress_html}
|
||||
{error_html}
|
||||
<table>
|
||||
<tr><td>Library files ≥ 2025-07-01</td><td>{s['recent_files']}</td></tr>
|
||||
<tr><td>Library files < 2025-07-01</td><td>{s['archive_files']}</td></tr>
|
||||
<tr><td>Started</td><td>{s['started_at'] or '-'}</td></tr>
|
||||
<tr><td>Finished</td><td>{s['finished_at'] or '-'}</td></tr>
|
||||
</table>
|
||||
</body></html>"""
|
||||
|
||||
|
||||
def write_output():
|
||||
with state_lock:
|
||||
s = dict(state)
|
||||
html = render_html(s)
|
||||
|
||||
tmp = os.path.join(OUT_DIR, "index.html.tmp")
|
||||
with open(tmp, "w") as f:
|
||||
f.write(html)
|
||||
os.replace(tmp, os.path.join(OUT_DIR, "index.html"))
|
||||
|
||||
tmp = os.path.join(OUT_DIR, "stats.json.tmp")
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(s, f, indent=2)
|
||||
os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
|
||||
|
||||
|
||||
def split_library_by_date():
|
||||
"""Walk source/library once, bucket relative file paths by mtime."""
|
||||
lib_root = os.path.join(SRC_ROOT, "library")
|
||||
recent, archive = [], []
|
||||
if os.path.isdir(lib_root):
|
||||
for dirpath, _dirs, files in os.walk(lib_root):
|
||||
for name in files:
|
||||
fp = os.path.join(dirpath, name)
|
||||
try:
|
||||
mtime = os.stat(fp).st_mtime
|
||||
except OSError:
|
||||
continue
|
||||
# rsync --files-from always wants "/"-separated paths
|
||||
rel = os.path.relpath(fp, lib_root).replace(os.sep, "/")
|
||||
(recent if mtime >= CUTOFF else archive).append(rel)
|
||||
return recent, archive
|
||||
|
||||
|
||||
def run_rsync(args, phase_label):
|
||||
with state_lock:
|
||||
state.update(status="syncing", phase=phase_label, processed=0, total=0, percent=0.0,
|
||||
bytes_transferred="", rate="", eta="")
|
||||
write_output()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
last_write = 0.0
|
||||
tail = []
|
||||
for line in proc.stdout:
|
||||
tail.append(line)
|
||||
tail[:] = tail[-20:]
|
||||
|
||||
m = LINE_RE.match(line)
|
||||
if m:
|
||||
bytes_transferred, percent, rate, eta, _xfr, to_chk_left, to_chk_total = m.groups()
|
||||
with state_lock:
|
||||
state["bytes_transferred"] = bytes_transferred
|
||||
state["percent"] = float(percent)
|
||||
state["rate"] = rate
|
||||
state["eta"] = eta
|
||||
if to_chk_total is not None:
|
||||
total = int(to_chk_total)
|
||||
left = int(to_chk_left)
|
||||
state["total"] = total
|
||||
state["processed"] = total - left
|
||||
|
||||
now = time.time()
|
||||
if now - last_write > 2:
|
||||
write_output()
|
||||
last_write = now
|
||||
|
||||
returncode = proc.wait()
|
||||
if returncode != 0:
|
||||
raise RuntimeError("".join(tail[-5:]).strip() or f"rsync exited {returncode}")
|
||||
|
||||
|
||||
def run_filelist_rsync(src_dir, dst_dir, filelist_path, phase_label):
|
||||
run_rsync(
|
||||
[
|
||||
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
||||
f"--files-from={filelist_path}", src_dir, dst_dir,
|
||||
],
|
||||
phase_label,
|
||||
)
|
||||
|
||||
|
||||
def run_other_folders():
|
||||
run_rsync(
|
||||
[
|
||||
"rsync", "-a", "--info=progress2", "--no-inc-recursive", "--outbuf=L",
|
||||
"--exclude=library", SRC_ROOT + "/", OTHER_DST + "/",
|
||||
],
|
||||
"other folders",
|
||||
)
|
||||
|
||||
|
||||
def main_flow():
|
||||
with state_lock:
|
||||
state.update(status="listing", phase="listing", started_at=datetime.now(timezone.utc).isoformat())
|
||||
write_output()
|
||||
|
||||
recent, archive = split_library_by_date()
|
||||
with state_lock:
|
||||
state["recent_files"] = len(recent)
|
||||
state["archive_files"] = len(archive)
|
||||
state["phases_done"].append("listing")
|
||||
write_output()
|
||||
|
||||
os.makedirs(FILELIST_DIR, exist_ok=True)
|
||||
recent_list = os.path.join(FILELIST_DIR, "recent.txt")
|
||||
archive_list = os.path.join(FILELIST_DIR, "archive.txt")
|
||||
with open(recent_list, "w") as f:
|
||||
f.write("\n".join(recent))
|
||||
with open(archive_list, "w") as f:
|
||||
f.write("\n".join(archive))
|
||||
|
||||
lib_src = os.path.join(SRC_ROOT, "library") + "/"
|
||||
run_filelist_rsync(lib_src, LIBRARY_RECENT_DST + "/", recent_list, "library (recent)")
|
||||
with state_lock:
|
||||
state["phases_done"].append("library (recent)")
|
||||
write_output()
|
||||
|
||||
run_filelist_rsync(lib_src, LIBRARY_ARCHIVE_DST + "/", archive_list, "library (archive)")
|
||||
with state_lock:
|
||||
state["phases_done"].append("library (archive)")
|
||||
write_output()
|
||||
|
||||
run_other_folders()
|
||||
with state_lock:
|
||||
state["phases_done"].append("other folders")
|
||||
write_output()
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
write_output()
|
||||
try:
|
||||
main_flow()
|
||||
with state_lock:
|
||||
state.update(status="done", percent=100.0, finished_at=datetime.now(timezone.utc).isoformat())
|
||||
except Exception as e:
|
||||
with state_lock:
|
||||
state.update(status="error", error=str(e), finished_at=datetime.now(timezone.utc).isoformat())
|
||||
write_output()
|
||||
# stay alive so the pod doesn't restart-loop after a one-shot sync;
|
||||
# the status page keeps showing the final result.
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -121,48 +121,6 @@ spec:
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
# one-shot migration copy from the old NAS-hosted docker install
|
||||
# into this install's own volumes (see "persistence" below).
|
||||
# rsync is idempotent, so a pod restart mid-copy just resumes
|
||||
# cleanly; once done it sleeps forever instead of exiting, so the
|
||||
# container doesn't restart-loop and the status page stays up.
|
||||
# (starts after all initContainers - fix-permissions and the
|
||||
# mergerfs sidecar's startupProbe - complete, same as "app")
|
||||
rsync:
|
||||
image:
|
||||
repository: python
|
||||
tag: 3.13-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- apk add --no-cache rsync >/dev/null && python3 /app/rsync.py
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
rsync-web:
|
||||
image:
|
||||
repository: nginx
|
||||
tag: 1.27-alpine
|
||||
probes:
|
||||
# explicit port: this controller has two services (app:2283,
|
||||
# rsync-web:80) and probe auto-detection picks the
|
||||
# controller's primary service port rather than this
|
||||
# container's own - without this it checks the wrong port.
|
||||
liveness:
|
||||
enabled: true
|
||||
port: 80
|
||||
readiness:
|
||||
enabled: true
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
|
||||
service:
|
||||
app:
|
||||
@@ -170,11 +128,6 @@ spec:
|
||||
ports:
|
||||
http:
|
||||
port: 2283
|
||||
rsync-web:
|
||||
controller: server
|
||||
ports:
|
||||
http:
|
||||
port: 80
|
||||
|
||||
ingress:
|
||||
app:
|
||||
@@ -193,22 +146,6 @@ spec:
|
||||
- secretName: immich-tls
|
||||
hosts:
|
||||
- photo.aggtaa.com
|
||||
rsync-web:
|
||||
className: traefik
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
hosts:
|
||||
- host: rsync.photo.aggtaa.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
service:
|
||||
identifier: rsync-web
|
||||
port: http
|
||||
tls:
|
||||
- secretName: immich-rsync-tls
|
||||
hosts:
|
||||
- rsync.photo.aggtaa.com
|
||||
|
||||
persistence:
|
||||
# Immich's "library" folder, mergerfs-merged from two disks: recent
|
||||
@@ -229,8 +166,6 @@ spec:
|
||||
- path: /disks/library-recent
|
||||
fix-permissions:
|
||||
- path: /disks/library-recent
|
||||
rsync:
|
||||
- path: /disks/library-recent
|
||||
library-archive:
|
||||
type: persistentVolumeClaim
|
||||
accessMode: ReadWriteOnce
|
||||
@@ -243,8 +178,6 @@ spec:
|
||||
- path: /disks/library-archive
|
||||
fix-permissions:
|
||||
- path: /disks/library-archive
|
||||
rsync:
|
||||
- path: /disks/library-archive
|
||||
merged-library:
|
||||
type: emptyDir
|
||||
advancedMounts:
|
||||
@@ -274,30 +207,3 @@ spec:
|
||||
server:
|
||||
app:
|
||||
- path: /data
|
||||
rsync:
|
||||
- path: /data
|
||||
nas-source:
|
||||
type: persistentVolumeClaim
|
||||
existingClaim: immich-nas-source
|
||||
advancedMounts:
|
||||
server:
|
||||
rsync:
|
||||
- path: /source
|
||||
readOnly: true
|
||||
rsync-script:
|
||||
type: configMap
|
||||
name: immich-rsync-script
|
||||
advancedMounts:
|
||||
server:
|
||||
rsync:
|
||||
- path: /app/rsync.py
|
||||
subPath: rsync.py
|
||||
readOnly: true
|
||||
rsync-www:
|
||||
type: emptyDir
|
||||
advancedMounts:
|
||||
server:
|
||||
rsync:
|
||||
- path: /www
|
||||
rsync-web:
|
||||
- path: /usr/share/nginx/html
|
||||
|
||||
@@ -3,8 +3,6 @@ kind: Kustomization
|
||||
resources:
|
||||
- helmrepository.yaml
|
||||
- secret.enc.yaml
|
||||
- configmap-rsync.yaml
|
||||
- pv-pvc-nas-source.yaml
|
||||
- helmrelease-postgres.yaml
|
||||
- helmrelease-redis.yaml
|
||||
- helmrelease-ml.yaml
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: immich-nas-source-nfs
|
||||
spec:
|
||||
capacity:
|
||||
storage: 1Ti
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ""
|
||||
mountOptions:
|
||||
- nfsvers=3
|
||||
- nolock
|
||||
nfs:
|
||||
server: 10.0.64.80
|
||||
# .../uploads specifically, not just .../immich: this aligns 1:1 with
|
||||
# immich's own folder names (library, backups, encoded-video, profile,
|
||||
# thumbs, upload) sitting directly under /source, matching what
|
||||
# rsync.py expects.
|
||||
path: /mnt/sdb2/immich/uploads
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: photo
|
||||
name: immich-nas-source
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: immich-nas-source
|
||||
namespace: photo
|
||||
annotations:
|
||||
helm.sh/resource-policy: keep
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
storageClassName: ""
|
||||
volumeName: immich-nas-source-nfs
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Ti
|
||||
@@ -4,19 +4,19 @@ metadata:
|
||||
name: immich-secrets
|
||||
namespace: photo
|
||||
stringData:
|
||||
DB_PASSWORD: ENC[AES256_GCM,data:qIyLbynKRWbr+HkLKHMPIYKwMY2mfnWeiuji9Ami/RE=,iv:taGELFgp++UOkjJ301AYb+PI93qXJ134yPeyLkC1Vm0=,tag:PpJKK9kxpzp9PeXVBrVlDA==,type:str]
|
||||
DB_PASSWORD: ENC[AES256_GCM,data:WADkONhWfLLFqA3s6N8E3NyDbHr/Yax42xmTfov39bk=,iv:5W9nnHFT6C2kiHIylb+shQCsXHWmlpxMXItiz5HM1t4=,tag:B3/xbKPK+T0U1ZEJZmj8Aw==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- recipient: age1wmm8qrqnjzvd5nkraed6ylrsxlpn3ru7d7zettdudj7y9sg9dd2su0u3mr
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2MHhJaytNWEhlSzBzY2Zt
|
||||
cUpUZ2UwRTlPWUZHTUkyS1hLZk9qUExTdnc0CkNDOVhkYmRxaG1nckJqQ1lDeEhF
|
||||
bjJtWGlvWDRJdmpZNkI2NHViTHdMOWMKLS0tIE1zNURyWkJpV1JOc09jbEZmeHp1
|
||||
VFpZT3I0YThVVXBwTVp0Tk5Qd0tQNVEKU5hd7cj0iWJcQFvzD9VowNgnRQOAjajJ
|
||||
CaPwQNOqanQ2AesaWEFAAk6ND5AvUW7f2rdk6gA2O9lkoPuVtMtLlg==
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYRDZ3Y1lHdTZCeDN4ZFhV
|
||||
KzJMYjRGNVdOZGc1Nzg3NWdmR1g3eGlKYUdvClZyLzVZcHVPSmdDWGlKcEZ0RU45
|
||||
dzlWUFFNUVFseDQxR3BneWhvUnQySk0KLS0tIEh6TFJKYWdqVWdnSEFLSlE5U3kw
|
||||
aDRUaWRLaDd4dDh5K3VtMG1PQmYzdlkK0ccZQukupy3OQ/vz+7RY2sXIjOTcrnE5
|
||||
sMhe/VpXnpVKJihvXx+FQ1ntxzQxyKpqs9oHKjZaE8gt36te2o/Qzg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-08-26T10:28:58Z"
|
||||
mac: ENC[AES256_GCM,data:xxoauF7/+GAST+o0gGJV+Ezwsg8C5OZgXqkrkyvDt0eVmRgRAN9Oa/OMRGFJP07eNRmGtYv23IzW8f0yCqp8Oxg5YBPES3jKlG0q41rIykvtJ0PGWnn94WY5lQOPg1PC8XezihrEByncu2WyD3nPL3p3LwMG1JjNnWXEN854K3s=,iv:LmmM/koypOWhr+Po8e75zRihhKsbMyQf302dCVSy6tk=,tag:XHC6M4bCsEnVcD6ZnJZPeQ==,type:str]
|
||||
lastmodified: "2026-08-26T10:29:07Z"
|
||||
mac: ENC[AES256_GCM,data:RnTkmAsk9VbU848rIWv+yZP7Iryeod+lJTeV2Klpd9NK7BRqYdoP2YoCdXrSpKNmbTDill6bi7tyTt5m+SgSOvySBp/QgPZzBXwdUMUVUd0XkZYWCsEW/4AJEghE3TtzEVSGAymAfTAkRv9gk4T4BdZYABrC1K5IgedXlIxfAHY=,iv:NOaMCirkW0zGpx1OVwmNnulJSjq6kFgoa8pbESfq2fM=,tag:9eq+d16GD6dy72phwMAeww==,type:str]
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.11.0
|
||||
|
||||
Reference in New Issue
Block a user