diff --git a/clusters/base/apps/photo-stats/configmap.yaml b/clusters/base/apps/photo-stats/configmap.yaml new file mode 100644 index 0000000..711aa65 --- /dev/null +++ b/clusters/base/apps/photo-stats/configmap.yaml @@ -0,0 +1,175 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: photo-stats-script + namespace: photo-stats +data: + scan.py: | + import os + import time + import json + import threading + from datetime import datetime, timezone + + ROOT = "/mnt/sdb2/immich/uploads/library" + OUT_DIR = "/www" + CUTOFF = datetime(2025, 1, 1, tzinfo=timezone.utc).timestamp() + RESCAN_INTERVAL_SECONDS = 6 * 3600 + + state_lock = threading.Lock() + state = { + "status": "starting", + "processed": 0, + "total": 0, + "percent": 0.0, + "total_files": 0, + "total_size": 0, + "recent_files": 0, + "recent_size": 0, + "started_at": None, + "finished_at": None, + "error": None, + } + + + def human_size(n): + n = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): + if abs(n) < 1024.0: + return f"{n:.1f}{unit}" + n /= 1024.0 + return f"{n:.1f}EiB" + + + def render_html(s): + progress_html = "" + if s["status"] == "counting": + progress_html = "

Counting files...

" + elif s["status"] == "scanning": + pct = s["percent"] + progress_html = f""" +

Progress: {s['processed']} / {s['total']} ({pct:.1f}%)

+
+ """ + error_html = f"

Error: {s['error']}

" if s["error"] else "" + return f""" + Photo library stats + + +

Photo library stats

+

Path: {ROOT}

+

Status: {s['status']}

+ {progress_html} + {error_html} + + + + + + + +
Total files{s['total_files']}
Total size{human_size(s['total_size'])} ({s['total_size']} bytes)
Files newer than 2025-01-01{s['recent_files']}
Size of files newer than 2025-01-01{human_size(s['recent_size'])} ({s['recent_size']} bytes)
Last scan started{s['started_at'] or '-'}
Last scan finished{s['finished_at'] or '-'}
+ """ + + + def write_output(): + with state_lock: + s = dict(state) + html = render_html(s) + + tmp = os.path.join(OUT_DIR, "index.html.tmp") + with open(tmp, "w") as f: + f.write(html) + os.replace(tmp, os.path.join(OUT_DIR, "index.html")) + + tmp = os.path.join(OUT_DIR, "stats.json.tmp") + with open(tmp, "w") as f: + json.dump(s, f, indent=2) + os.replace(tmp, os.path.join(OUT_DIR, "stats.json")) + + + def scan_once(): + with state_lock: + state.update(status="counting", started_at=datetime.now(timezone.utc).isoformat(), + finished_at=None, error=None, processed=0, total=0, percent=0.0) + write_output() + + total = 0 + try: + for _dirpath, _dirs, files in os.walk(ROOT): + total += len(files) + except OSError as e: + with state_lock: + state.update(status="error", error=str(e)) + write_output() + return + + with state_lock: + state.update(total=total, status="scanning") + write_output() + + processed = 0 + total_files = 0 + total_size = 0 + recent_files = 0 + recent_size = 0 + last_write = time.time() + + try: + for dirpath, _dirs, files in os.walk(ROOT): + for name in files: + fp = os.path.join(dirpath, name) + processed += 1 + try: + st = os.stat(fp) + except OSError: + continue + total_files += 1 + total_size += st.st_size + if st.st_mtime >= CUTOFF: + recent_files += 1 + recent_size += st.st_size + + now = time.time() + if now - last_write > 2: + with state_lock: + state.update( + processed=processed, total_files=total_files, + total_size=total_size, recent_files=recent_files, + recent_size=recent_size, + percent=(processed / total * 100) if total else 100.0, + ) + write_output() + last_write = now + except OSError as e: + with state_lock: + state.update(status="error", error=str(e)) + write_output() + return + + with state_lock: + state.update( + processed=processed, total_files=total_files, total_size=total_size, + recent_files=recent_files, recent_size=recent_size, percent=100.0, + status="done", finished_at=datetime.now(timezone.utc).isoformat(), + ) + write_output() + + + def main(): + os.makedirs(OUT_DIR, exist_ok=True) + write_output() + while True: + scan_once() + time.sleep(RESCAN_INTERVAL_SECONDS) + + + if __name__ == "__main__": + main() diff --git a/clusters/base/apps/photo-stats/helmrelease.yaml b/clusters/base/apps/photo-stats/helmrelease.yaml new file mode 100644 index 0000000..ded6cd0 --- /dev/null +++ b/clusters/base/apps/photo-stats/helmrelease.yaml @@ -0,0 +1,99 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: photo-stats + namespace: photo-stats +spec: + interval: 30m + chartRef: + kind: OCIRepository + name: photo-stats + driftDetection: + mode: enabled + values: + controllers: + app: + containers: + # walks the immich library over NFS (read-only) and periodically + # writes index.html/stats.json into the shared "www" volume; the + # "web" container just serves whatever is currently on disk there. + scanner: + image: + repository: python + tag: 3.13-alpine + command: + - python3 + - /app/scan.py + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + web: + image: + repository: nginx + tag: 1.27-alpine + probes: + liveness: + enabled: true + readiness: + enabled: true + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + + service: + app: + controller: app + ports: + http: + port: 80 + + ingress: + app: + className: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + hosts: + - host: stats.photo.aggtaa.com + paths: + - path: / + pathType: Prefix + service: + identifier: app + port: http + tls: + - secretName: photo-stats-tls + hosts: + - stats.photo.aggtaa.com + + persistence: + script: + type: configMap + name: photo-stats-script + advancedMounts: + app: + scanner: + - path: /app/scan.py + subPath: scan.py + readOnly: true + data: + type: persistentVolumeClaim + existingClaim: photo-stats-nas1-sdb2 + advancedMounts: + app: + scanner: + - path: /mnt/sdb2 + readOnly: true + www: + type: emptyDir + advancedMounts: + app: + scanner: + - path: /www + web: + - path: /usr/share/nginx/html diff --git a/clusters/base/apps/photo-stats/helmrepository.yaml b/clusters/base/apps/photo-stats/helmrepository.yaml new file mode 100644 index 0000000..c8f381a --- /dev/null +++ b/clusters/base/apps/photo-stats/helmrepository.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: photo-stats + namespace: photo-stats +spec: + interval: 1h + url: oci://ghcr.io/bjw-s-labs/helm/app-template + ref: + tag: "5.0.1" diff --git a/clusters/base/apps/photo-stats/kustomization.yaml b/clusters/base/apps/photo-stats/kustomization.yaml new file mode 100644 index 0000000..39a396d --- /dev/null +++ b/clusters/base/apps/photo-stats/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helmrepository.yaml + - configmap.yaml + - pv-pvc-data.yaml + - helmrelease.yaml diff --git a/clusters/base/apps/photo-stats/pv-pvc-data.yaml b/clusters/base/apps/photo-stats/pv-pvc-data.yaml new file mode 100644 index 0000000..29bb943 --- /dev/null +++ b/clusters/base/apps/photo-stats/pv-pvc-data.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: photo-stats-nas1-sdb2-nfs +spec: + capacity: + storage: 1Ti + accessModes: + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + mountOptions: + - nfsvers=3 + - nolock + nfs: + server: 10.0.64.80 + path: /mnt/sdb2 + readOnly: true + claimRef: + namespace: photo-stats + name: photo-stats-nas1-sdb2 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: photo-stats-nas1-sdb2 + namespace: photo-stats + annotations: + helm.sh/resource-policy: keep +spec: + accessModes: + - ReadOnlyMany + storageClassName: "" + volumeName: photo-stats-nas1-sdb2-nfs + resources: + requests: + storage: 1Ti diff --git a/clusters/base/namespaces/photo-stats.yaml b/clusters/base/namespaces/photo-stats.yaml new file mode 100644 index 0000000..0331213 --- /dev/null +++ b/clusters/base/namespaces/photo-stats.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: photo-stats diff --git a/clusters/home/apps/kustomization.yaml b/clusters/home/apps/kustomization.yaml index 5e191bb..dcd3c1e 100644 --- a/clusters/home/apps/kustomization.yaml +++ b/clusters/home/apps/kustomization.yaml @@ -14,4 +14,5 @@ resources: - fava.yaml - infcloud.yaml - syncthing.yaml - - nfs-photo.yaml \ No newline at end of file + - nfs-photo.yaml + - photo-stats.yaml \ No newline at end of file diff --git a/clusters/home/apps/photo-stats.yaml b/clusters/home/apps/photo-stats.yaml new file mode 100644 index 0000000..3086112 --- /dev/null +++ b/clusters/home/apps/photo-stats.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: photo-stats + namespace: flux-system +spec: + interval: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./clusters/base/apps/photo-stats + prune: true + wait: true + dependsOn: + - name: namespaces + - name: infra + retryInterval: 5s diff --git a/clusters/home/namespaces/kustomization.yaml b/clusters/home/namespaces/kustomization.yaml index 8a8d849..cb7ffb7 100644 --- a/clusters/home/namespaces/kustomization.yaml +++ b/clusters/home/namespaces/kustomization.yaml @@ -15,3 +15,4 @@ resources: - ../../base/namespaces/finance.yaml - ../../base/namespaces/nfs.yaml - ../../base/namespaces/syncthing.yaml + - ../../base/namespaces/photo-stats.yaml