photo-stats

clusters/base/apps/photo-stats/configmap.yaml

@@ -0,0 +1,175 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: photo-stats-script
+  namespace: photo-stats
+data:
+  scan.py: |
+    import os
+    import time
+    import json
+    import threading
+    from datetime import datetime, timezone
+
+    ROOT = "/mnt/sdb2/immich/uploads/library"
+    OUT_DIR = "/www"
+    CUTOFF = datetime(2025, 1, 1, tzinfo=timezone.utc).timestamp()
+    RESCAN_INTERVAL_SECONDS = 6 * 3600
+
+    state_lock = threading.Lock()
+    state = {
+        "status": "starting",
+        "processed": 0,
+        "total": 0,
+        "percent": 0.0,
+        "total_files": 0,
+        "total_size": 0,
+        "recent_files": 0,
+        "recent_size": 0,
+        "started_at": None,
+        "finished_at": None,
+        "error": None,
+    }
+
+
+    def human_size(n):
+        n = float(n)
+        for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"):
+            if abs(n) < 1024.0:
+                return f"{n:.1f}{unit}"
+            n /= 1024.0
+        return f"{n:.1f}EiB"
+
+
+    def render_html(s):
+        progress_html = ""
+        if s["status"] == "counting":
+            progress_html = "<p>Counting files...</p>"
+        elif s["status"] == "scanning":
+            pct = s["percent"]
+            progress_html = f"""
+    <p>Progress: {s['processed']} / {s['total']} ({pct:.1f}%)</p>
+    <div class="bar"><div class="fill" style="width:{pct:.1f}%"></div></div>
+    """
+        error_html = f"<p class='err'>Error: {s['error']}</p>" if s["error"] else ""
+        return f"""<!doctype html>
+    <html><head><meta charset="utf-8"><title>Photo library stats</title>
+    <meta http-equiv="refresh" content="5">
+    <style>
+    body {{ font-family: monospace; background: #111; color: #eee; padding: 2rem; }}
+    table {{ border-collapse: collapse; }}
+    td {{ padding: .25rem 1.5rem .25rem 0; }}
+    .bar {{ background: #333; border-radius: 4px; overflow: hidden; width: 300px; height: 14px; }}
+    .fill {{ background: #4caf50; height: 100%; }}
+    .err {{ color: #f66; }}
+    </style></head><body>
+    <h1>Photo library stats</h1>
+    <p>Path: {ROOT}</p>
+    <p>Status: <b>{s['status']}</b></p>
+    {progress_html}
+    {error_html}
+    <table>
+    <tr><td>Total files</td><td>{s['total_files']}</td></tr>
+    <tr><td>Total size</td><td>{human_size(s['total_size'])} ({s['total_size']} bytes)</td></tr>
+    <tr><td>Files newer than 2025-01-01</td><td>{s['recent_files']}</td></tr>
+    <tr><td>Size of files newer than 2025-01-01</td><td>{human_size(s['recent_size'])} ({s['recent_size']} bytes)</td></tr>
+    <tr><td>Last scan started</td><td>{s['started_at'] or '-'}</td></tr>
+    <tr><td>Last scan finished</td><td>{s['finished_at'] or '-'}</td></tr>
+    </table>
+    </body></html>"""
+
+
+    def write_output():
+        with state_lock:
+            s = dict(state)
+        html = render_html(s)
+
+        tmp = os.path.join(OUT_DIR, "index.html.tmp")
+        with open(tmp, "w") as f:
+            f.write(html)
+        os.replace(tmp, os.path.join(OUT_DIR, "index.html"))
+
+        tmp = os.path.join(OUT_DIR, "stats.json.tmp")
+        with open(tmp, "w") as f:
+            json.dump(s, f, indent=2)
+        os.replace(tmp, os.path.join(OUT_DIR, "stats.json"))
+
+
+    def scan_once():
+        with state_lock:
+            state.update(status="counting", started_at=datetime.now(timezone.utc).isoformat(),
+                         finished_at=None, error=None, processed=0, total=0, percent=0.0)
+        write_output()
+
+        total = 0
+        try:
+            for _dirpath, _dirs, files in os.walk(ROOT):
+                total += len(files)
+        except OSError as e:
+            with state_lock:
+                state.update(status="error", error=str(e))
+            write_output()
+            return
+
+        with state_lock:
+            state.update(total=total, status="scanning")
+        write_output()
+
+        processed = 0
+        total_files = 0
+        total_size = 0
+        recent_files = 0
+        recent_size = 0
+        last_write = time.time()
+
+        try:
+            for dirpath, _dirs, files in os.walk(ROOT):
+                for name in files:
+                    fp = os.path.join(dirpath, name)
+                    processed += 1
+                    try:
+                        st = os.stat(fp)
+                    except OSError:
+                        continue
+                    total_files += 1
+                    total_size += st.st_size
+                    if st.st_mtime >= CUTOFF:
+                        recent_files += 1
+                        recent_size += st.st_size
+
+                    now = time.time()
+                    if now - last_write > 2:
+                        with state_lock:
+                            state.update(
+                                processed=processed, total_files=total_files,
+                                total_size=total_size, recent_files=recent_files,
+                                recent_size=recent_size,
+                                percent=(processed / total * 100) if total else 100.0,
+                            )
+                        write_output()
+                        last_write = now
+        except OSError as e:
+            with state_lock:
+                state.update(status="error", error=str(e))
+            write_output()
+            return
+
+        with state_lock:
+            state.update(
+                processed=processed, total_files=total_files, total_size=total_size,
+                recent_files=recent_files, recent_size=recent_size, percent=100.0,
+                status="done", finished_at=datetime.now(timezone.utc).isoformat(),
+            )
+        write_output()
+
+
+    def main():
+        os.makedirs(OUT_DIR, exist_ok=True)
+        write_output()
+        while True:
+            scan_once()
+            time.sleep(RESCAN_INTERVAL_SECONDS)
+
+
+    if __name__ == "__main__":
+        main()

clusters/base/apps/photo-stats/helmrelease.yaml

@@ -0,0 +1,99 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: photo-stats
+  namespace: photo-stats
+spec:
+  interval: 30m
+  chartRef:
+    kind: OCIRepository
+    name: photo-stats
+  driftDetection:
+    mode: enabled
+  values:
+    controllers:
+      app:
+        containers:
+          # walks the immich library over NFS (read-only) and periodically
+          # writes index.html/stats.json into the shared "www" volume; the
+          # "web" container just serves whatever is currently on disk there.
+          scanner:
+            image:
+              repository: python
+              tag: 3.13-alpine
+            command:
+              - python3
+              - /app/scan.py
+            resources:
+              requests:
+                cpu: 50m
+                memory: 64Mi
+              limits:
+                memory: 256Mi
+          web:
+            image:
+              repository: nginx
+              tag: 1.27-alpine
+            probes:
+              liveness:
+                enabled: true
+              readiness:
+                enabled: true
+            resources:
+              requests:
+                cpu: 10m
+                memory: 16Mi
+              limits:
+                memory: 64Mi
+
+    service:
+      app:
+        controller: app
+        ports:
+          http:
+            port: 80
+
+    ingress:
+      app:
+        className: traefik
+        annotations:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+        hosts:
+          - host: stats.photo.aggtaa.com
+            paths:
+              - path: /
+                pathType: Prefix
+                service:
+                  identifier: app
+                  port: http
+        tls:
+          - secretName: photo-stats-tls
+            hosts:
+              - stats.photo.aggtaa.com
+
+    persistence:
+      script:
+        type: configMap
+        name: photo-stats-script
+        advancedMounts:
+          app:
+            scanner:
+              - path: /app/scan.py
+                subPath: scan.py
+                readOnly: true
+      data:
+        type: persistentVolumeClaim
+        existingClaim: photo-stats-nas1-sdb2
+        advancedMounts:
+          app:
+            scanner:
+              - path: /mnt/sdb2
+                readOnly: true
+      www:
+        type: emptyDir
+        advancedMounts:
+          app:
+            scanner:
+              - path: /www
+            web:
+              - path: /usr/share/nginx/html

clusters/base/apps/photo-stats/helmrepository.yaml

@@ -0,0 +1,10 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: photo-stats
+  namespace: photo-stats
+spec:
+  interval: 1h
+  url: oci://ghcr.io/bjw-s-labs/helm/app-template
+  ref:
+    tag: "5.0.1"

clusters/base/apps/photo-stats/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helmrepository.yaml
+  - configmap.yaml
+  - pv-pvc-data.yaml
+  - helmrelease.yaml

clusters/base/apps/photo-stats/pv-pvc-data.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: photo-stats-nas1-sdb2-nfs
+spec:
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadOnlyMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  mountOptions:
+    - nfsvers=3
+    - nolock
+  nfs:
+    server: 10.0.64.80
+    path: /mnt/sdb2
+    readOnly: true
+  claimRef:
+    namespace: photo-stats
+    name: photo-stats-nas1-sdb2
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: photo-stats-nas1-sdb2
+  namespace: photo-stats
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  accessModes:
+    - ReadOnlyMany
+  storageClassName: ""
+  volumeName: photo-stats-nas1-sdb2-nfs
+  resources:
+    requests:
+      storage: 1Ti

clusters/base/namespaces/photo-stats.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: photo-stats

clusters/home/apps/kustomization.yaml

@@ -14,4 +14,5 @@ resources:
   - fava.yaml
   - infcloud.yaml
   - syncthing.yaml
-  - nfs-photo.yaml
+  - nfs-photo.yaml
+  - photo-stats.yaml

clusters/home/apps/photo-stats.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: photo-stats
+  namespace: flux-system
+spec:
+  interval: 10m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./clusters/base/apps/photo-stats
+  prune: true
+  wait: true
+  dependsOn:
+    - name: namespaces
+    - name: infra
+  retryInterval: 5s

clusters/home/namespaces/kustomization.yaml

@@ -15,3 +15,4 @@ resources:
   - ../../base/namespaces/finance.yaml
   - ../../base/namespaces/nfs.yaml
   - ../../base/namespaces/syncthing.yaml
+  - ../../base/namespaces/photo-stats.yaml