From ad55dbc5e02fb4e3b13ba8a687dbdcd92d7ee36f Mon Sep 17 00:00:00 2001
From: David Gwilliam <dhgwilliam@gmail.com>
Date: Wed, 24 Jun 2026 02:38:51 -0700
Subject: [PATCH] add viz diagnostic metrics to prometheus-exporter

- Viz writes stats to /tmp/viz-stats.json (frames_sent, frames_dropped, queue_depth)
- Exporter reads viz-stats.json and exposes:
  - viz_frames_sent_total (counter)
  - viz_frames_dropped_total (counter)
  - viz_pending_queue (gauge)
  - viz_queue_full (gauge)
---
 scripts/led-matrix-viz.py      | 24 +++++++++++++++++++++
 scripts/prometheus-exporter.py | 38 +++++++++++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/scripts/led-matrix-viz.py b/scripts/led-matrix-viz.py
index 23569de..818df6c 100644
--- a/scripts/led-matrix-viz.py
+++ b/scripts/led-matrix-viz.py
@@ -15,6 +15,7 @@ ROWS = 8
 frame_interval = 1.0 / 30.0  # 30 FPS target
 next_frame_time = 0.0
 
+STATS_FILE = "/tmp/viz-stats.json"
 _poll = select.poll()
 POLLIN = select.POLLIN
 POLLOUT = select.POLLOUT
@@ -191,6 +192,29 @@ def main():
                     frames_dropped += 1
                 last_frame = frame
 
+            # Write stats periodically (every 2s)
+            if now - _stats_last_write >= 2.0:
+                _write_stats(frames_sent, frames_dropped, pending_frames)
+                _stats_last_write = now
+
+
+_stats_last_write = 0.0
+
+
+def _write_stats(sent, dropped, pending):
+    import json
+    try:
+        with open(STATS_FILE, "w") as f:
+            json.dump({
+                "frames_sent": sent,
+                "frames_dropped": dropped,
+                "pending_queue": len(pending),
+                "queue_full": pending.full(),
+                "ts": time.time()
+            }, f)
+    except OSError:
+        pass
+
 
 if __name__ == "__main__":
     main()
\ No newline at end of file
diff --git a/scripts/prometheus-exporter.py b/scripts/prometheus-exporter.py
index 6251609..0f495c6 100644
--- a/scripts/prometheus-exporter.py
+++ b/scripts/prometheus-exporter.py
@@ -4,7 +4,7 @@
 Exposes MIDI, audio, and service health metrics on :9090/metrics.
 No external dependencies — uses only stdlib.
 """
-import http.server, time, os, re, subprocess, socket, threading
+import http.server, time, os, re, subprocess, socket, threading, json
 
 PORT = int(os.environ.get("METRICS_PORT", "9090"))
 SSH_KEY = os.environ.get("SSH_KEY", "/home/david/.ssh/unoq_deploy_key")
@@ -24,11 +24,25 @@ metrics = {
     "synth_hphl_volume": 0,
     "synth_hphr_volume": 0,
     "synth_up": 1,
+    "viz_frames_sent": 0,
+    "viz_frames_dropped": 0,
+    "viz_pending_queue": 0,
+    "viz_queue_full": 0,
 }
 
 prev_hw_ptr = 0
 prev_hw_ptr_time = 0
 
+VIZ_STATS_FILE = "/tmp/viz-stats.json"
+
+
+def read_viz_stats():
+    try:
+        with open(VIZ_STATS_FILE) as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return {}
+
 
 def read_local_file(path):
     try:
@@ -106,6 +120,12 @@ def collect_metrics():
             metrics["synth_hphr_switch"] = get_mixer_value("HPHR Switch")
             metrics["synth_hphl_volume"] = get_mixer_value("HPHL Volume")
             metrics["synth_hphr_volume"] = get_mixer_value("HPHR Volume")
+
+            viz = read_viz_stats()
+            metrics["viz_frames_sent"] = viz.get("frames_sent", 0)
+            metrics["viz_frames_dropped"] = viz.get("frames_dropped", 0)
+            metrics["viz_pending_queue"] = viz.get("pending_queue", 0)
+            metrics["viz_queue_full"] = 1 if viz.get("queue_full", False) else 0
         except Exception:
             pass
 
@@ -158,6 +178,22 @@ def format_metrics():
     lines.append("# TYPE synth_up gauge")
     lines.append(f"synth_up {metrics['synth_up']}")
 
+    lines.append("# HELP viz_frames_sent_total Total frames sent to router")
+    lines.append("# TYPE viz_frames_sent_total counter")
+    lines.append(f"viz_frames_sent_total {metrics['viz_frames_sent']}")
+
+    lines.append("# HELP viz_frames_dropped_total Total frames dropped (queue full)")
+    lines.append("# TYPE viz_frames_dropped_total counter")
+    lines.append(f"viz_frames_dropped_total {metrics['viz_frames_dropped']}")
+
+    lines.append("# HELP viz_pending_queue Current pending frame queue depth")
+    lines.append("# TYPE viz_pending_queue gauge")
+    lines.append(f"viz_pending_queue {metrics['viz_pending_queue']}")
+
+    lines.append("# HELP viz_queue_full Router write buffer was full")
+    lines.append("# TYPE viz_queue_full gauge")
+    lines.append(f"viz_queue_full {metrics['viz_queue_full']}")
+
     return "\n".join(lines) + "\n"