From 5c747899f6d11b45741ed1329b907b11131b04ba Mon Sep 17 00:00:00 2001
From: David Gwilliam <dhgwilliam@gmail.com>
Date: Sat, 29 Aug 2026 23:07:53 -0700
Subject: [PATCH] zigbee: heartbeat uses live RestartCount to detect
 crash-loops

Old heartbeat trusted the retained zigbee2mqtt/bridge/state topic, which stays
{"state":"online"} even while the container crash-loops (6900+ restarts
went undetected). New logic: RestartCount increase since last run =
authoritative DOWN signal; container running = secondary guard; bridge state
is informational only (z2m publishes it on transition, not continuously, so
missing in a short window is normal).
---
 infra/zigbee2mqtt/scripts/kuma-heartbeat.sh | 36 +++++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/infra/zigbee2mqtt/scripts/kuma-heartbeat.sh b/infra/zigbee2mqtt/scripts/kuma-heartbeat.sh
index 44383b0..01e046f 100755
--- a/infra/zigbee2mqtt/scripts/kuma-heartbeat.sh
+++ b/infra/zigbee2mqtt/scripts/kuma-heartbeat.sh
@@ -1,20 +1,36 @@
-#!/bin/bash
+#!/bin/sh
 set -u
 
-KUMA_BASE="https://up.notsosm.art"
+KUMA_BASE="${KUMA_BASE:-https://up.notsosm.art}"
 TOKEN_FILE="/root/nr-flow-validator/infra/.secrets/kuma-push-token"
+STATE_FILE="/root/nr-flow-validator/infra/.secrets/.zb-restart-count"
 
 [ -s "$TOKEN_FILE" ] || exit 1
 TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")"
 
 reason=""
 
-state=$(docker exec mosquitto mosquitto_sub -h 127.0.0.1 -t "zigbee2mqtt/bridge/state" -C 1 -W 10 2>/dev/null | tr -d '[:space:]')
-if [ "$state" != '{"state":"online"}' ]; then
-    reason="bridge_state=${state:-none}"
-else
-    running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null)
-    [ "$running" = "true" ] || reason="${reason}container_running=${running:-unknown}"
+# 1. Container liveness.
+running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null)
+[ "$running" = "true" ] || reason="container_running=${running:-unknown}"
+
+# 2. Crash-loop detection: RestartCount climbing since last run.
+rc=$(docker inspect zigbee2mqtt --format '{{.RestartCount}}' 2>/dev/null | tr -d '[:space:]')
+if [ -n "$rc" ] && [ -f "$STATE_FILE" ]; then
+    prev=$(tr -d '[:space:]' < "$STATE_FILE")
+    if [ "${prev:-0}" -lt "$rc" ] 2>/dev/null; then
+        reason="${reason:+$reason }crashed_recently(restarts=$rc from $prev)"
+    fi
+fi
+printf '%s' "${rc:-0}" > "$STATE_FILE"
+
+# 3. Bridge state — informational only. z2m publishes /bridge/state on
+#    transitions, NOT continuously, so a missing message in a short window
+#    is expected during steady operation and must not cause a false DOWN.
+#    The restart-count check above is the authoritative crash-loop signal.
+state=$(docker exec mosquitto mosquitto_sub -R -t 'zigbee2mqtt/bridge/state' -C 1 -W 6 2>/dev/null | tr -d '[:space:]')
+if [ -z "$reason" ] && [ "$state" != '{"state":"online"}' ]; then
+    note="bridge_idle_or_down"
 fi
 
 if [ -n "$reason" ]; then
@@ -22,10 +38,10 @@ if [ -n "$reason" ]; then
     msg="coordinator: ${reason}"
 else
     status="up"
-    msg="coordinator ok"
+    msg="coordinator ok${note:+ ($note)}"
 fi
 
 curl -fsS -m 10 -o /dev/null --get \
     --data-urlencode "status=$status" \
     --data-urlencode "msg=$msg" \
-    "$KUMA_BASE/api/push/$TOKEN"
+    "$KUMA_BASE/api/push/$TOKEN"
\ No newline at end of file