zigbee: heartbeat uses live RestartCount to detect crash-loops

Old heartbeat trusted the retained zigbee2mqtt/bridge/state topic, which stays
{"state":"online"} even while the container crash-loops (6900+ restarts
went undetected). New logic: RestartCount increase since last run =
authoritative DOWN signal; container running = secondary guard; bridge state
is informational only (z2m publishes it on transition, not continuously, so
missing in a short window is normal).
This commit is contained in:
2026-08-29 23:07:53 -07:00
parent f4d6be0364
commit 5c747899f6
+26 -10
View File
@@ -1,20 +1,36 @@
#!/bin/bash #!/bin/sh
set -u set -u
KUMA_BASE="https://up.notsosm.art" KUMA_BASE="${KUMA_BASE:-https://up.notsosm.art}"
TOKEN_FILE="/root/nr-flow-validator/infra/.secrets/kuma-push-token" TOKEN_FILE="/root/nr-flow-validator/infra/.secrets/kuma-push-token"
STATE_FILE="/root/nr-flow-validator/infra/.secrets/.zb-restart-count"
[ -s "$TOKEN_FILE" ] || exit 1 [ -s "$TOKEN_FILE" ] || exit 1
TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")" TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")"
reason="" reason=""
state=$(docker exec mosquitto mosquitto_sub -h 127.0.0.1 -t "zigbee2mqtt/bridge/state" -C 1 -W 10 2>/dev/null | tr -d '[:space:]') # 1. Container liveness.
if [ "$state" != '{"state":"online"}' ]; then running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null)
reason="bridge_state=${state:-none}" [ "$running" = "true" ] || reason="container_running=${running:-unknown}"
else
running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null) # 2. Crash-loop detection: RestartCount climbing since last run.
[ "$running" = "true" ] || reason="${reason}container_running=${running:-unknown}" rc=$(docker inspect zigbee2mqtt --format '{{.RestartCount}}' 2>/dev/null | tr -d '[:space:]')
if [ -n "$rc" ] && [ -f "$STATE_FILE" ]; then
prev=$(tr -d '[:space:]' < "$STATE_FILE")
if [ "${prev:-0}" -lt "$rc" ] 2>/dev/null; then
reason="${reason:+$reason }crashed_recently(restarts=$rc from $prev)"
fi
fi
printf '%s' "${rc:-0}" > "$STATE_FILE"
# 3. Bridge state — informational only. z2m publishes /bridge/state on
# transitions, NOT continuously, so a missing message in a short window
# is expected during steady operation and must not cause a false DOWN.
# The restart-count check above is the authoritative crash-loop signal.
state=$(docker exec mosquitto mosquitto_sub -R -t 'zigbee2mqtt/bridge/state' -C 1 -W 6 2>/dev/null | tr -d '[:space:]')
if [ -z "$reason" ] && [ "$state" != '{"state":"online"}' ]; then
note="bridge_idle_or_down"
fi fi
if [ -n "$reason" ]; then if [ -n "$reason" ]; then
@@ -22,10 +38,10 @@ if [ -n "$reason" ]; then
msg="coordinator: ${reason}" msg="coordinator: ${reason}"
else else
status="up" status="up"
msg="coordinator ok" msg="coordinator ok${note:+ ($note)}"
fi fi
curl -fsS -m 10 -o /dev/null --get \ curl -fsS -m 10 -o /dev/null --get \
--data-urlencode "status=$status" \ --data-urlencode "status=$status" \
--data-urlencode "msg=$msg" \ --data-urlencode "msg=$msg" \
"$KUMA_BASE/api/push/$TOKEN" "$KUMA_BASE/api/push/$TOKEN"