zigbee: heartbeat uses live RestartCount to detect crash-loops
Old heartbeat trusted the retained zigbee2mqtt/bridge/state topic, which stays
{"state":"online"} even while the container crash-loops (6900+ restarts
went undetected). New logic: RestartCount increase since last run =
authoritative DOWN signal; container running = secondary guard; bridge state
is informational only (z2m publishes it on transition, not continuously, so
missing in a short window is normal).
This commit is contained in:
@@ -1,20 +1,36 @@
|
|||||||
#!/bin/bash
|
#!/bin/sh
|
||||||
set -u
|
set -u
|
||||||
|
|
||||||
KUMA_BASE="https://up.notsosm.art"
|
KUMA_BASE="${KUMA_BASE:-https://up.notsosm.art}"
|
||||||
TOKEN_FILE="/root/nr-flow-validator/infra/.secrets/kuma-push-token"
|
TOKEN_FILE="/root/nr-flow-validator/infra/.secrets/kuma-push-token"
|
||||||
|
STATE_FILE="/root/nr-flow-validator/infra/.secrets/.zb-restart-count"
|
||||||
|
|
||||||
[ -s "$TOKEN_FILE" ] || exit 1
|
[ -s "$TOKEN_FILE" ] || exit 1
|
||||||
TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")"
|
TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")"
|
||||||
|
|
||||||
reason=""
|
reason=""
|
||||||
|
|
||||||
state=$(docker exec mosquitto mosquitto_sub -h 127.0.0.1 -t "zigbee2mqtt/bridge/state" -C 1 -W 10 2>/dev/null | tr -d '[:space:]')
|
# 1. Container liveness.
|
||||||
if [ "$state" != '{"state":"online"}' ]; then
|
|
||||||
reason="bridge_state=${state:-none}"
|
|
||||||
else
|
|
||||||
running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null)
|
running=$(docker inspect zigbee2mqtt --format '{{.State.Running}}' 2>/dev/null)
|
||||||
[ "$running" = "true" ] || reason="${reason}container_running=${running:-unknown}"
|
[ "$running" = "true" ] || reason="container_running=${running:-unknown}"
|
||||||
|
|
||||||
|
# 2. Crash-loop detection: RestartCount climbing since last run.
|
||||||
|
rc=$(docker inspect zigbee2mqtt --format '{{.RestartCount}}' 2>/dev/null | tr -d '[:space:]')
|
||||||
|
if [ -n "$rc" ] && [ -f "$STATE_FILE" ]; then
|
||||||
|
prev=$(tr -d '[:space:]' < "$STATE_FILE")
|
||||||
|
if [ "${prev:-0}" -lt "$rc" ] 2>/dev/null; then
|
||||||
|
reason="${reason:+$reason }crashed_recently(restarts=$rc from $prev)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
printf '%s' "${rc:-0}" > "$STATE_FILE"
|
||||||
|
|
||||||
|
# 3. Bridge state — informational only. z2m publishes /bridge/state on
|
||||||
|
# transitions, NOT continuously, so a missing message in a short window
|
||||||
|
# is expected during steady operation and must not cause a false DOWN.
|
||||||
|
# The restart-count check above is the authoritative crash-loop signal.
|
||||||
|
state=$(docker exec mosquitto mosquitto_sub -R -t 'zigbee2mqtt/bridge/state' -C 1 -W 6 2>/dev/null | tr -d '[:space:]')
|
||||||
|
if [ -z "$reason" ] && [ "$state" != '{"state":"online"}' ]; then
|
||||||
|
note="bridge_idle_or_down"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$reason" ]; then
|
if [ -n "$reason" ]; then
|
||||||
@@ -22,7 +38,7 @@ if [ -n "$reason" ]; then
|
|||||||
msg="coordinator: ${reason}"
|
msg="coordinator: ${reason}"
|
||||||
else
|
else
|
||||||
status="up"
|
status="up"
|
||||||
msg="coordinator ok"
|
msg="coordinator ok${note:+ ($note)}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
curl -fsS -m 10 -o /dev/null --get \
|
curl -fsS -m 10 -o /dev/null --get \
|
||||||
|
|||||||
Reference in New Issue
Block a user