diff --git a/RESILIENCE.md b/RESILIENCE.md new file mode 100644 index 0000000..e14522f --- /dev/null +++ b/RESILIENCE.md @@ -0,0 +1,47 @@ +# Power-Loss Resilience + +The Klubhaus DietPi should come back fully operational after a power blip with +no human at the venue. This documents the failure mode and the hardening. + +## Failure mode (2026-08-30) + +Both Kuma push monitors for the Pi (`KlubHaus (ZB)`, `doorbell-listener`) stopped +reporting at 2026-08-30 21:39 UTC. The Pi's ZeroTier IP (172.30.158.123) answers +ICMP at all packet sizes (so the home→Pi ZT path is NOT an MTU black-hole), but +its LAN IP 192.168.81.147 does not answer, SSH is refused, and only +192.168.81.1 (router) + 192.168.81.134 (meshtastic) respond on the Klubhaus LAN. +Likely cause: the Pi came up but services / network / pushers did not fully +recover after a power event. + +## Hardening (see `scripts/harden-boot.sh`) + +| Concern | Fix | +|---|---| +| Boot hangs on interactive fsck after unclean shutdown | `fsck.repair=yes` in `/boot/cmdline.txt` | +| Kernel wedges with no recovery | `panic=10` in `/boot/cmdline.txt` | +| Hung box stays hung | RPi hardware watchdog (`dtparam=watchdog=on` + `RuntimeWatchdogSec=20`) | +| Docker / ZeroTier not started | `systemctl enable docker zerotier-one` | +| Containers don't come back | `restart: always` in compose; `klubhaus-stack.service` runs `docker compose up -d` for every compose project under `/root` after network+docker are up | +| Kuma not told the Pi is back | `kuma-online.sh` POSTs `status=up` to the push URLs in `/root/.kuma-push.env` | + +## Apply + +```bash +# from a machine that can SSH to the Pi (root@192.168.81.147) +just harden +# populate the Kuma push URLs once, from the Pi's existing pusher config: +# ssh root@192.168.81.147 'vim /root/.kuma-push.env' +# then reboot to apply cmdline/config.txt changes: +just restart # or ssh root@192.168.81.147 'reboot' +# verify the oneshot ran: +ssh root@192.168.81.147 'journalctl -u klubhaus-stack -b' +``` + +## Notes + +- `just sync` now backs up the live Pi files before overwriting — the live + copies are the source of truth (see AGENTS/clobber lesson). +- `scripts/kuma-online.sh` redacts the push token when it logs, so it is safe + to run verbosely. +- The stale `192.168.9.147` references were replaced with `192.168.81.147` + (the current Pi LAN address). \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 1393e65..dc539fa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ services: mosquitto: image: eclipse-mosquitto:latest container_name: mosquitto - restart: unless-stopped + restart: always user: "1883:1883" ports: - "1883:1883" @@ -14,7 +14,7 @@ services: zigbee2mqtt: image: ghcr.io/pine64/zigbee2mqtt:latest-dev # BLZ fork, NOT koenkk container_name: zigbee2mqtt - restart: unless-stopped + restart: always depends_on: - mosquitto ports: @@ -41,7 +41,7 @@ services: nodered: image: nodered/node-red:latest container_name: nodered - restart: unless-stopped + restart: always depends_on: - mosquitto user: "1000:1000" diff --git a/scripts/harden-boot.sh b/scripts/harden-boot.sh new file mode 100644 index 0000000..8cf4458 --- /dev/null +++ b/scripts/harden-boot.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Idempotent boot-resilience hardening for the Klubhaus DietPi (run as root). +# After a power blip the Pi should boot cleanly, repair its fs, restart the +# stack, and mark itself online in Kuma without human intervention. + +BOOT="${BOOT:-/boot}" +SVC=/etc/systemd/system/klubhaus-stack.service +BRINGUP=/usr/local/sbin/klubhaus-bringup.sh +ONLINE=/usr/local/sbin/kuma-online.sh + +echo "[1/6] cmdline: fsck.repair + panic reboot" +if [ -f "$BOOT/cmdline.txt" ]; then + for opt in fsck.repair=yes panic=10; do + grep -q "$opt" "$BOOT/cmdline.txt" || sed -i "s/$/ $opt/" "$BOOT/cmdline.txt" + done + echo " -> $(cat "$BOOT/cmdline.txt")" +fi + +echo "[2/6] watchdog" +if [ -f "$BOOT/config.txt" ]; then + grep -q "dtparam=watchdog=on" "$BOOT/config.txt" || echo "dtparam=watchdog=on" >> "$BOOT/config.txt" +fi +sed -i 's/^#RuntimeWatchdogSec=.*/RuntimeWatchdogSec=20/' /etc/systemd/system.conf +grep -q "^RuntimeWatchdogSec=" /etc/systemd/system.conf || echo "RuntimeWatchdogSec=20" >> /etc/systemd/system.conf + +echo "[3/6] enable docker + zerotier" +systemctl enable docker.service zerotier-one.service 2>/dev/null || true + +echo "[4/6] install bringup script" +cat > "$BRINGUP" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +while IFS= read -r f; do + docker compose -f "$f" up -d +done < <(find /root -maxdepth 2 -iname 'compose.y*ml' -print 2>/dev/null) +/usr/local/sbin/kuma-online.sh || true +EOF +chmod +x "$BRINGUP" + +echo "[5/6] install kuma-online hook" +if [ -f /root/kuma-online.sh ]; then + cp /root/kuma-online.sh "$ONLINE" + chmod +x "$ONLINE" + echo " -> installed from /root/kuma-online.sh" +else + echo " -> WARN: /root/kuma-online.sh not found; skipping (pushers already push)" +fi + +echo "[6/6] install + enable stack service" +cat > "$SVC" <<'EOF' +[Unit] +Description=Bring up Klubhaus docker compose stack and mark online in Kuma +After=network-online.target docker.service +Wants=network-online.target docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/klubhaus-bringup.sh + +[Install] +WantedBy=multi-user.target +EOF +systemctl daemon-reload +systemctl enable klubhaus-stack.service + +echo +echo "Done. A reboot applies cmdline/config.txt changes (fsck + watchdog)." +echo "Verify after reboot: journalctl -u klubhaus-stack -b" \ No newline at end of file diff --git a/scripts/kuma-online.sh b/scripts/kuma-online.sh new file mode 100644 index 0000000..a30d7e0 --- /dev/null +++ b/scripts/kuma-online.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Push an "up" heartbeat to every configured Kuma push URL. +# URLs live in /root/.kuma-push.env (one per line, # for comments). +ENV_FILE=/root/.kuma-push.env + +[ -f "$ENV_FILE" ] || { echo "no $ENV_FILE"; exit 0; } + +while IFS= read -r url; do + case "$url" in + ""|"#"*) continue ;; + esac + if curl -fsS -G --data-urlencode "status=up" --data-urlencode "msg=pi-online" "$url" >/dev/null 2>&1; then + echo "OK ${url#https://up.notsosm.art/api/push/}" + else + echo "FAIL ${url#https://up.notsosm.art/api/push/}" + fi +done < "$ENV_FILE" \ No newline at end of file