naywatch: add naywatch

The network may crash on the SoCs, but the SoC itself does not. This
leads to a node no longer being accessible in a mesh network.  If the
node is placed in a location that is not easily accessible, e.g. in a
high tower, it can cause a lot of problems. Therefore we check the
link-local connectivity on the configured interfaces.

Signed-off-by: Nick Hainke <vincent@systemli.org>
Tested-by: Simon Polack <spolack+git@mailbox.org>
Signed-off-by: Simon Polack <spolack+git@mailbox.org>

naywatch/Makefile

@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2021 Nick Hainke <vincent@systemli.org>
+#
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=naywatch
+PKG_VERSION:=1
+PKG_RELEASE:=$(AUTORELEASE)
+
+PKG_MAINTAINER:=Nick Hainke <vincent@systemli.org>
+PKG_LICENSE:=GPL-2.0-only
+
+include $(INCLUDE_DIR)/package.mk
+
+define Package/naywatch
+  SECTION:=utils
+  CATEGORY:=Utilities
+  TITLE:=Watchdog for IPv6 links
+  PKGARCH:=all
+  DEPENDS:=@IPV6 +owipcalc
+endef
+
+define Package/naywatch/description
+Reboots or triggers watchdog if no link local neighbor is available.
+endef
+
+define Package/naywatch/conffiles
+/etc/config/naywatch
+endef
+
+define Build/Compile
+endef
+
+define Package/naywatch/install
+	$(INSTALL_DIR) $(1)/etc/init.d
+	$(INSTALL_BIN) ./files/naywatch.init $(1)/etc/init.d/naywatch
+	$(INSTALL_DIR) $(1)/usr/bin
+	$(INSTALL_BIN) ./files/naywatch.sh $(1)/usr/bin/naywatch
+	$(INSTALL_DIR) $(1)/etc/config
+	$(INSTALL_DATA) ./files/naywatch.config $(1)/etc/config/naywatch
+endef
+
+$(eval $(call BuildPackage,naywatch))

naywatch/files/naywatch.config

@@ -0,0 +1,8 @@
+config naywatch general
+    option check_interval   '50'
+    option watchdog_timeout '60'
+    option use_watchdog     '1'
+    option save_logs        '1'
+    list interface          'lan'
+    list interface          'wan'
+    list save_cmd           'dmesg'

naywatch/files/naywatch.init

@@ -0,0 +1,65 @@
+#!/bin/sh /etc/rc.common
+
+USE_PROCD=1
+START=95
+STOP=01
+
+log() {
+    local msg="$1"
+    logger -t naywatch "$msg"
+}
+
+wait_for_network()
+{
+    ubus -t 15 wait_for network.interface.$1 2>/dev/null
+}
+
+boot()
+{
+    local _interfaces
+    config_load naywatch
+    config_get _interfaces general interface
+
+    for interface in $_interfaces; do
+        wait_for_network interface
+    done
+
+    rc_procd start_service
+}
+
+start_service() {
+    procd_open_instance
+
+    config_load naywatch
+    local _check_interval
+    local _watchdog_timeout
+    local _use_watchdog
+    local _save_logs
+    local _interfaces
+
+    config_get _check_interval general "check_interval"
+    config_get _watchdog_timeout general "watchdog_timeout"
+    config_get _use_watchdog general "use_watchdog"
+    config_get _save_logs general "save_logs"
+    config_get _interfaces general "interface"
+
+    procd_set_param command /usr/bin/naywatch "$_check_interval" "$_watchdog_timeout" "$_use_watchdog" "$_save_logs" "$_interfaces"
+
+    procd_set_param respawn 3600 15 0
+
+    procd_set_param stdout 1
+    procd_set_param stderr 1
+
+    procd_close_instance
+}
+
+stop_service() {
+    exec 3>&- # close file again
+    sync && wait
+}
+
+service_stopped() {
+    log "Naywatch Stopped!"
+    log "Handover Watchdog to procd again:"
+    ubus call system watchdog '{"magicclose":true,"stop":false}' > /dev/null
+}

naywatch/files/naywatch.sh

@@ -0,0 +1,117 @@
+#!/bin/sh
+
+. /lib/functions.sh
+. /lib/functions/network.sh
+
+CHECK_INTERVAL=$1
+shift
+WATCHDOG_TIMEOUT=$1
+shift
+USE_WATCHDOG=$1
+shift
+SAVE_LOGS=$1
+shift
+INTERFACES="$*"
+
+ACTIVE=0
+
+log() {
+    local msg="$1"
+    logger -t naywatch "$msg"
+}
+
+write_logs() {
+    save_log() {
+        eval $1 > /root/$(date +%s)-"$1".log
+    }
+    config_load naywatch
+    config_list_foreach general save_cmd save_log
+    sync
+}
+
+neighbors_available() {
+    local phy
+
+    for interface in $INTERFACES; do
+        network_get_physdev phy $interface > /dev/null 2>&1
+        linklocal=$(ip -6 a list dev $phy | grep "scope link" | awk '{print $2}' | sed 's/\/64//') 2> /dev/null
+        ips=$(ping ff02::1%$phy -w5 -W5 -c2 | awk '/from/{print($4)}' | sed 's/.$//') 2> /dev/null
+        for ip in $ips; do
+            if [ $ip != $linklocal ] && [ $(owipcalc $ip linklocal) -eq 1 ]; then
+                echo 1
+                return 0
+            fi
+        done
+    done
+
+    echo 0
+}
+
+activate_watchdog() {
+    # disable openwrt instrumentation:
+    ubus call system watchdog '{"magicclose":true,"stop":true,"timeout":'${WATCHDOG_TIMEOUT}'}' > /dev/null
+    exec 3>/dev/watchdog
+}
+
+reboot_now() {
+    # copied from watch-cat
+    reboot &
+
+    [ "$1" -ge 1 ] && {
+        sleep "$1"
+        echo 1 >/proc/sys/kernel/sysrq
+        echo b >/proc/sysrq-trigger
+    }
+}
+
+no_neighbors() {
+    log "No Neighbors Available!"
+
+    if [ $ACTIVE -eq 0 ]; then
+        return 0
+    fi
+
+    if [ $SAVE_LOGS ]; then
+        log "Saving Logs!"
+        write_logs
+    fi
+
+    if [ $USE_WATCHDOG -eq 0 ]; then
+        reboot_now
+    fi
+}
+
+log "Naywatch Started!"
+
+neighbors() {
+    ACTIVE=1
+    if [ $USE_WATCHDOG ]; then
+        echo 1 >&3
+    fi
+}
+
+not_active() {
+    if [ $USE_WATCHDOG ]; then
+        echo 1 >&3
+    fi 
+}
+
+if [ $USE_WATCHDOG ]; then
+    activate_watchdog
+fi
+
+while [ 1 ]; do
+    # first sleep
+    sleep $CHECK_INTERVAL
+
+    has_neighbor=$(neighbors_available)
+    if [ $has_neighbor -eq 0 ] && [ $ACTIVE -eq 1 ]; then
+        no_neighbors
+    elif [ $has_neighbor -eq 1 ]; then
+        neighbors
+    else
+        not_active
+    fi
+done
+
+exit 0