Merge pull request #14225 from nickberry17/update_watchcat_util

watchcat: add restart_iface and ping size features + refactor
This commit is contained in:
Rosen Penev 2020-12-22 20:47:44 -08:00 committed by GitHub
commit d5047303d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 293 additions and 139 deletions

View File

@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
PKG_NAME:=watchcat
PKG_VERSION:=1
PKG_RELEASE:=8
PKG_RELEASE:=11
PKG_MAINTAINER:=Roger D <rogerdammit@gmail.com>
PKG_LICENSE:=GPL-2.0

View File

@ -5,123 +5,120 @@ START=97
PIDFILE="/tmp/run/watchcat"
append_string() {
local varname="$1"; local add="$2"; local separator="${3:- }"; local actual
varname="$1"
add="$2"
separator="${3:- }"
actual
eval "actual=\$$varname"
new="${actual:+$actual$separator}$add"
eval "$varname=\$new"
}
timetoseconds() {
local time=$1
unset seconds
time_to_seconds() {
time=$1
{ [ "$time" -ge 1 ] 2> /dev/null && seconds="$time"; } || \
{ [ "${time%s}" -ge 1 ] 2> /dev/null && seconds="${time%s}"; } || \
{ [ "${time%m}" -ge 1 ] 2> /dev/null && seconds=$((${time%m}*60)); } || \
{ [ "${time%h}" -ge 1 ] 2> /dev/null && seconds=$((${time%h}*3600)); } || \
{ [ "${time%d}" -ge 1 ] 2> /dev/null && seconds=$((${time%d}*86400)); }
{ [ "$time" -ge 1 ] 2>/dev/null && seconds="$time"; } ||
{ [ "${time%s}" -ge 1 ] 2>/dev/null && seconds="${time%s}"; } ||
{ [ "${time%m}" -ge 1 ] 2>/dev/null && seconds=$((${time%m} * 60)); } ||
{ [ "${time%h}" -ge 1 ] 2>/dev/null && seconds=$((${time%h} * 3600)); } ||
{ [ "${time%d}" -ge 1 ] 2>/dev/null && seconds=$((${time%d} * 86400)); }
echo $seconds
unset seconds
unset time
}
load_watchcat() {
config_get period $1 period
config_get mode $1 mode
config_get pinghosts $1 pinghosts
config_get pingperiod $1 pingperiod
config_get nopingtime $1 nopingtime
config_get forcedelay $1 forcedelay
local nopingtime_dflt="900"
local forcedelay_dflt="60"
# Fix potential typo in mode (backward compatibility).
[ "$mode" = "allways" ] && mode="always"
config_get period "$1" period "120"
config_get mode "$1" mode "restart_iface"
config_get pinghosts "$1" pinghosts "8.8.8.8"
config_get pingperiod "$1" pingperiod "60"
config_get forcedelay "$1" forcedelay "60"
config_get pingsize "$1" pingsize "standard"
config_get interface "$1" interface
config_get mmifacename "$1" mmifacename
config_get unlockbands "$1" unlockbands "0"
# Fix potential typo in mode and provide backward compatibility.
[ "$mode" = "allways" ] && mode="periodic_reboot"
[ "$mode" = "always" ] && mode="periodic_reboot"
[ "$mode" = "ping" ] && mode="ping_reboot"
error=""
warn=""
if [ -z "$period" ]
then
append_string "error" "period is not set! Use time value(ex: '30'; '4m'; '6h'; '2d')." "; "
else
timetoseconds "$period";period="$seconds"
[ "$period" -ge 1 ] \
|| append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "
# Checks for settings common to all operation modes
if [ "$mode" != "periodic_reboot" ] && [ "$mode" != "ping_reboot" ] && [ "$mode" != "restart_iface" ]; then
append_string "error" "mode must be 'periodic_reboot' or 'ping_reboot' or 'restart_iface'" "; "
fi
[ "$mode" = "always" -o "$mode" = "ping" ] \
|| append_string "error" "mode must be 'always' or 'ping'" "; "
if [ -z "$forcedelay" ]
then
forcedelay="$forcedelay_dflt"
append_string "warn" "forcedelay is not configured! Defaulted to $forcedelay seconds" "; "
else
[ "$forcedelay" -ge 0 ] || {
forcedelay="$forcedelay_dflt"
append_string "warn" "forcedelay is invalid! Defaulted to $forcedelay seconds" "; "
}
period="$(time_to_seconds "$period")"
[ "$period" -ge 1 ] ||
append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "
# ping_reboot mode and restart_iface mode specific checks
if [ "$mode" = "ping_reboot" ] || [ "$mode" = "restart_iface" ]; then
if [ -z "$error" ]; then
pingperiod_default="$((period / 5))"
pingperiod="$(time_to_seconds "$pingperiod")"
if [ "$pingperiod" -ge 0 ] && [ "$pingperiod" -ge "$period" ]; then
pingperiod="$(time_to_seconds "$pingperiod_default")"
append_string "warn" "pingperiod cannot be greater than $period. Defaulted to $pingperiod_default seconds (1/5 of period)" "; "
fi
if [ "$pingperiod" -lt 0 ]; then
append_string "warn" "pingperiod cannot be a negative value." "; "
fi
if [ "$mmifacename" != "" ] && [ "$period" -lt 30 ]; then
append_string "error" "Check interval is less than 30s. For robust operation with ModemManager modem interfaces it is recommended to set the period to at least 30s."
fi
fi
fi
[ -z "$error" -a "$mode" = "ping" ] && {
[ -z "$pinghosts" ] \
&& append_string "error" "pinghosts must be set in 'ping' mode! Use space separated address list (ex: '8.8.8.8 9.9.9.9')" "; "
if [ -z "$nopingtime" ]
then
nopingtime="$nopingtime_dflt"
append_string "warn" "nopingtime is not configured! Defaulted to $nopingtime seconds" "; "
else
timetoseconds "$nopingtime";nopingtime="$seconds"
[ "$nopingtime" -ge 0 ] || {
nopingtime="$nopingtime_dflt"
append_string "warn" "nopingtime invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $nopingtime seconds" "; "
}
fi
local pingperiod_dflt="$((period/5))"
if [ -z "$pingperiod" ]
then
pingperiod="$pingperiod_dflt"
append_string "warn" "pingperiod is not configured! Defaulted to $pingperiod seconds(1/5 of period)" "; "
else
timetoseconds "$pingperiod";pingperiod="$seconds"
[ "$pingperiod" -ge 0 -a "$pingperiod" -ge "$period" ] && {
pingperiod="$pingperiod_dflt"
append_string "warn" "pingperiod is invalid value(greater than period)! Defaulted to $pingperiod seconds(1/5 of period)" "; "
}
[ "$pingperiod" -ge 0 ] || {
pingperiod="$pingperiod_dflt"
append_string "warn" "pingperiod has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $pingperiod seconds(1/5 of period)" "; "
}
fi
}
# ping_reboot mode and periodic_reboot mode specific checks
if [ "$mode" = "ping_reboot" ] || [ "$mode" = "periodic_reboot" ]; then
forcedelay="$(time_to_seconds "$forcedelay")"
fi
[ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn"
[ -n "$error" ] && { logger -p user.err -t "watchcat" "reboot program $1 not started - $error"; return; }
[ -n "$error" ] && {
logger -p user.err -t "watchcat" "reboot program $1 not started - $error"
return
}
if [ "$mode" = "always" ]
then
/usr/bin/watchcat.sh "always" "$period" "$forcedelay" &
case "$mode" in
periodic_reboot)
/usr/bin/watchcat.sh "periodic_reboot" "$period" "$forcedelay" &
logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)"
else
/usr/bin/watchcat.sh "ping" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$nopingtime" &
logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;nopingtime=$nopingtime)"
fi
;;
ping_reboot)
/usr/bin/watchcat.sh "ping_reboot" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$pingsize" &
logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;pingsize=$pingsize)"
;;
restart_iface)
/usr/bin/watchcat.sh "restart_iface" "$period" "$pinghosts" "$pingperiod" "$pingsize" "$interface" "$mmifacename" &
logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;pingsize=$pingsize;interface=$interface;mmifacename=$mmifacename;unlockbands=$unlockbands)"
;;
*)
echo "Error starting Watchcat service. Invalid mode selection: $mode"
;;
esac
echo $! >> "${PIDFILE}.pids"
echo $! >>"${PIDFILE}.pids"
}
stop() {
if [ -f "${PIDFILE}.pids" ]
then
if [ -f "${PIDFILE}.pids" ]; then
logger -p user.info -t "watchcat" "stopping all tasks"
while read pid
do
while read pid; do
kill -KILL "$pid"
done < "${PIDFILE}.pids"
done <"${PIDFILE}.pids"
rm "${PIDFILE}.pids"
@ -135,8 +132,7 @@ start() {
[ -f "${PIDFILE}.pids" ] && stop
config_load system
if [ -n "$(uci show system.@watchcat[0])" ] # at least one watchcat section exists
then
if [ -n "$(uci show system.@watchcat[0])" ]; then # at least one watchcat section exists
logger -p user.info -t "watchcat" "starting all tasks"
config_foreach load_watchcat watchcat
logger -p user.info -t "watchcat" "all tasks started"

View File

@ -3,7 +3,7 @@
uci -q show system.@watchcat[0] || {
uci add system watchcat
uci set system.@watchcat[0].period=6h
uci set system.@watchcat[0].mode=ping
uci set system.@watchcat[0].mode=ping_reboot
uci set system.@watchcat[0].pinghosts=8.8.8.8
uci set system.@watchcat[0].forcedelay=30
uci commit

View File

@ -1,65 +1,223 @@
#!/bin/sh
#
# Copyright (C) 2010 segal.di.ubi.pt
# Copyright (C) 2020 nbembedded.com
#
# This is free software, licensed under the GNU General Public License v2.
#
reboot_now() {
reboot &
[ "$1" -ge 1 ] && {
sleep "$1"
echo 1 > /proc/sys/kernel/sysrq
echo b > /proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
}
get_ping_size() {
ps=$1
case "$ps" in
small)
ps="1"
;;
windows)
ps="32"
;;
standard)
ps="56"
;;
big)
ps="248"
;;
huge)
ps="1492"
;;
jumbo)
ps="9000"
;;
*)
echo "Error: invalid ping_size. ping_size should be either: small, windows, standard, big, huge or jumbo"
echo "Cooresponding ping packet sizes (bytes): small=1, windows=32, standard=56, big=248, huge=1492, jumbo=9000"
;;
esac
echo $ps
}
watchcat_always() {
local period="$1"; local forcedelay="$2"
reboot_now() {
reboot &
sleep "$period" && reboot_now "$forcedelay"
[ "$1" -ge 1 ] && {
sleep "$1"
echo 1 >/proc/sys/kernel/sysrq
echo b >/proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
}
}
watchcat_periodic() {
failure_period="$1"
force_reboot_delay="$2"
sleep "$failure_period" && reboot_now "$force_reboot_delay"
}
watchcat_restart_modemmanager_iface() {
[ "$2" -gt 0 ] && {
logger -t INFO "Resetting current-bands to 'any' on modem: \"$1\" now."
/usr/bin/mmcli -m any --set-current-bands=any
}
logger -t INFO "Reconnecting modem: \"$1\" now."
/etc/init.d/modemmanager restart
ifup "$1"
}
watchcat_restart_network_iface() {
logger -t INFO "Restarting network interface: \"$1\"."
ip link set "$1" down
ip link set "$1" up
}
watchcat_restart_all_network() {
logger -t INFO "Restarting networking now by running: /etc/init.d/network restart"
/etc/init.d/network restart
}
watchcat_monitor_network() {
failure_period="$1"
ping_hosts="$2"
ping_frequency_interval="$3"
ping_size="$4"
iface="$5"
mm_iface_name="$6"
mm_iface_unlock_bands="$7"
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
[ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_lastcheck="$time_now"
time_lastcheck_withinternet="$time_now"
ping_size="$(get_ping_size "$ping_size")"
while true; do
# account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_diff="$((time_now - time_lastcheck))"
[ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_lastcheck="$time_now"
for host in $ping_hosts; do
if [ "$iface" != "" ]; then
ping_result="$(
ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
echo $?
)"
else
ping_result="$(
ping -s "$ping_size" -c 1 "$host" &>/dev/null
echo $?
)"
fi
if [ "$ping_result" -eq 0 ]; then
time_lastcheck_withinternet="$time_now"
else
if [ "$iface" != "" ]; then
logger -p daemon.info -t "watchcat[$$]" "Could not reach $host via \"$iface\" for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting \"$iface\" after reaching \"$failure_period\" seconds"
else
logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting networking after reaching \"$failure_period\" seconds"
fi
fi
done
[ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && {
if [ "$mm_iface_name" != "" ]; then
watchcat_restart_modemmanager_iface "$mm_iface_name" "$mm_iface_unlock_bands"
fi
if [ "$iface" != "" ]; then
watchcat_restart_network_iface "$iface"
else
watchcat_restart_all_network
fi
/etc/init.d/watchcat start
}
done
}
watchcat_ping() {
local period="$1"; local forcedelay="$2"; local pinghosts="$3"; local pingperiod="$4"; local nopingtime="$5"
failure_period="$1"
force_reboot_delay="$2"
ping_hosts="$3"
ping_frequency_interval="$4"
ping_size="$5"
local time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
[ "$time_now" -lt "$nopingtime" ] && sleep "$((nopingtime-time_now))"
[ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
local time_lastcheck="$time_now"
local time_lastcheck_withinternet="$time_now"
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_lastcheck="$time_now"
time_lastcheck_withinternet="$time_now"
while true
do
# account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
time_now="$(cat /proc/uptime)"; time_now="${time_now%%.*}"
local time_diff="$((time_now-time_lastcheck))"
ping_size="$(get_ping_size "$ping_size")"
[ "$time_diff" -lt "$pingperiod" ] && sleep "$((pingperiod-time_diff))"
while true; do
# account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_diff="$((time_now - time_lastcheck))"
time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
time_lastcheck="$time_now"
[ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
for host in $pinghosts
do
if ping -c 1 "$host" &> /dev/null
then
time_lastcheck_withinternet="$time_now"
else
logger -p daemon.info -t "watchcat[$$]" "no internet connectivity for $((time_now-time_lastcheck_withinternet)). Reseting when reaching $period"
fi
done
[ "$((time_now-time_lastcheck_withinternet))" -ge "$period" ] && reboot_now "$forcedelay"
done
time_now="$(cat /proc/uptime)"
time_now="${time_now%%.*}"
time_lastcheck="$time_now"
for host in $ping_hosts; do
if [ "$iface" != "" ]; then
ping_result="$(
ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
echo $?
)"
else
ping_result="$(
ping -s "$ping_size" -c 1 "$host" &>/dev/null
echo $?
)"
fi
if [ "$ping_result" -eq 0 ]; then
time_lastcheck_withinternet="$time_now"
else
logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for $((time_now - time_lastcheck_withinternet)). Rebooting after reaching $failure_period"
fi
done
[ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && reboot_now "$force_reboot_delay"
done
}
if [ "$1" = "always" ]
then
watchcat_always "$2" "$3"
else
watchcat_ping "$2" "$3" "$4" "$5" "$6"
fi
mode="$1"
# Fix potential typo in mode and provide backward compatibility.
[ "$mode" = "allways" ] && mode="periodic_reboot"
[ "$mode" = "always" ] && mode="periodic_reboot"
[ "$mode" = "ping" ] && mode="ping_reboot"
case "$mode" in
periodic_reboot)
watchcat_periodic "$2" "$3"
;;
ping_reboot)
watchcat_ping "$2" "$3" "$4" "$5" "$6"
;;
restart_iface)
watchcat_monitor_network "$2" "$3" "$4" "$5" "$6" "$7"
;;
*)
echo "Error: invalid mode selected: $mode"
;;
esac