From 72df8e4c66d5ac6cb3777d93d10763b30edb1d60 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Sat, 5 Jan 2019 16:28:44 +0100 Subject: [PATCH] banIP: release 0.1.0 * add automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode * add a 'backup mode' to re-use blocklist backups during startup, get fresh lists via reload or restart action * procd interface trigger now supports multiple WAN interfaces * change URL for abuse.ch/feodo list source in default config * small fixes * update readme Signed-off-by: Dirk Brenken --- net/banip/Makefile | 4 +- net/banip/files/README.md | 30 ++++- net/banip/files/banip.conf | 3 +- net/banip/files/banip.init | 22 ++-- net/banip/files/banip.sh | 247 +++++++++++++++++++++++-------------- 5 files changed, 201 insertions(+), 105 deletions(-) diff --git a/net/banip/Makefile b/net/banip/Makefile index 519f91091f..612825cbcd 100644 --- a/net/banip/Makefile +++ b/net/banip/Makefile @@ -1,12 +1,12 @@ # -# Copyright (c) 2018 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2018-2019 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # include $(TOPDIR)/rules.mk PKG_NAME:=banip -PKG_VERSION:=0.0.7 +PKG_VERSION:=0.1.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/banip/files/README.md b/net/banip/files/README.md index 982a713c7f..1df1f7cdce 100644 --- a/net/banip/files/README.md +++ b/net/banip/files/README.md @@ -23,6 +23,8 @@ IP address blocking is commonly used to protect against brute force attacks, pre * minimal status & error logging to syslog, enable debug logging to receive more output * procd based init system support (start/stop/restart/reload/status) * procd network interface trigger support +* automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode +* 'backup mode' to re-use blocklist backups during startup, get fresh lists via reload or restart action * output comprehensive runtime information via LuCI or via 'status' init command * strong LuCI support * optional: add new banIP sources on your own @@ -43,6 +45,24 @@ IP address blocking is commonly used to protect against brute force attacks, pre * install 'luci-app-banip' (_opkg install luci-app-banip_) * the application is located in LuCI under 'Services' menu +## banIP config options +* usually the pre-configured banIP setup works quite well and no manual overrides are needed +* the following options apply to the 'global' config section: + * ban\_enabled => main switch to enable/disable banIP service (bool/default: '0', disabled) + * ban\_automatic => determine the L2/L3 WAN network device automatically (bool/default: '1', enabled) + * ban\_fetchutil => name of the used download utility: 'uclient-fetch', 'wget', 'curl', 'aria2c', 'wget-nossl'. 'busybox' (default: 'uclient-fetch') + * ban\_iface => space separated list of WAN network interface(s)/device(s) used by banIP (default: automatically set by banIP ('ban_automatic')) + +* the following options apply to the 'extra' config section: + * ban\_debug => enable/disable banIP debug output (default: '0', disabled) + * ban\_nice => set the nice level of the banIP process and all sub-processes (int/default: '0', standard priority) + * ban\_triggerdelay => additional trigger delay in seconds before banIP processing begins (int/default: '2') + * ban\_backup => create compressed blocklist backups, they will be used in case of download errors or during startup in 'backup mode' (bool/default: '0', disabled) + * ban\_backupdir => target directory for adblock backups (default: not set) + * ban\_backupboot => do not automatically update blocklists during startup, use their backups instead (bool/default: '0', disabled) + * ban\_maxqueue => size of the download queue to handle downloads & IPSet processing in parallel (int/default: '8') + * ban\_fetchparm => special config options for the download utility (default: not set) + ## Examples **receive banIP runtime information:** @@ -50,11 +70,11 @@ IP address blocking is commonly used to protect against brute force attacks, pre /etc/init.d/banip status ::: banIP runtime information + status : enabled - + version : 0.0.5 + + version : 0.1.0 + fetch_info : /bin/uclient-fetch (libustream-ssl) - + ipset_info : 3 IPSets with overall 29510 IPs/Prefixes - + last_run : 08.11.2018 15:03:50 - + system : GL-AR750S, OpenWrt SNAPSHOT r8419-860de2e1aa + + ipset_info : 1 IPSets with overall 516 IPs/Prefixes (backup mode) + + last_run : 05.01.2019 14:48:18 + + system : TP-LINK RE450, OpenWrt SNAPSHOT r8910+72-25d8aa7d02 **cronjob for a regular block list update (/etc/crontabs/root):** @@ -65,7 +85,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre ## Support -Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-new-project-needs-testers-feedback/16985) or contact me by mail +Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail ## Removal * stop all banIP related services with _/etc/init.d/banip stop_ diff --git a/net/banip/files/banip.conf b/net/banip/files/banip.conf index d93088dbc0..7de03e8af6 100644 --- a/net/banip/files/banip.conf +++ b/net/banip/files/banip.conf @@ -9,6 +9,7 @@ config banip 'global' config banip 'extra' option ban_debug '0' + option ban_backup '0' option ban_maxqueue '8' config source 'whitelist' @@ -116,7 +117,7 @@ config source 'ransomware' option ban_src_on '0' config source 'feodo' - option ban_src 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist' + option ban_src 'https://feodotracker.abuse.ch/downloads/ipblocklist.txt' option ban_src_desc 'Feodo Tracker by abuse.ch (IPv4)' option ban_src_rset '/^(([0-9]{1,3}\.){3}[0-9]{1,3})([[:space:]]|$)/{print \"add feodo \"\$1}' option ban_src_settype 'ip' diff --git a/net/banip/files/banip.init b/net/banip/files/banip.init index 1fe5f01d47..a0b5836687 100755 --- a/net/banip/files/banip.init +++ b/net/banip/files/banip.init @@ -37,17 +37,22 @@ start_service() fi } +refresh() +{ + rc_procd start_service refresh +} + +reload_service() +{ + rc_procd start_service reload +} + stop_service() { rc_procd "${ban_script}" stop rc_procd start_service } -refresh() -{ - rc_procd start_service "refresh" -} - status() { local key keylist value rtfile="$(uci_get banip global ban_rtfile)" @@ -71,10 +76,13 @@ status() service_triggers() { - local iface="$(uci_get banip global ban_iface)" + local ban_iface="$(uci_get banip global ban_iface)" local delay="$(uci_get banip extra ban_triggerdelay)" PROCD_RELOAD_DELAY=$((${delay:-2} * 1000)) - procd_add_interface_trigger "interface.*.up" "${iface:-"wan"}" "${ban_init}" start + for iface in ${ban_iface:-"wan"} + do + procd_add_interface_trigger "interface.*.up" "${iface}" "${ban_init}" start + done procd_add_reload_trigger "banip" "firewall" } diff --git a/net/banip/files/banip.sh b/net/banip/files/banip.sh index 963439441e..b12aed6b1a 100755 --- a/net/banip/files/banip.sh +++ b/net/banip/files/banip.sh @@ -10,12 +10,15 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -ban_ver="0.0.7" +ban_ver="0.1.0" ban_sysver="unknown" ban_enabled=0 ban_automatic="1" ban_iface="" ban_debug=0 +ban_backup=0 +ban_backupboot=0 +ban_backupdir="/mnt" ban_maxqueue=8 ban_fetchutil="uclient-fetch" ban_ip="$(command -v ip)" @@ -105,6 +108,7 @@ f_envload() then f_jsnup disabled f_ipset destroy + f_rmbackup f_rmtemp f_log "info" "banIP is currently disabled, please set ban_enabled to '1' to use this service" exit 0 @@ -232,6 +236,16 @@ f_rmtemp() > "${ban_pidfile}" } +# remove backup files +# +f_rmbackup() +{ + if [ -d "${ban_backupdir}" ] + then + rm -f "${ban_backupdir}/banIP."*.gz + fi +} + # iptables rules engine # f_iptrule() @@ -326,6 +340,31 @@ f_ipset() fi case "${mode}" in + backup) + ban_rc=4 + if [ -d "${ban_backupdir}" ] + then + gzip -cf "${tmp_load}" 2>/dev/null > "${ban_backupdir}/banIP.${src_name}.gz" + ban_rc=${?} + fi + f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}" + ;; + restore) + ban_rc=4 + if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ] + then + gunzip -cf "${ban_backupdir}/banIP.${src_name}.gz" 2>/dev/null > "${tmp_load}" + ban_rc=${?} + fi + f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}" + ;; + remove) + if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ] + then + rm -f "${ban_backupdir}/banIP.${src_name}.gz" + fi + f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}" + ;; initial) if [ -z "$("${ban_ipt}" "${timeout}" -nL "${ban_chain}" 2>/dev/null)" ] then @@ -373,7 +412,6 @@ f_ipset() printf "%s\n" "${cnt}" > "${tmp_cnt}" fi f_iptadd - end_ts="$(date +%s)" f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, settype: ${src_settype:-"-"}, setipv: "${src_setipv}", ruletype: ${src_ruletype:-"-"}, count(sum/ip/cidr): ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))" ;; @@ -391,7 +429,6 @@ f_ipset() fi f_iptadd fi - end_ts="$(date +%s)" f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, count: ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))" ;; @@ -447,6 +484,7 @@ f_log() then f_jsnup error f_ipset destroy + f_rmbackup f_rmtemp logger -p "${class}" -t "banIP-[${ban_ver}]" "Please also check 'https://github.com/openwrt/packages/blob/master/net/banip/files/README.md'" exit 1 @@ -465,7 +503,7 @@ f_main() mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)" mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)" - f_log "debug" "f_main ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}" + f_log "debug" "f_main ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, backup: ${ban_backup:-"-"}, backup_boot: ${ban_backupboot:-"-"}, backup_dir: ${ban_backupdir:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}" f_ipset initial @@ -515,6 +553,7 @@ f_main() [ -z "${src_settype}" ] || [ -z "${src_ruletype}" ] then f_ipset flush + f_ipset remove continue elif [ "${ban_action}" = "refresh" ] then @@ -526,89 +565,117 @@ f_main() # ( start_ts="$(date +%s)" - if [ -f "${src_url}" ] + if [ ! -f "${src_url}" ] && [ ${ban_backup} -eq 1 ] && [ ${ban_backupboot} -eq 1 ] && [ "${ban_action}" = "start" ] then - src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")" - ban_rc=${?} + f_ipset restore + fi - case "${src_name}" in - whitelist) - src_addon="${ban_subnets}" - ;; - whitelist_6) - src_addon="${ban_subnets6}" - ;; - blacklist) - pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" - for pid in ${pid_list} - do - src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" - done - ;; - blacklist_6) - pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" - for pid in ${pid_list} - do - src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" - done - ;; - esac - - for ip in ${src_addon} - do - if [ -z "$(grep -F "${ip}" "${src_url}")" ] - then - printf '\n%s\n' "${ip}" >> "${tmp_load}" - printf '\n%s\n' "${ip}" >> "${src_url}" - fi - done - elif [ -n "${src_cat}" ] + if [ ${ban_rc} -ne 0 ] || [ ! -s "${tmp_load}" ] then - if [ "${src_cat//[0-9]/}" != "${src_cat}" ] + if [ -f "${src_url}" ] then - for as in ${src_cat} - do - src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)" - ban_rc=${?} - if [ ${ban_rc} -eq 0 ] - then - jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}" - else - break - fi - done - else - for co in ${src_cat} - do - src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)" - ban_rc=${?} - if [ ${ban_rc} -eq 0 ] - then - if [ "${src_name##*_}" = "6" ] - then - jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}" - else - jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}" - fi - else - break - fi - done - fi - else - src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)" - ban_rc=${?} - if [ ${ban_rc} -eq 0 ] - then - zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}" + src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")" ban_rc=${?} - if [ ${ban_rc} -ne 0 ] + case "${src_name}" in + whitelist) + src_addon="${ban_subnets}" + ;; + whitelist_6) + src_addon="${ban_subnets6}" + ;; + blacklist) + pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" + for pid in ${pid_list} + do + src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" + done + ;; + blacklist_6) + pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" + for pid in ${pid_list} + do + src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')" + done + ;; + esac + for ip in ${src_addon} + do + if [ -z "$(grep -F "${ip}" "${src_url}")" ] + then + printf '%s\n' "${ip}" >> "${tmp_load}" + printf '%s\n' "${ip}" >> "${src_url}" + fi + done + elif [ -n "${src_cat}" ] + then + if [ "${src_cat//[0-9]/}" != "${src_cat}" ] then - mv -f "${tmp_raw}" "${tmp_load}" + for as in ${src_cat} + do + src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)" + ban_rc=${?} + if [ ${ban_rc} -eq 0 ] + then + jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}" + else + break + fi + done + if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ] + then + f_ipset backup + elif [ ${ban_backup} -eq 1 ] + then + f_ipset restore + fi + else + for co in ${src_cat} + do + src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)" + ban_rc=${?} + if [ ${ban_rc} -eq 0 ] + then + if [ "${src_name##*_}" = "6" ] + then + jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}" + else + jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}" + fi + else + break + fi + done + if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ] + then + f_ipset backup + elif [ ${ban_backup} -eq 1 ] + then + f_ipset restore + fi + fi + else + src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)" + ban_rc=${?} + if [ ${ban_rc} -eq 0 ] + then + zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}" ban_rc=${?} + if [ ${ban_rc} -ne 0 ] + then + mv -f "${tmp_raw}" "${tmp_load}" + ban_rc=${?} + fi + if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ] + then + f_ipset backup + fi + elif [ ${ban_backup} -eq 1 ] + then + f_ipset restore fi fi fi + if [ ${ban_rc} -eq 0 ] then awk "${src_rset}" "${tmp_load}" 2>/dev/null | sort -u > "${tmp_file}" @@ -655,16 +722,26 @@ f_main() # f_jsnup() { - local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" status="${1:-"enabled"}" + local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" mode="normal mode" status="${1:-"enabled"}" ban_cntinfo="${ban_setcnt} IPSets with overall ${ban_cnt} IPs/Prefixes" + if [ ${ban_backupboot} -eq 1 ] + then + mode="backup mode" + fi + + > "${ban_rtfile}" + json_load_file "${ban_rtfile}" >/dev/null 2>&1 + json_init + json_add_object "data" json_add_string "status" "${status}" json_add_string "version" "${ban_ver}" json_add_string "fetch_info" "${ban_fetchinfo:-"-"}" - json_add_string "ipset_info" "${ban_cntinfo:-"-"}" + json_add_string "ipset_info" "${ban_cntinfo:-"-"} (${mode})" json_add_string "last_run" "${rundate:-"-"}" json_add_string "system" "${ban_sysver}" + json_close_object json_dump > "${ban_rtfile}" f_log "debug" "f_jsnup ::: status: ${status}, setcnt: ${ban_setcnt}, cnt: ${ban_cnt}" @@ -681,17 +758,6 @@ else f_log "err" "system libraries not found" fi -# initialize json runtime file -# -json_load_file "${ban_rtfile}" >/dev/null 2>&1 -json_select data >/dev/null 2>&1 -if [ ${?} -ne 0 ] -then - > "${ban_rtfile}" - json_init - json_add_object "data" -fi - # handle different banIP actions # f_envload @@ -699,6 +765,7 @@ case "${ban_action}" in stop) f_jsnup stopped f_ipset destroy + f_rmbackup f_rmtemp ;; start|restart|reload|refresh)