banIP: release 0.1.0

* add automatic blocklist backup & restore, they will be used
  in case of download errors or during startup in backup mode
* add a 'backup mode' to re-use blocklist backups during startup,
  get fresh lists via reload or restart action
* procd interface trigger now supports multiple WAN interfaces
* change URL for abuse.ch/feodo list source in default config
* small fixes
* update readme

Signed-off-by: Dirk Brenken <dev@brenken.org>
This commit is contained in:
Dirk Brenken 2019-01-05 16:28:44 +01:00
parent f56d487897
commit 72df8e4c66
5 changed files with 201 additions and 105 deletions

View File

@ -1,12 +1,12 @@
#
# Copyright (c) 2018 Dirk Brenken (dev@brenken.org)
# Copyright (c) 2018-2019 Dirk Brenken (dev@brenken.org)
# This is free software, licensed under the GNU General Public License v3.
#
include $(TOPDIR)/rules.mk
PKG_NAME:=banip
PKG_VERSION:=0.0.7
PKG_VERSION:=0.1.0
PKG_RELEASE:=1
PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>

View File

@ -23,6 +23,8 @@ IP address blocking is commonly used to protect against brute force attacks, pre
* minimal status & error logging to syslog, enable debug logging to receive more output
* procd based init system support (start/stop/restart/reload/status)
* procd network interface trigger support
* automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode
* 'backup mode' to re-use blocklist backups during startup, get fresh lists via reload or restart action
* output comprehensive runtime information via LuCI or via 'status' init command
* strong LuCI support
* optional: add new banIP sources on your own
@ -43,6 +45,24 @@ IP address blocking is commonly used to protect against brute force attacks, pre
* install 'luci-app-banip' (_opkg install luci-app-banip_)
* the application is located in LuCI under 'Services' menu
## banIP config options
* usually the pre-configured banIP setup works quite well and no manual overrides are needed
* the following options apply to the 'global' config section:
* ban\_enabled => main switch to enable/disable banIP service (bool/default: '0', disabled)
* ban\_automatic => determine the L2/L3 WAN network device automatically (bool/default: '1', enabled)
* ban\_fetchutil => name of the used download utility: 'uclient-fetch', 'wget', 'curl', 'aria2c', 'wget-nossl'. 'busybox' (default: 'uclient-fetch')
* ban\_iface => space separated list of WAN network interface(s)/device(s) used by banIP (default: automatically set by banIP ('ban_automatic'))
* the following options apply to the 'extra' config section:
* ban\_debug => enable/disable banIP debug output (default: '0', disabled)
* ban\_nice => set the nice level of the banIP process and all sub-processes (int/default: '0', standard priority)
* ban\_triggerdelay => additional trigger delay in seconds before banIP processing begins (int/default: '2')
* ban\_backup => create compressed blocklist backups, they will be used in case of download errors or during startup in 'backup mode' (bool/default: '0', disabled)
* ban\_backupdir => target directory for adblock backups (default: not set)
* ban\_backupboot => do not automatically update blocklists during startup, use their backups instead (bool/default: '0', disabled)
* ban\_maxqueue => size of the download queue to handle downloads & IPSet processing in parallel (int/default: '8')
* ban\_fetchparm => special config options for the download utility (default: not set)
## Examples
**receive banIP runtime information:**
@ -50,11 +70,11 @@ IP address blocking is commonly used to protect against brute force attacks, pre
/etc/init.d/banip status
::: banIP runtime information
+ status : enabled
+ version : 0.0.5
+ version : 0.1.0
+ fetch_info : /bin/uclient-fetch (libustream-ssl)
+ ipset_info : 3 IPSets with overall 29510 IPs/Prefixes
+ last_run : 08.11.2018 15:03:50
+ system : GL-AR750S, OpenWrt SNAPSHOT r8419-860de2e1aa
+ ipset_info : 1 IPSets with overall 516 IPs/Prefixes (backup mode)
+ last_run : 05.01.2019 14:48:18
+ system : TP-LINK RE450, OpenWrt SNAPSHOT r8910+72-25d8aa7d02
</code></pre>
**cronjob for a regular block list update (/etc/crontabs/root):**
@ -65,7 +85,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
## Support
Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-new-project-needs-testers-feedback/16985) or contact me by mail <dev@brenken.org>
Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail <dev@brenken.org>
## Removal
* stop all banIP related services with _/etc/init.d/banip stop_

View File

@ -9,6 +9,7 @@ config banip 'global'
config banip 'extra'
option ban_debug '0'
option ban_backup '0'
option ban_maxqueue '8'
config source 'whitelist'
@ -116,7 +117,7 @@ config source 'ransomware'
option ban_src_on '0'
config source 'feodo'
option ban_src 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist'
option ban_src 'https://feodotracker.abuse.ch/downloads/ipblocklist.txt'
option ban_src_desc 'Feodo Tracker by abuse.ch (IPv4)'
option ban_src_rset '/^(([0-9]{1,3}\.){3}[0-9]{1,3})([[:space:]]|$)/{print \"add feodo \"\$1}'
option ban_src_settype 'ip'

View File

@ -37,17 +37,22 @@ start_service()
fi
}
refresh()
{
rc_procd start_service refresh
}
reload_service()
{
rc_procd start_service reload
}
stop_service()
{
rc_procd "${ban_script}" stop
rc_procd start_service
}
refresh()
{
rc_procd start_service "refresh"
}
status()
{
local key keylist value rtfile="$(uci_get banip global ban_rtfile)"
@ -71,10 +76,13 @@ status()
service_triggers()
{
local iface="$(uci_get banip global ban_iface)"
local ban_iface="$(uci_get banip global ban_iface)"
local delay="$(uci_get banip extra ban_triggerdelay)"
PROCD_RELOAD_DELAY=$((${delay:-2} * 1000))
procd_add_interface_trigger "interface.*.up" "${iface:-"wan"}" "${ban_init}" start
for iface in ${ban_iface:-"wan"}
do
procd_add_interface_trigger "interface.*.up" "${iface}" "${ban_init}" start
done
procd_add_reload_trigger "banip" "firewall"
}

View File

@ -10,12 +10,15 @@
#
LC_ALL=C
PATH="/usr/sbin:/usr/bin:/sbin:/bin"
ban_ver="0.0.7"
ban_ver="0.1.0"
ban_sysver="unknown"
ban_enabled=0
ban_automatic="1"
ban_iface=""
ban_debug=0
ban_backup=0
ban_backupboot=0
ban_backupdir="/mnt"
ban_maxqueue=8
ban_fetchutil="uclient-fetch"
ban_ip="$(command -v ip)"
@ -105,6 +108,7 @@ f_envload()
then
f_jsnup disabled
f_ipset destroy
f_rmbackup
f_rmtemp
f_log "info" "banIP is currently disabled, please set ban_enabled to '1' to use this service"
exit 0
@ -232,6 +236,16 @@ f_rmtemp()
> "${ban_pidfile}"
}
# remove backup files
#
f_rmbackup()
{
if [ -d "${ban_backupdir}" ]
then
rm -f "${ban_backupdir}/banIP."*.gz
fi
}
# iptables rules engine
#
f_iptrule()
@ -326,6 +340,31 @@ f_ipset()
fi
case "${mode}" in
backup)
ban_rc=4
if [ -d "${ban_backupdir}" ]
then
gzip -cf "${tmp_load}" 2>/dev/null > "${ban_backupdir}/banIP.${src_name}.gz"
ban_rc=${?}
fi
f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}"
;;
restore)
ban_rc=4
if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ]
then
gunzip -cf "${ban_backupdir}/banIP.${src_name}.gz" 2>/dev/null > "${tmp_load}"
ban_rc=${?}
fi
f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, rc: ${ban_rc}"
;;
remove)
if [ -d "${ban_backupdir}" ] && [ -f "${ban_backupdir}/banIP.${src_name}.gz" ]
then
rm -f "${ban_backupdir}/banIP.${src_name}.gz"
fi
f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}"
;;
initial)
if [ -z "$("${ban_ipt}" "${timeout}" -nL "${ban_chain}" 2>/dev/null)" ]
then
@ -373,7 +412,6 @@ f_ipset()
printf "%s\n" "${cnt}" > "${tmp_cnt}"
fi
f_iptadd
end_ts="$(date +%s)"
f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, settype: ${src_settype:-"-"}, setipv: "${src_setipv}", ruletype: ${src_ruletype:-"-"}, count(sum/ip/cidr): ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))"
;;
@ -391,7 +429,6 @@ f_ipset()
fi
f_iptadd
fi
end_ts="$(date +%s)"
f_log "debug" "f_ipset ::: name: ${src_name:-"-"}, mode: ${mode:-"-"}, count: ${cnt:-0}/${cnt_ip:-0}/${cnt_cidr:-0}, time(s): $(( end_ts - start_ts ))"
;;
@ -447,6 +484,7 @@ f_log()
then
f_jsnup error
f_ipset destroy
f_rmbackup
f_rmtemp
logger -p "${class}" -t "banIP-[${ban_ver}]" "Please also check 'https://github.com/openwrt/packages/blob/master/net/banip/files/README.md'"
exit 1
@ -465,7 +503,7 @@ f_main()
mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
f_log "debug" "f_main ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}"
f_log "debug" "f_main ::: fetch_util: ${ban_fetchinfo:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}, interface(s): ${ban_iface:-"-"}, device(s): ${ban_dev:-"-"}, all_devices: ${ban_dev_all:-"-"}, backup: ${ban_backup:-"-"}, backup_boot: ${ban_backupboot:-"-"}, backup_dir: ${ban_backupdir:-"-"}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${ban_maxqueue}"
f_ipset initial
@ -515,6 +553,7 @@ f_main()
[ -z "${src_settype}" ] || [ -z "${src_ruletype}" ]
then
f_ipset flush
f_ipset remove
continue
elif [ "${ban_action}" = "refresh" ]
then
@ -526,89 +565,117 @@ f_main()
#
(
start_ts="$(date +%s)"
if [ -f "${src_url}" ]
if [ ! -f "${src_url}" ] && [ ${ban_backup} -eq 1 ] && [ ${ban_backupboot} -eq 1 ] && [ "${ban_action}" = "start" ]
then
src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")"
ban_rc=${?}
f_ipset restore
fi
case "${src_name}" in
whitelist)
src_addon="${ban_subnets}"
;;
whitelist_6)
src_addon="${ban_subnets6}"
;;
blacklist)
pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
for pid in ${pid_list}
do
src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
done
;;
blacklist_6)
pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
for pid in ${pid_list}
do
src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
done
;;
esac
for ip in ${src_addon}
do
if [ -z "$(grep -F "${ip}" "${src_url}")" ]
then
printf '\n%s\n' "${ip}" >> "${tmp_load}"
printf '\n%s\n' "${ip}" >> "${src_url}"
fi
done
elif [ -n "${src_cat}" ]
if [ ${ban_rc} -ne 0 ] || [ ! -s "${tmp_load}" ]
then
if [ "${src_cat//[0-9]/}" != "${src_cat}" ]
if [ -f "${src_url}" ]
then
for as in ${src_cat}
do
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}"
else
break
fi
done
else
for co in ${src_cat}
do
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
if [ "${src_name##*_}" = "6" ]
then
jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}"
else
jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}"
fi
else
break
fi
done
fi
else
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}"
src_log="$(cat "${src_url}" 2>/dev/null > "${tmp_load}")"
ban_rc=${?}
if [ ${ban_rc} -ne 0 ]
case "${src_name}" in
whitelist)
src_addon="${ban_subnets}"
;;
whitelist_6)
src_addon="${ban_subnets6}"
;;
blacklist)
pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
for pid in ${pid_list}
do
src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9]{1,3}\.){3}[0-9]{1,3}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
done
;;
blacklist_6)
pid_list="$(printf "%s\n" "${log_content}" | grep -F "Exit before auth" | awk 'match($0,/(\[[0-9]+\])/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
for pid in ${pid_list}
do
src_addon="${src_addon} $(printf "%s\n" "${log_content}" | grep -F "${pid}" | awk 'match($0,/([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}/){ORS=" ";print substr($0,RSTART,RLENGTH)}')"
done
;;
esac
for ip in ${src_addon}
do
if [ -z "$(grep -F "${ip}" "${src_url}")" ]
then
printf '%s\n' "${ip}" >> "${tmp_load}"
printf '%s\n' "${ip}" >> "${src_url}"
fi
done
elif [ -n "${src_cat}" ]
then
if [ "${src_cat//[0-9]/}" != "${src_cat}" ]
then
mv -f "${tmp_raw}" "${tmp_load}"
for as in ${src_cat}
do
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}AS${as}" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
jsonfilter -i "${tmp_raw}" -e '@.data.prefixes.*.prefix' 2>/dev/null >> "${tmp_load}"
else
break
fi
done
if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
then
f_ipset backup
elif [ ${ban_backup} -eq 1 ]
then
f_ipset restore
fi
else
for co in ${src_cat}
do
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}${co}&v4_format=prefix" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
if [ "${src_name##*_}" = "6" ]
then
jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv6.*' 2>/dev/null >> "${tmp_load}"
else
jsonfilter -i "${tmp_raw}" -e '@.data.resources.ipv4.*' 2>/dev/null >> "${tmp_load}"
fi
else
break
fi
done
if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
then
f_ipset backup
elif [ ${ban_backup} -eq 1 ]
then
f_ipset restore
fi
fi
else
src_log="$("${ban_fetchutil}" ${ban_fetchparm} "${tmp_raw}" "${src_url}" 2>&1)"
ban_rc=${?}
if [ ${ban_rc} -eq 0 ]
then
zcat "${tmp_raw}" 2>/dev/null > "${tmp_load}"
ban_rc=${?}
if [ ${ban_rc} -ne 0 ]
then
mv -f "${tmp_raw}" "${tmp_load}"
ban_rc=${?}
fi
if [ ${ban_rc} -eq 0 ] && [ ${ban_backup} -eq 1 ]
then
f_ipset backup
fi
elif [ ${ban_backup} -eq 1 ]
then
f_ipset restore
fi
fi
fi
if [ ${ban_rc} -eq 0 ]
then
awk "${src_rset}" "${tmp_load}" 2>/dev/null | sort -u > "${tmp_file}"
@ -655,16 +722,26 @@ f_main()
#
f_jsnup()
{
local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" status="${1:-"enabled"}"
local rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" mode="normal mode" status="${1:-"enabled"}"
ban_cntinfo="${ban_setcnt} IPSets with overall ${ban_cnt} IPs/Prefixes"
if [ ${ban_backupboot} -eq 1 ]
then
mode="backup mode"
fi
> "${ban_rtfile}"
json_load_file "${ban_rtfile}" >/dev/null 2>&1
json_init
json_add_object "data"
json_add_string "status" "${status}"
json_add_string "version" "${ban_ver}"
json_add_string "fetch_info" "${ban_fetchinfo:-"-"}"
json_add_string "ipset_info" "${ban_cntinfo:-"-"}"
json_add_string "ipset_info" "${ban_cntinfo:-"-"} (${mode})"
json_add_string "last_run" "${rundate:-"-"}"
json_add_string "system" "${ban_sysver}"
json_close_object
json_dump > "${ban_rtfile}"
f_log "debug" "f_jsnup ::: status: ${status}, setcnt: ${ban_setcnt}, cnt: ${ban_cnt}"
@ -681,17 +758,6 @@ else
f_log "err" "system libraries not found"
fi
# initialize json runtime file
#
json_load_file "${ban_rtfile}" >/dev/null 2>&1
json_select data >/dev/null 2>&1
if [ ${?} -ne 0 ]
then
> "${ban_rtfile}"
json_init
json_add_object "data"
fi
# handle different banIP actions
#
f_envload
@ -699,6 +765,7 @@ case "${ban_action}" in
stop)
f_jsnup stopped
f_ipset destroy
f_rmbackup
f_rmtemp
;;
start|restart|reload|refresh)