From c07fae25e70a159b48154fadcb3fcb3fd4f73ef1 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Thu, 6 Apr 2023 19:37:28 +0200 Subject: [PATCH] banip: release 0.8.3-1 * add the new init command 'lookup', to lookup the IPs of domain names in the local lists and update them * significant acceleration of the domain lookup function * multiple small fixes and improvements * readme update * luci update (separate commit) Signed-off-by: Dirk Brenken --- net/banip/Makefile | 4 +-- net/banip/files/README.md | 32 ++++++++++------- net/banip/files/banip-functions.sh | 58 +++++++++++++++--------------- net/banip/files/banip-service.sh | 26 ++++++++------ net/banip/files/banip.init | 29 ++++++++++----- net/banip/files/banip.tpl | 14 +++++--- 6 files changed, 96 insertions(+), 67 deletions(-) diff --git a/net/banip/Makefile b/net/banip/Makefile index 65fa9e2703..d2e54a2ad1 100644 --- a/net/banip/Makefile +++ b/net/banip/Makefile @@ -7,8 +7,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=banip -PKG_VERSION:=0.8.2 -PKG_RELEASE:=6 +PKG_VERSION:=0.8.3 +PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/net/banip/files/README.md b/net/banip/files/README.md index f4502816bd..f4116889b7 100644 --- a/net/banip/files/README.md +++ b/net/banip/files/README.md @@ -75,7 +75,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre * Provides a set search engine for certain IPs * Feed parsing by fast & flexible regex rulesets * Minimal status & error logging to syslog, enable debug logging to receive more output -* Procd based init system support (start/stop/restart/reload/status/report/search/survey) +* Procd based init system support (start/stop/restart/reload/status/report/search/survey/lookup) * Procd network interface trigger support * Ability to add new banIP feeds on your own @@ -114,6 +114,7 @@ Available commands: report [text|json|mail] Print banIP related set statistics search [|] Check if an element exists in a banIP set survey [] List all elements of a given banIP set + lookup Lookup the IPs of domain names in the local lists and update them running Check if service is running status Service status trace Start with syscall trace @@ -226,18 +227,16 @@ Available commands: ~# /etc/init.d/banip status ::: banIP runtime information + status : active (nft: ✔, monitor: ✔) - + version : 0.8.2-2 - + element_count : 211397 - + active_feeds : allowlistvMAC, allowlistv4, allowlistv6, adawayv4, adawayv6, adguardv4, adguardtrackersv4, adguardv6, adguardtrackersv - 6, antipopadsv4, antipopadsv6, cinsscorev4, countryv6, countryv4, deblv4, deblv6, dohv4, dohv6, firehol1v4, oisdsmallv - 6, oisdsmallv4, stevenblackv6, stevenblackv4, webclientv4, blocklistvMAC, blocklistv4, blocklistv6 - + active_devices : eth2 ::: wan, wan6 - + active_subnets : 91.64.148.211/24, 2b02:710c:0:80:e442:4b0c:637d:1d33/128 + + version : 0.8.3-1 + + element_count : 281161 + + active_feeds : allowlistvMAC, allowlistv6, allowlistv4, adawayv4, adguardtrackersv4, adawayv6, adguardv6, adguardv4, adguardtrackersv6, antipopadsv6, antipopadsv4, cinsscorev4, deblv4, countryv6, countryv4, deblv6, dohv4, dohv6, iblockadsv4, firehol1v4, oisdbigv4, yoyov6, threatviewv4, yoyov4, oisdbigv6, blocklistvMAC, blocklistv4, blocklistv6 + + active_devices : br-wan ::: wan, wan6 + + active_subnets : 91.64.169.252/24, 2a02:710c:0:60:958b:3bd0:9e14:abb/128 + nft_info : priority: -200, policy: memory, loglevel: warn, expiry: - + run_info : base: /mnt/data/banIP, backup: /mnt/data/banIP/backup, report: /mnt/data/banIP/report, feed: /etc/banip/banip.feeds + run_flags : auto: ✔, proto (4/6): ✔/✔, log (wan-inp/wan-fwd/lan-fwd): ✔/✔/✔, dedup: ✔, split: ✘, allowed only: ✘ - + last_run : action: restart, duration: 0m 55s, date: 2023-03-10 19:33:08 - + system_info : cores: 2, memory: 1830, device: Turris Omnia, OpenWrt SNAPSHOT r22248-bf055fcdca + + last_run : action: reload, duration: 1m 0s, date: 2023-04-06 12:34:10 + + system_info : cores: 4, memory: 1822, device: Bananapi BPI-R3, OpenWrt SNAPSHOT r22498-75f7e2d10b ``` **banIP search information** @@ -288,15 +287,22 @@ list ban_logterm 'SecurityEvent=\"InvalidAccountID\".*RemoteAddress=' **allow-/blocklist handling** banIP supports local allow and block lists (IPv4, IPv6, CIDR notation or domain names), located in /etc/banip/banip.allowlist and /etc/banip/banip.blocklist. Unsuccessful login attempts or suspicious requests will be tracked and added to the local blocklist (see the 'ban\_autoblocklist' option). The blocklist behaviour can be further tweaked with the 'ban\_nftexpiry' option. -Furthermore the uplink subnet will be added to local allowlist (see 'ban\_autowallowlist' option). -Both lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted in a detached background process and added to the sets. +Furthermore the uplink subnet will be added to local allowlist (see 'ban\_autoallowlist' option). +Both lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted and added to the sets. You can also start the domain lookup separately via /etc/init.d/banip lookup at any time. **allowlist-only mode** banIP supports an "allowlist only" mode. This option restricts the internet access from/to a small number of secure websites/IPs, and block access from/to the rest of the internet. All IPs and Domains which are _not_ listed in the allowlist are blocked. -**redirect Asterisk security logs to lodg/logread** +**redirect Asterisk security logs to lodg/logread** banIP only supports logfile scanning via logread, so to monitor attacks on Asterisk, its security log must be available via logread. To do this, edit '/etc/asterisk/logger.conf' and add the line 'syslog.local0 = security', then run 'asterisk -rx reload logger' to update the running Asterisk configuration. +**send status E-Mails and update the banIP lists via cron job** +For a regular, automatic status mailing and update of the used lists on a daily basis set up a cron job, e.g. +``` +55 03 * * * /etc/init.d/banip report mail +00 04 * * * /etc/init.d/banip reload +``` + **tweaks for low memory systems** nftables supports the atomic loading of rules/sets/members, which is cool but unfortunately is also very memory intensive. To reduce the memory pressure on low memory systems (i.e. those with 256-512Mb RAM), you should optimize your configuration with the following options: diff --git a/net/banip/files/banip-functions.sh b/net/banip/files/banip-functions.sh index a5049aa8bf..991d1147e8 100644 --- a/net/banip/files/banip-functions.sh +++ b/net/banip/files/banip-functions.sh @@ -78,6 +78,7 @@ ban_debug="0" f_system() { local cpu core + [ -z "${ban_dev}" ] && ban_cores="$(uci_get banip global ban_cores)" ban_memory="$("${ban_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>/dev/null)" ban_ver="$(${ban_ubuscmd} -S call rpc-sys packagelist '{ "all": true }' 2>/dev/null | jsonfilter -ql1 -e '@.packages.banip')" ban_sysver="$(${ban_ubuscmd} -S call system board 2>/dev/null | jsonfilter -ql1 -e '@.model' -e '@.release.description' | @@ -426,7 +427,7 @@ f_getsub() { f_getelements() { local file="${1}" - [ -s "${file}" ] && printf "%s" "elements={ $(cat "${file}") };" + [ -s "${file}" ] && printf "%s" "elements={ $(cat "${file}" 2>/dev/null) };" } # build initial nft file with base table, chains and rules @@ -975,8 +976,6 @@ f_getstatus() { done json_select ".." fi - value="$(printf "%s" "${value}" | - awk '{NR=1;max=118;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-24s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" printf " + %-17s : %s\n" "${key}" "${value:-"-"}" done else @@ -987,7 +986,7 @@ f_getstatus() { # domain lookup # f_lookup() { - local cnt list domain lookup ip start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" + local cnt list domain lookup ip elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" start_time="$(date "+%s")" if [ "${feed}" = "allowlist" ]; then @@ -1004,32 +1003,36 @@ f_lookup() { else if { [ "${feed}" = "allowlist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_allowlist}"; } || { [ "${feed}" = "blocklist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_blocklist}"; }; then - cnt_ip="$((cnt_ip + 1))" if [ "${ip##*:}" = "${ip}" ]; then - if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${ip} }" >/dev/null 2>&1; then - f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v4 set" - continue - fi + elementsv4="${elementsv4} ${ip}," else - if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${ip} }" >/dev/null 2>&1; then - f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v6 set" - continue - fi + elementsv6="${elementsv6} ${ip}," fi if [ "${feed}" = "allowlist" ] && [ "${ban_autoallowlist}" = "1" ]; then printf "%-42s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}" elif [ "${feed}" = "blocklist" ] && [ "${ban_autoblocklist}" = "1" ]; then printf "%-42s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_blocklist}" fi + cnt_ip="$((cnt_ip + 1))" fi fi done cnt_domain="$((cnt_domain + 1))" done + if [ -n "${elementsv4}" ]; then + if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${elementsv4} }" >/dev/null 2>&1; then + f_log "info" "failed to add lookup file to ${feed}v4 set" + fi + fi + if [ -n "${elementsv6}" ]; then + if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${elementsv6} }" >/dev/null 2>&1; then + f_log "info" "failed to add lookup file to ${feed}v6 set" + fi + fi end_time="$(date "+%s")" duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s" - f_log "debug" "f_lookup ::: name: ${feed}, cnt_domain: ${cnt_domain}, cnt_ip: ${cnt_ip}, duration: ${duration}" + f_log "info" "Lookup summary for the local ${feed}: Domains processed: ${cnt_domain}, IPs added: ${cnt_ip}, Duration: ${duration}" } # table statistics @@ -1198,7 +1201,7 @@ f_report() { # set search # f_search() { - local table_sets ip proto run_search search="${1}" + local set table_sets ip proto run_search hold cnt search="${1}" if [ -n "${search}" ]; then ip="$(printf "%s" "${search}" | "${ban_awkcmd}" 'BEGIN{RS="(([0-9]{1,3}\\.){3}[0-9]{1,3})+"}{printf "%s",RT}')" @@ -1215,14 +1218,15 @@ f_search() { return fi printf "%s\n%s\n%s\n" ":::" "::: banIP Search" ":::" - printf "%s\n" " Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")" - printf "%s\n" " ---" + printf " %s\n" "Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")" + printf " %s\n" "---" cnt="1" run_search="/var/run/banIP.search" for set in ${table_sets}; do + [ -f "${run_search}" ] && break ( if "${ban_nftcmd}" get element inet banIP "${set}" "{ ${ip} }" >/dev/null 2>&1; then - printf "%s\n" " IP found in Set '${set}'" + printf " %s\n" "IP found in Set '${set}'" : >"${run_search}" fi ) & @@ -1231,11 +1235,8 @@ f_search() { cnt="$((cnt + 1))" done wait - if [ ! -f "${run_search}" ]; then - printf "%s\n" " IP not found" - else - rm -f "${run_search}" - fi + [ ! -f "${run_search}" ] && printf " %s\n" "IP not found" + rm -f "${run_search}" } # set survey @@ -1243,16 +1244,15 @@ f_search() { f_survey() { local set_elements set="${1}" - [ -n "${set}" ] && set_elements="$("${ban_nftcmd}" -j list set inet banIP "${set}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]')" - - if [ -z "${set}" ] || [ -z "${set_elements}" ]; then + if [ -z "${set}" ]; then printf "%s\n%s\n%s\n" ":::" "::: no valid survey input" ":::" return fi + [ -n "${set}" ] && set_elements="$("${ban_nftcmd}" -j list set inet banIP "${set}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]')" printf "%s\n%s\n%s\n" ":::" "::: banIP Survey" ":::" - printf "%s\n" " List the elements of Set '${set}' on $(date "+%Y-%m-%d %H:%M:%S")" - printf "%s\n" " ---" - printf "%s\n" "${set_elements}" + printf " %s\n" "List the elements of Set '${set}' on $(date "+%Y-%m-%d %H:%M:%S")" + printf " %s\n" "---" + [ -n "${set_elements}" ] && printf "%s\n" "${set_elements}" || printf " %s\n" "empty set" } # send status mails diff --git a/net/banip/files/banip-service.sh b/net/banip/files/banip-service.sh index e753a88107..10f0b97478 100755 --- a/net/banip/files/banip-service.sh +++ b/net/banip/files/banip-service.sh @@ -124,21 +124,25 @@ for feed in allowlist ${ban_feed} blocklist; do fi done wait - -# start background domain lookup -# -f_log "info" "start detached banIP domain lookup" -(f_lookup "allowlist") & -hold="$((cnt % ban_cores))" -[ "${hold}" = "0" ] && wait -(f_lookup "blocklist") & - -# end processing -# f_rmset f_rmdir "${ban_tmpdir}" f_genstatus "active" f_log "info" "finished banIP download processes" + +# start domain lookup +# +f_log "info" "start banIP domain lookup" +cnt="1" +for list in allowlist blocklist; do + (f_lookup "${list}") & + hold="$((cnt % ban_cores))" + [ "${hold}" = "0" ] && wait + cnt="$((cnt + 1))" +done +wait + +# end processing +# if [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ]; then ( sleep ${ban_triggerdelay} diff --git a/net/banip/files/banip.init b/net/banip/files/banip.init index d8faafdaa0..6822a1bcb2 100755 --- a/net/banip/files/banip.init +++ b/net/banip/files/banip.init @@ -12,6 +12,7 @@ USE_PROCD=1 extra_command "report" "[text|json|mail] Print banIP related set statistics" extra_command "search" "[|] Check if an element exists in a banIP set" extra_command "survey" "[] List all elements of a given banIP set" +extra_command "lookup" "Lookup the IPs of domain names in the local lists and update them" ban_init="/etc/init.d/banip" ban_service="/usr/bin/banip-service.sh" @@ -20,10 +21,10 @@ ban_pidfile="/var/run/banip.pid" ban_lock="/var/run/banip.lock" [ "${action}" = "boot" ] && /etc/init.d/banip running && exit 0 -[ "${action}" = "stop" ] && ! /etc/init.d/banip running && exit 0 -[ ! -r "${ban_funlib}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "stop" ] || [ "${action}" = "report" ] || [ "${action}" = "search" ] || [ "${action}" = "survey" ] || [ "${action}" = "status" ]; } && exit 1 -[ -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ]; } && exit 1 -[ ! -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ]; } && mkdir -p "${ban_lock}" +{ [ "${action}" = "stop" ] || [ "${action}" = "lookup" ]; } && ! /etc/init.d/banip running && exit 0 +[ ! -r "${ban_funlib}" ] && [ "${action}" != "boot" ] && exit 1 +[ -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ]; } && exit 1 +[ ! -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ]; } && mkdir -p "${ban_lock}" boot() { : >"${ban_pidfile}" @@ -61,7 +62,7 @@ stop_service() { "${ban_nftcmd}" delete table inet banIP >/dev/null 2>&1 f_genstatus "stopped" f_rmpid - rm -rf "${ban_lock}" + [ "${action}" = "stop" ] && rm -rf "${ban_lock}" } restart() { @@ -74,10 +75,8 @@ status() { } status_service() { - local actual="${1}" - [ -z "$(command -v "f_system")" ] && . "${ban_funlib}" - [ -n "${actual}" ] && f_actual || f_getstatus + f_getstatus } report() { @@ -95,6 +94,20 @@ survey() { f_survey "${1}" } +lookup() { + local list hold cnt="1" + + [ -z "$(command -v "f_system")" ] && . "${ban_funlib}" + for list in allowlist blocklist; do + (f_lookup "${list}") & + hold="$((cnt % ban_cores))" + [ "${hold}" = "0" ] && wait + cnt="$((cnt + 1))" + done + wait + rm -rf "${ban_lock}" +} + service_triggers() { local iface trigger trigger_action delay diff --git a/net/banip/files/banip.tpl b/net/banip/files/banip.tpl index 0474eb3449..f6bd5214c2 100644 --- a/net/banip/files/banip.tpl +++ b/net/banip/files/banip.tpl @@ -6,7 +6,7 @@ # local banip_info report_info log_info system_info mail_text -banip_info="$(/etc/init.d/banip status 2>/dev/null)" +banip_info="$(/etc/init.d/banip status 2>/dev/null | awk '{NR=1;max=140;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" report_info="$(cat ${ban_reportdir}/ban_report.txt 2>/dev/null)" log_info="$("${ban_logreadcmd}" -l 100 -e "banIP/" 2>/dev/null | awk '{NR=1;max=140;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" system_info="$( @@ -14,11 +14,17 @@ system_info="$( ubus call system board | awk 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' )" -# mail body +# content header # mail_text="$(printf "%s\n" "
")"
+
+# content body
+#
 mail_text="$(printf "%s\n" "${mail_text}\n++\n++ System Information ++\n++\n${system_info:-"-"}")"
 mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ banIP Status ++\n++\n${banip_info:-"-"}")"
-mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ banIP Report ++\n++\n${report_info:-"-"}")"
-mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ Logfile Information ++\n++\n${log_info}")"
+[ -n "${report_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ banIP Report ++\n++\n${report_info}")"
+[ -n "${log_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n++\n++ Logfile Information ++\n++\n${log_info}")"
+
+# content footer
+#
 mail_text="$(printf "%s\n" "${mail_text}
")"