From 909a87c2f59ce49b4006383aa3a8bcb7e8b7039f Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Mon, 15 Mar 2021 20:38:46 +0100 Subject: [PATCH] banip: update to 0.7.5-2 * refine the new dns resolving process * add a caching mechanism for the resolved IPs, the detached name lookup takes place only during 'restart' or 'reload' action, 'start' and 'refresh' actions are using an auto-generated backup instead. * update the readme Signed-off-by: Dirk Brenken --- net/banip/Makefile | 2 +- net/banip/files/README.md | 68 +++++++++++++++------------ net/banip/files/banip.dns | 99 ++++++++++++++++++++++++++++----------- net/banip/files/banip.sh | 15 +++--- 4 files changed, 118 insertions(+), 66 deletions(-) diff --git a/net/banip/Makefile b/net/banip/Makefile index a43b67d960..8540356c51 100644 --- a/net/banip/Makefile +++ b/net/banip/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=banip PKG_VERSION:=0.7.5 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/net/banip/files/README.md b/net/banip/files/README.md index 83fbf40811..c9aadb32f7 100644 --- a/net/banip/files/README.md +++ b/net/banip/files/README.md @@ -40,39 +40,40 @@ IP address blocking is commonly used to protect against brute force attacks, pre | yoyo | Ad protection blacklist | [Link](https://pgl.yoyo.org/adservers/) | * zero-conf like automatic installation & setup, usually no manual changes needed -* automatically selects one of the following download utilities: aria2c, curl, uclient-fetch, wget -* Really fast downloads & list processing as they are handled in parallel as background jobs in a configurable 'Download Queue' +* automatically selects one of the following supported download utilities: aria2c, curl, uclient-fetch, wget +* fast downloads & list processing as they are handled in parallel as background jobs in a configurable 'Download Queue' * full IPv4 and IPv6 support * ipsets (one per source) are used to ban a large number of IP addresses * supports blocking by ASN numbers * supports blocking by iso country codes -* supports local white & blacklist (IPv4, IPv6 & CIDR notation), located by default in /etc/banip/banip.whitelist and /etc/banip/banip.blacklist -* auto-add unsuccessful LuCI and ssh login attempts via 'dropbear' or 'sshd' to local blacklist (see 'ban_autoblacklist' option) -* auto-add the uplink subnet to local whitelist (see 'ban_autowhitelist' option) +* supports local black- & whitelist (IPv4, IPv6, CIDR notation or domain names) +* auto-add unsuccessful LuCI, nginx or ssh login attempts via 'dropbear'/'sshd' to local blacklist +* auto-add the uplink subnet to local whitelist +* black- and whitelist also accept domain names as input to allow IP filtering based on these names * provides a small background log monitor to ban unsuccessful login attempts in real-time * per source configuration of SRC (incoming) and DST (outgoing) * integrated IPSet-Lookup -* integrated RIPE-Lookup +* integrated bgpview-Lookup * blocklist source parsing by fast & flexible regex rulesets * minimal status & error logging to syslog, enable debug logging to receive more output * procd based init system support (start/stop/restart/reload/refresh/status) * procd network interface trigger support * automatic blocklist backup & restore, they will be used in case of download errors or during startup -* Provides comprehensive runtime information -* Provides a detailed IPSet Report -* Provides a powerful query function to quickly find blocked IPs/CIDR in banIP related IPSets -* Provides an easily configurable blocklist update scheduler called 'Refresh Timer' +* provides comprehensive runtime information +* provides a detailed IPSet Report +* provides a powerful query function to quickly find blocked IPs/CIDR in banIP related IPSets +* provides an easily configurable blocklist update scheduler called 'Refresh Timer' * strong LuCI support * optional: add new banIP sources on your own ## Prerequisites -* [OpenWrt](https://openwrt.org), tested with the stable release series (19.07.x) and with the latest rolling snapshot releases. On turris devices it has been successfully tested with TurrisOS 5.2.x - Please note: Older OpenWrt releases like 18.06.x or 17.01.x are _not_ supported! +* [OpenWrt](https://openwrt.org), tested with the stable release series (21.02.x) and with the latest rolling snapshot releases. On turris devices it has been successfully tested with TurrisOS 5.2.x + Please note: Ancient OpenWrt releases like 18.06.x or 17.01.x are _not_ supported! Please note: Devices with less than 128 MByte RAM are _not_ supported! Please note: If you're updating from former banIP 0.3x please manually remove your config (/etc/config/banip) before you start! * A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required * A certificate store like 'ca-bundle', as banIP checks the validity of the SSL certificates of all download sites by default -* Optional E-Mail notification support: for E-Mail notifications you need to install the additional 'msmtp' package +* Optional E-Mail notification support: for E-Mail notifications you need to install and setup the additional 'msmtp' package ## Installation & Usage * Update your local opkg repository (_opkg update_) @@ -160,8 +161,7 @@ Available commands: | ban_nginx_logcount | option | 5 | number of the failed nginx requests of the same ip in the log before banning | ## Examples -**list/edit banIP sources:** - +**list/edit banIP sources:** 

 ~# /etc/init.d/banip list
 ::: Available banIP sources
@@ -171,6 +171,7 @@ Available commands:
   + asn                            ASN blocks                          https://asn.ipinfo.app
   + bogon                          Bogon prefixes                      https://team-cymru.com
   + country              x         Country blocks                      https://www.ipdeny.com/ipblocks
+  + darklist             x         Blocks suspicious attacker IPs      https://darklist.de
   + debl                 x         Fail2ban IP blacklist               https://www.blocklist.de
   + doh                  x         Public DoH-Provider                 https://github.com/dibdot/DoH-IP-blocklists
   + drop                 x         Spamhaus drop compilation           https://www.spamhaus.org
@@ -181,12 +182,14 @@ Available commands:
   + firehol2                       Firehol Level 2 compilation         https://iplists.firehol.org/?ipset=firehol_level2
   + firehol3                       Firehol Level 3 compilation         https://iplists.firehol.org/?ipset=firehol_level3
   + firehol4                       Firehol Level 4 compilation         https://iplists.firehol.org/?ipset=firehol_level4
+  + greensnow            x         Blocks suspicious server IPs        https://greensnow.co
   + iblockads                      Advertising blocklist               https://www.iblocklist.com
   + iblockspy            x         Malicious spyware blocklist         https://www.iblocklist.com
   + myip                           Myip Live IP blacklist              https://myip.ms
   + nixspam              x         iX spam protection                  http://www.nixspam.org
   + proxy                          Firehol list of open proxies        https://iplists.firehol.org/?ipset=proxylists
   + sslbl                x         SSL botnet IP blacklist             https://sslbl.abuse.ch
+  + talos                x         Cisco Talos IP Blacklist            https://talosintelligence.com/reputation_center
   + threat               x         Emerging Threats                    https://rules.emergingthreats.net
   + tor                  x         Tor exit nodes                      https://fissionrelays.net/lists
   + uceprotect1          x         Spam protection level 1             http://www.uceprotect.net/en/index.php
@@ -198,28 +201,31 @@ Available commands:
   * Configured Countries: af, bd, br, cn, hk, hu, id, il, in, iq, ir, kp, kr, no, pk, pl, ro, ru, sa, th, tr, ua, gb
-**receive banIP runtime information:** - +**receive banIP runtime information:** 

 ~# /etc/init.d/banip status
 ::: banIP runtime information
   + status          : enabled
-  + version         : 0.7.0
-  + ipset_info      : 23 IPSets with 302008 IPs/Prefixes
-  + active_sources  : blacklist, country, debl, doh, drop, dshield, feodo, firehol1, iblockspy, nixspam, sslbl, threat, 
-                      tor, uceprotect1, voip, whitelist, yoyo
+  + version         : 0.7.5
+  + ipset_info      : 27 IPSets with 280704 IPs/Prefixes
+  + active_sources  : blacklist, country, darklist, debl, doh, drop, dshield, feodo, firehol1, greensnow, iblockspy, nix
+                      spam, sslbl, talos, threat, tor, uceprotect1, voip, whitelist, yoyo
   + active_devs     : eth3
   + active_ifaces   : wan, wan6
-  + active_logterms : dropbear, sshd, luci
-  + active_subnets  : xxx.xxx.x.xxx/24, xxxx:xxxx:xxxx:x:xxxx:xxxx:xxxx:xxxx/64
-  + run_infos       : settype: src+dst, backup_dir: /mnt/data/banip, report_dir: /tmp/banIP-Report
+  + active_logterms : dropbear, luci
+  + active_subnets  : xxx.xxx.x.xxx/24, xxxx:xxxx:xxxx:0:xxxx:xxxx:xxxx:xxxx/64
+  + run_infos       : settype: src+dst, backup_dir: /mnt/data/banIP/backup, report_dir: /mnt/data/banIP/report
   + run_flags       : protocols (4/6): ✔/✔, log (src/dst): ✔/✘, monitor: ✔, mail: ✔
-  + last_run        : refresh, 0m 16s, 4019/3527/3680, 03.02.2021 19:57:46
-  + system          : PC Engines apu4, OpenWrt SNAPSHOT r15556-20a0d435d8
+  + last_run        : refresh, 0m 15s, 4019/3743/3784, 15.03.2021 09:28:01
+  + system          : PC Engines apu4, OpenWrt SNAPSHOT r16186-bf4aa0c6a2
-**generate an IPSet report:** - +**black-/whitelist handling:** +banIP supports a local black & whitelist (IPv4, IPv6, CIDR notation or domain names), located by default in /etc/banip/banip.whitelist and /etc/banip/banip.blacklist. +Unsuccessful LuCI logins, suspicious nginx request or ssh login attempts via 'dropbear'/'sshd' could be tracked and automatically added to the local blacklist (see the 'ban_autoblacklist' option). Furthermore the uplink subnet could be automatically added to local whitelist (see 'ban_autowhitelist' option). The list behaviour could be further tweaked with different timeout and counter options (see the config options section above). +Last but not least, both lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be resolved in a detached background process and added to the IPsets. The detached name lookup takes place only during 'restart' or 'reload' action, 'start' and 'refresh' actions are using an auto-generated backup instead. + +**generate an IPSet report:** 

 ~# /etc/init.d/banip report
 :::
@@ -338,9 +344,9 @@ syslog          LOG_MAIL
 account         ban_notify
 host            smtp.gmail.com
 port            587
-from            
k@gmail.com
-user            
-password        
+from            <address>@gmail.com
+user            <gmail-user>
+password        <password>
Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI. diff --git a/net/banip/files/banip.dns b/net/banip/files/banip.dns index c5b2b9a635..198a501cc2 100755 --- a/net/banip/files/banip.dns +++ b/net/banip/files/banip.dns @@ -15,10 +15,17 @@ if [ -r "/lib/functions.sh" ] then . "/lib/functions.sh" ban_debug="$(uci_get banip global ban_debug "0")" + ban_tmpbase="$(uci_get banip global ban_tmpbase "/tmp")" + ban_backupdir="$(uci_get banip global ban_backupdir "${ban_tmpbase}/banIP-Backup")" + ban_proto4_enabled="$(uci_get banip global ban_proto4_enabled "0")" + ban_proto6_enabled="$(uci_get banip global ban_proto6_enabled "0")" +else + exit 1 fi ban_ver="${1}" -ban_src_name="${2}" -ban_src_file="${3}" +ban_action="${2}" +ban_src_name="${3}" +ban_src_file="${4}" ban_ipset_cmd="$(command -v ipset)" ban_lookup_cmd="$(command -v nslookup)" ban_logger_cmd="$(command -v logger)" @@ -39,23 +46,47 @@ f_log() fi } -while read -r domain -do - update_ips="" - result="$("${ban_lookup_cmd}" "${domain}" 2>/dev/null; printf "%s" "${?}")" - if [ "$(printf "%s" "${result}" | tail -1)" = "0" ] - then - ips="$(printf "%s" "${result}" | awk '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - for ip in ${ips} - do - for proto in "4" "6" +if [ "${ban_action}" = "start" ] || [ "${ban_action}" = "refresh" ] +then + for proto in "4" "6" + do + if [ -s "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}.gz" ] + then + gzip -df "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}.gz" + if [ "${?}" = "0" ] + then + ban_rc=0 + else + ban_rc=1 + break + fi + fi + done +fi + +if [ "${ban_rc}" = "1" ] +then + > "${ban_backupdir}/banIP.${ban_src_name}_addon_4" + > "${ban_backupdir}/banIP.${ban_src_name}_addon_6" + while read -r domain + do + update_ips="" + result="$("${ban_lookup_cmd}" "${domain}" 2>/dev/null; printf "%s" "${?}")" + if [ "$(printf "%s" "${result}" | tail -1)" = "0" ] + then + ips="$(printf "%s" "${result}" | awk '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" + for ip in ${ips} do - if { [ "${proto}" = "4" ] && [ -n "$("${ban_ipset_cmd}" -q -n list "${ban_src_name}_${proto}")" ] && [ -n "$(printf "%s" "${ip}" | awk '/^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/[0-9]{1,2})?)([[:space:]]|$)/{print $1}')" ]; } || \ - { [ "${proto}" = "6" ] && [ -n "$("${ban_ipset_cmd}" -q -n list "${ban_src_name}_${proto}")" ] && [ -z "$(printf "%s" "${ip}" | awk '/^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/[0-9]{1,2})?)([[:space:]]|$)/{print $1}')" ]; } - then - "${ban_ipset_cmd}" add "${ban_src_name}_${proto}" "${ip}" 2>/dev/null - if [ "${?}" = "0" ] + for proto in "4" "6" + do + if { [ "${proto}" = "4" ] && [ "${ban_proto4_enabled}" = "1" ] && \ + [ -n "$("${ban_ipset_cmd}" -q -n list "${ban_src_name}_${proto}")" ] && \ + [ -n "$(printf "%s" "${ip}" | awk '/^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/[0-9]{1,2})?)([[:space:]]|$)/{print $1}')" ]; } || \ + { [ "${proto}" = "6" ] && [ "${ban_proto6_enabled}" = "1" ] && \ + [ -n "$("${ban_ipset_cmd}" -q -n list "${ban_src_name}_${proto}")" ] && \ + [ -z "$(printf "%s" "${ip}" | awk '/^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/[0-9]{1,2})?)([[:space:]]|$)/{print $1}')" ]; } then + printf "%s\n" "add ${ban_src_name}_${proto} ${ip}" >> "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}" if [ -z "${update_ips}" ] then update_ips="${ip}" @@ -63,17 +94,29 @@ do update_ips="${update_ips}, ${ip}" fi fi - break - fi + done done - done - if [ -n "${update_ips}" ] - then - ban_rc=0 - f_log "debug" "dns_imp ::: source '${ban_src_name}' supplemented by '${domain}' (${update_ips})" + if [ -n "${update_ips}" ] + then + ban_rc=0 + f_log "debug" "dns_imp ::: source '${ban_src_name}' supplemented by '${domain}' (${update_ips})" + fi fi - fi -done < "${ban_src_file}" -rm -f "${ban_src_file}" + done < "${ban_src_file}" +fi + +if [ "${ban_rc}" = "0" ] +then + for proto in "4" "6" + do + if [ -n "$("${ban_ipset_cmd}" -q -n list "${ban_src_name}_${proto}")" ] && [ -s "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}" ] + then + "${ban_ipset_cmd}" -q -! restore < "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}" + gzip -f "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}" + fi + rm -f "${ban_backupdir}/banIP.${ban_src_name}_addon_${proto}" + done +fi f_log "info" "banIP domain import for source '${ban_src_name}' has been finished with rc '${ban_rc}'" -exit ${ban_rc} +rm -f "${ban_src_file}" +exit "${ban_rc}" diff --git a/net/banip/files/banip.sh b/net/banip/files/banip.sh index 5e045a748d..258b16e873 100755 --- a/net/banip/files/banip.sh +++ b/net/banip/files/banip.sh @@ -740,7 +740,8 @@ f_ipset() return "${out_rc}" ;; "create") - if [ -s "${tmp_file}" ] && [ -z "$("${ban_ipset_cmd}" -q -n list "${src_name}")" ] + if [ -z "$("${ban_ipset_cmd}" -q -n list "${src_name}")" ] && \ + { [ -s "${tmp_file}" ] || [ "${src_name%_*}" = "whitelist" ] || [ "${src_name%_*}" = "blacklist" ]; } then cnt="$(awk 'END{print NR}' "${tmp_file}" 2>/dev/null)" cnt=$((cnt+262144)) @@ -760,7 +761,8 @@ f_ipset() "${ban_ipset_cmd}" create "${src_name}" hash:net hashsize 64 maxelem "${cnt}" family "${src_ipver}" counters out_rc="${?}" fi - else + elif [ -n "$("${ban_ipset_cmd}" -q -n list "${src_name}")" ] + then "${ban_ipset_cmd}" -q flush "${src_name}" out_rc="${?}" fi @@ -1000,21 +1002,22 @@ f_down() # case "${src_name%_*}" in "blacklist"|"whitelist") + printf "%s\n" "0" > "${tmp_cnt}" awk "${src_rule}" "${src_url}" > "${tmp_file}" src_rc="${?}" if [ "${src_rc}" = "0" ] then f_ipset "create" - src_name="${src_name%_*}" - tmp_dns="${ban_tmpbase}/${src_name}.dns" - if [ ! -f "${tmp_dns}" ] && [ "${proto}" = "4" ] + if [ ! -f "${tmp_dns}" ] && { { [ "${proto}" = "4" ] && [ "${ban_proto4_enabled}" = "1" ]; } || \ + { [ "${proto}" = "6" ] && [ "${ban_proto6_enabled}" = "1" ] && [ "${ban_proto4_enabled}" = "0" ]; }; } then + tmp_dns="${ban_tmpbase}/${src_name%_*}.dns" src_rule="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" awk "${src_rule}" "${src_url}" > "${tmp_dns}" src_rc="${?}" if [ "${src_rc}" = "0" ] && [ -s "${tmp_dns}" ] then - ( "${ban_dnsservice}" "${ban_ver}" "${src_name}" "${tmp_dns}" & ) + ( "${ban_dnsservice}" "${ban_ver}" "${ban_action}" "${src_name%_*}" "${tmp_dns}" & ) else rm -f "${tmp_dns}" fi