From c44e40809eb30e937e390b541807515ed594b664 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Wed, 31 Jan 2018 13:13:32 +0100 Subject: [PATCH] adblock: update 3.5.0 * major performance boost: add a flexible 'Download Queue' to handle downloads & list processing in parallel, default queue size is '4', you can raise this e.g. to '8' or '16' to get it really fast * replace former 'whitelist mode': the new 'Jail' option builds an additional 'adb_list.jail' list in parallel which can be used manually for guest wifi or kidsafe configurations * regex parser & query function now fully support IDN domains with non-ASCII characters * add error handling in tld compression, to handle OOM conditions better * adblock.notify sends now html emails, to get a better look & feel, even on mobile devices * add czech regional blocklist maintained by turris omnia users * LuCI: Support new 'Download Queue' & 'Jail' options * LuCI: fix field width in "Runtime Information" section Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 2 +- net/adblock/files/README.md | 73 +++--- net/adblock/files/adblock.conf | 121 +++++----- net/adblock/files/adblock.notify | 25 +- net/adblock/files/adblock.sh | 395 ++++++++++++++++++------------- 5 files changed, 356 insertions(+), 260 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index e40e18c2b7..37936c1d4d 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=3.4.3 +PKG_VERSION:=3.5.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 71d67ef4f9..0b575e71e5 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -31,6 +31,8 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 150 entries * [reg_cn](https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt) * => regional blocklist for China, daily updates, approx. 1.600 entries + * [reg_cz](https://raw.githubusercontent.com/qxstyles/turris-hole-czech-block-list/master/turris-hole-czech-block-list) + * => regional blocklist for Czechia, maintained by Turris Omnia Users, infrequent updates, approx. 100 entries * [reg_de](https://easylist-downloads.adblockplus.org/easylistgermany+easylist.txt) * => regional blocklist for Germany, daily updates, approx. 9.200 entries * [reg_id](https://easylist-downloads.adblockplus.org/abpindo+easylist.txt) @@ -63,10 +65,11 @@ A lot of people already use adblocker plugins within their desktop browsers, but * simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that * supports five different dns backends / blocklist formats: dnsmasq, unbound, named (bind), kresd and dnscrypt-proxy * supports six different download utilities: uclient-fetch, wget, curl, aria2c, wget-nossl, busybox-wget +* Really fast downloads & list processing as they are handled in parallel as background jobs in a configurable 'Download Queue' * provides 'http only' mode without installed ssl library for all non-SSL blocklist sources * supports a wide range of router modes, even AP modes are supported * full IPv4 and IPv6 support -* provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the dns backends +* provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the dns backend * blocklist source parsing by fast & flexible regex rulesets * overall duplicate removal in central blocklist 'adb_list.overall' * additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist @@ -74,7 +77,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * minimal status & error logging to syslog, enable debug logging to receive more output * procd based init system support (start/stop/restart/reload/suspend/resume/query/status) * procd network interface trigger support or classic time based startup -* keep the dns cache intact after adblock processing (currently supported by unbound and named) +* keep the dns cache intact after adblock processing (currently supported by unbound, named and kresd) * conditional dns backend restarts by old/new blocklist comparison with sha256sum (default) or md5sum * suspend & resume adblock actions temporarily without blocklist reloading * output comprehensive runtime information via LuCI or via 'status' init command @@ -84,12 +87,12 @@ A lot of people already use adblocker plugins within their desktop browsers, but * optional: force overall sort / duplicate removal for low memory devices (handle with care!) * optional: automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode * optional: 'backup mode' to re-use blocklist backups during startup, get fresh lists only via reload or restart action -* optional: 'whitelist mode' to block access to all domains except those explicitly listed in the whitelist file +* optional: 'Jail' blocklist generation which builds an additional list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file. You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations * optional: send notification emails in case of a processing error or if the overall domain count is ≤ 0 -* optional: add new adblock sources on your own via uci config +* optional: add new adblock sources on your own, see example below ## Prerequisites -* [LEDE project](https://www.lede-project.org), tested with latest stable release (LEDE 17.01) and with current LEDE snapshot +* [OpenWrt](https://openwrt.org), tested with the stable release series (17.01.x) and with the latest OpenWrt snapshot * a usual setup with an enabled dns backend at minimum - dump AP modes without a working dns backend are _not_ supported * a download utility: * to support all blocklist sources a full version (with ssl support) of 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required @@ -98,7 +101,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but ## Installation & Usage * install 'adblock' (_opkg install adblock_) -* at minimum configure the appropriate dns backend ('dnsmasq' by default) and enable the adblock service in _/etc/config/adblock_ +* at minimum configure the appropriate dns backend ('dnsmasq' by default), the donwload utility and enable the adblock service in _/etc/config/adblock_ * control the adblock service manually with _/etc/init.d/adblock_ start/stop/restart/reload/suspend/resume/status or use the LuCI frontend ## LuCI adblock companion package @@ -109,13 +112,14 @@ A lot of people already use adblocker plugins within their desktop browsers, but ## Tweaks * **runtime information:** the adblock status is available via _/etc/init.d/adblock status_ (see example below) * **debug logging:** for script debugging please set the config option 'adb\_debug' to '1' and check the runtime output with _logread -e "adblock"_ -* **storage expansion:** to process and store all blocklist sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details +* **storage expansion:** to process and store all blocklist sources at once it might helpful to enlarge your temp directory with a swap partition => see [OpenWrt Wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details * **add white- / blacklist entries:** add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are empty and located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) * **backup & restore blocklists:** enable this feature, to restore automatically the latest compressed backup of your blocklists in case of any processing error (e.g. a single blocklist source is not available during update). Please use an (external) solid partition and _not_ your volatile router temp directory for this +* **download queue size:** for further download & list processing performance improvements you can raise the 'adb\_maxqueue' value, e.g. '8' or '16' should be safe * **scheduled list updates:** for a scheduled call of the adblock service add an appropriate crontab entry (see example below) -* **change startup behaviour:** by default the startup will be triggered by the 'wan' procd interface trigger. Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface. +* **change startup behaviour:** by default the startup will be triggered by the 'wan' procd interface trigger. Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface * **suspend & resume adblocking:** to quickly switch the adblock service 'on' or 'off', simply use _/etc/init.d/adblock [suspend|resume]_ -* **domain query:** to query the active blocklist for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) +* **domain query:** to query the active blocklist for a certain domain, please use the LuCI frontend or run _/etc/init.d/adblock query ``_ (see example below) * **add new list sources:** you could add new blocklist sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) * **disable active dns probing in windows 10:** to prevent a yellow exclamation mark on your internet connection icon (which wrongly means connected, but no internet), please change the following registry key/value from "1" to "0" _HKLM\SYSTEM\CurrentControlSet\Services\NlaSvc\Parameters\Internet\EnableActiveProbing_ @@ -131,14 +135,15 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_trigger => set the startup trigger to a certain interface, to 'timed' or to 'none' (default: 'wan') * the following options apply to the 'extra' config section: - * adb\_triggerdelay => additional trigger delay in seconds before adblock processing begins (default: '1') - * adb\_forcedns => force dns requests to local resolver (default: '0', disabled) - * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) - * adb\_backup_mode => do not automatically update blocklists during startup, use backups instead (default: '0', disabled) - * adb\_whitelist_mode => block access to all domains except those explicitly listed in the whitelist file (default: '0', disabled) - * adb\_dnsflush => flush DNS cache after adblock processing, i.e. enable the old restart behavior (default: '0', disabled) - * adb\_notify => send notification emails in case of a processing error or if the overall domain count is ≤ 0 (default: '0', disabled) - * adb\_notifycnt => Raise minimum domain count email notification trigger (default: '0') + * adb\_triggerdelay => additional trigger delay in seconds before adblock processing begins (int/default: '2') + * adb\_forcedns => force dns requests to local resolver (bool/default: '0', disabled) + * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (bool/default: '0', disabled) + * adb\_backup_mode => do not automatically update blocklists during startup, use backups instead (bool/default: '0', disabled) + * adb\maxqueue => size of the download queue to handle downloads & list processing in parallel (int/default: '4') + * adb\_jail => builds an additional 'Jail' list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file (bool/default: '0', disabled) + * adb\_dnsflush => flush DNS cache after adblock processing, i.e. enable the old restart behavior (bool/default: '0', disabled) + * adb\_notify => send notification emails in case of a processing error or if the overall domain count is ≤ 0 (bool/default: '0', disabled) + * adb\_notifycnt => Raise minimum domain count email notification trigger (int/default: '0') ## Examples **change default dns backend to 'unbound':** @@ -172,7 +177,7 @@ Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no furt **change default dns backend to 'dnscrypt-proxy':** The required 'blacklist' option of dnscrypt-proxy is not enabled by default, because the package will be compiled without plugins support. -Take a custom LEDE build with plugins support to use this feature. Adblock deposits the final blocklist 'adb_list.overall' in '/tmp'. +Take a custom OpenWrt build with plugins support to use this feature. Adblock deposits the final blocklist 'adb_list.overall' in '/tmp'. To use the blocklist please modify '/etc/config/dnscrypt-proxy' per instance: 

   list blacklist 'domains:/tmp/adb_list.overall'
@@ -207,12 +212,12 @@ Finally make this file executable via 'chmod' and test it directly. If no more e
 /etc/init.d/adblock status
 ::: adblock runtime information
   + adblock_status  : enabled
-  + adblock_version : 3.4.0
-  + overall_domains : 5167 (normal/backup mode)
-  + fetch_utility   : wget (built-in)
+  + adblock_version : 3.5.0
+  + overall_domains : 102324 (normal mode)
+  + fetch_utility   : /usr/bin/wget (built-in)
   + dns_backend     : kresd (/etc/kresd)
-  + last_rundate    : 27.12.2017 20:52:35
-  + system_release  : Turris Omnia, OpenWrt omnia 15.05/3.9.1
+  + last_rundate    : 30.01.2018 21:24:11
+  + system_release  : Turris Omnia, OpenWrt omnia 15.05/3.9.4
**cronjob for a regular block list update (/etc/crontabs/root):** @@ -255,18 +260,22 @@ This entry does not remove: The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain. For every (sub-)domain it returns the first ten relevant results. 

 /etc/init.d/adblock query www.example.google.com
-::: max. ten results for domain 'www.example.google.com'
+::: results for domain 'www.example.google.com'
   - no match
-::: max. ten results for domain 'example.google.com'
+::: results for domain 'example.google.com'
   - no match
-::: max. ten results for domain 'google.com'
+::: results for domain 'google.com'
+  + ads.google.com
+  + adservices.google.com
+  + adwords.google.com
+  + ampcid.google.com
   + analytics.google.com
+  + gg.google.com
+  + google.com.analytics.kdgsrltkcun.com
   + googleadapis.l.google.com
-  + pagead.l.google.com
-  + partnerad.l.google.com
-  + ssl-google-analytics.l.google.com
-  + video-stats.video.google.com
-  + www-google-analytics.l.google.com
+  + id.google.com
+  + pagead-googlehosted.l.google.com
+  + [...]
**add a new blocklist source:** @@ -279,7 +288,7 @@ the source name, the url and the description - that's all! config source 'reg_ro' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 9.400 entries' diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 16a3436f1e..2feae31ff1 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -12,170 +12,177 @@ config adblock 'extra' option adb_forcesrt '0' option adb_forcedns '0' option adb_backup '0' + option adb_maxqueue '4' config source 'adaway' - option enabled '1' option adb_src 'https://adaway.org/hosts.txt' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'focus on mobile ads, infrequent updates, approx. 400 entries' + option enabled '1' config source 'adguard' - option enabled '0' option adb_src 'https://filters.adtidy.org/windows/filters/15.txt' - option adb_src_rset 'BEGIN{FS=\"[/|^|\r]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+([\/\^\r]|$)/{print tolower(\$3)}' - option adb_src_desc 'combined adguard dns filter list, frequent updates, approx. 15.700 entries' + option adb_src_rset 'BEGIN{FS=\"[/|^|\r]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([\/\^\r]|$)/{print tolower(\$3)}' + option adb_src_desc 'combined adguard dns filter list, frequent updates, approx. 17.000 entries' + option enabled '0' config source 'bitcoin' - option enabled '0' option adb_src 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt' - option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' - option adb_src_desc 'focus on malicious bitcoin mining sites, infrequent updates, approx. 20 entries' + option adb_src_rset '/^0\.0\.0\.0[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'focus on malicious bitcoin mining sites, infrequent updates, approx. 80 entries' + option enabled '0' config source 'blacklist' - option enabled '1' option adb_src '/etc/adblock/adblock.blacklist' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'static local domain blacklist, always deny these domains' + option enabled '1' config source 'disconnect' - option enabled '1' option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' - option adb_src_desc 'mozilla driven blocklist, numerous updates on the same day, approx. 4.600 entries' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'mozilla driven blocklist, numerous updates on the same day, approx. 4.700 entries' + option enabled '1' config source 'dshield' - option enabled '0' option adb_src 'https://www.dshield.org/feeds/suspiciousdomains_Low.txt' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'generic blocklist, daily updates, approx. 3.500 entries' + option enabled '0' config source 'feodo' - option enabled '0' option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'focus on feodo botnet, daily updates, approx. 0-10 entries' + option enabled '0' config source 'hphosts' - option enabled '0' option adb_src 'https://hosts-file.net/ad_servers.txt' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|\$)+/{print tolower(\$2)}' option adb_src_desc 'broad blocklist, monthly updates, approx. 19.200 entries' + option enabled '0' config source 'malware' - option enabled '0' option adb_src 'https://mirror.espoch.edu.ec/malwaredomains/justdomains' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'broad blocklist, daily updates, approx. 18.300 entries' + option enabled '0' config source 'malwarelist' - option enabled '0' option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'focus on malware, daily updates, approx. 1.200 entries' + option enabled '0' config source 'openphish' - option enabled '0' option adb_src 'https://openphish.com/feed.txt' - option adb_src_rset 'BEGIN{FS=\"/\"}\$0~/^http[s]?:\/\/([[:alnum:]_-]+\.){1,}[[:alpha:]]+(\/|$)/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"/\"}/^http[s]?:\/\/([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+(\/|$)/{print tolower(\$3)}' option adb_src_desc 'focus on phishing, numerous updates on the same day, approx. 2.400 entries' + option enabled '0' config source 'ransomware' - option enabled '0' option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|\r|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 1900 entries' + option enabled '0' config source 'reg_cn' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on chinese ads plus generic easylist additions, daily updates, approx. 11.700 entries' + option enabled '0' + +config source 'reg_cz' + option adb_src 'https://raw.githubusercontent.com/qxstyles/turris-hole-czech-block-list/master/turris-hole-czech-block-list' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'focus on czech ads maintained by Turris Omnia Users, infrequent updates, approx. 100 entries' + option enabled '0' config source 'reg_de' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistgermany+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on german ads plus generic easylist additions, daily updates, approx. 9.200 entries' + option enabled '0' config source 'reg_id' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/abpindo+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on indonesian ads plus generic easylist additions, weekly updates, approx. 9.600 entries' + option enabled '0' config source 'reg_nl' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistdutch+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on dutch ads plus generic easylist additions, weekly updates, approx. 9.400 entries' + option enabled '0' config source 'reg_pl' - option enabled '0' option adb_src 'http://adblocklist.org/adblock-pxf-polish.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on polish ads, daily updates, approx. 90 entries' + option enabled '0' config source 'reg_ro' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 9.400 entries' + option enabled '0' config source 'reg_ru' - option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt' - option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}/^\|\|([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on russian ads plus generic easylist additions, weekly updates, approx. 14.500 entries' + option enabled '0' config source 'shalla' - option enabled '0' option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'broad blocklist subdivided in different categories, daily updates, approx. 31.700 entries' list adb_src_cat 'adv' list adb_src_cat 'costtraps' list adb_src_cat 'spyware' list adb_src_cat 'tracker' list adb_src_cat 'warez' + option enabled '0' config source 'spam404' - option enabled '0' option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)+/{print tolower(\$1)}' option adb_src_desc 'generic blocklist, infrequent updates, approx. 6.000 entries' + option enabled '0' config source 'sysctl' - option enabled '0' option adb_src 'http://sysctl.org/cameleon/hosts' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'broad blocklist, weekly updates, approx. 16.500 entries' + option enabled '0' config source 'whocares' - option enabled '0' option adb_src 'http://someonewhocares.org/hosts/hosts' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'broad blocklist, weekly updates, approx. 10.000 entries' + option enabled '0' config source 'winspy' - option enabled '0' option adb_src 'https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/win10/spy.txt' - option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^0\.0\.0\.0[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'focus on windows spy & telemetry domains, infrequent updates, approx. 300 entries' + option enabled '0' config source 'winhelp' - option enabled '0' option adb_src 'http://winhelp2002.mvps.org/hosts.txt' - option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_rset '/^0\.0\.0\.0[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'broad blocklist, infrequent updates, approx. 13.000 entries' + option enabled '0' config source 'yoyo' - option enabled '1' option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'focus on ad related domains, weekly updates, approx. 2.400 entries' + option enabled '1' config source 'zeus' - option enabled '0' option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist' - option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'focus on zeus botnet, daily updates, approx. 400 entries' + option enabled '0' diff --git a/net/adblock/files/adblock.notify b/net/adblock/files/adblock.notify index dc5acf5a48..10dc4d15f7 100644 --- a/net/adblock/files/adblock.notify +++ b/net/adblock/files/adblock.notify @@ -10,22 +10,28 @@ LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -mail_ver="1.0.1" +mail_ver="1.0.2" mail_daemon="$(command -v msmtp)" mail_profile="adb_notify" -mail_debug="--debug" +#mail_debug="--debug" mail_rc=1 +# mail daemon check +# if [ ! -x "${mail_daemon}" ] then mail_daemon="$(command -v sendmail)" fi +# info preparation +# +sys_info="$(strings /etc/banner 2>/dev/null; ubus call system board | sed -e 's/\"release\": {//' | sed -e 's/^[ \t]*//' | sed -e 's/[{}\",]//g' | sed -e 's/[ ]/ \t/' | sed '/^$/d' 2>/dev/null)" +adb_info="$(/etc/init.d/adblock status 2>/dev/null)" if [ -f "/var/log/messages" ] then - logfile="$(cat /var/log/messages | grep "adblock-")" + log_info="$(awk '/adblock-/{NR=1;max=79;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max),"↵"} else {print " ",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}' /var/log/messages)" else - logfile="$(logread -e "adblock-")" + log_info="$(logread -e "adblock-" | awk '{NR=1;max=79;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max),"↵"} else {print " ",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" fi # mail header @@ -33,14 +39,15 @@ fi mail_sender="no-reply@adblock" mail_receiver="!!!ChangeMe!!!" mail_topic="adblock notification" -mail_head="From: ${mail_sender}\nTo: ${mail_receiver}\nSubject: ${mail_topic}\nReply-to: ${mail_sender}\n\n" +mail_head="From: ${mail_sender}\nTo: ${mail_receiver}\nSubject: ${mail_topic}\nReply-to: ${mail_sender}\nMime-Version: 1.0\nContent-Type: text/html\nContent-Disposition: inline\n\n" # mail body # -mail_text="adblock status, generated at $(date 2>&1)!" -mail_text="${mail_text}\n++\n++ System Information ++\n++\n$(cat /etc/banner 2>&1)" -mail_text="${mail_text}\n\n++\n++ Adblock Information ++\n++\n$(/etc/init.d/adblock status 2>&1)" -mail_text="${mail_text}\n\n++\n++ Logfile Information ++\n++\n${logfile}\n++\n++\n" +mail_text="
"
+mail_text="${mail_text}\n++\n++ System Information ++\n++\n${sys_info}"
+mail_text="${mail_text}\n\n++\n++ Adblock Information ++\n++\n${adb_info}"
+mail_text="${mail_text}\n\n++\n++ Logfile Information ++\n++\n${log_info}"
+mail_text="${mail_text}
" # send mail # diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 465da07c8a..099cb6066c 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,14 +10,15 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="3.4.3" +adb_ver="3.5.0" adb_sysver="unknown" adb_enabled=0 adb_debug=0 adb_backup_mode=0 -adb_whitelist_mode=0 adb_forcesrt=0 adb_forcedns=0 +adb_jail=0 +adb_maxqueue=4 adb_notify=0 adb_notifycnt=0 adb_triggerdelay=0 @@ -27,6 +28,7 @@ adb_fetchutil="uclient-fetch" adb_dns="dnsmasq" adb_dnsprefix="adb_list" adb_dnsfile="${adb_dnsprefix}.overall" +adb_dnsjail="${adb_dnsprefix}.jail" adb_dnsflush=0 adb_whitelist="/etc/adblock/adblock.whitelist" adb_rtfile="/tmp/adb_runtime.json" @@ -123,11 +125,11 @@ f_envload() adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" adb_dnsdir="${adb_dnsdir:-"/tmp"}" adb_dnsheader="" - adb_dnsformat="awk '{print \"server=/\"\$0\"/\"}'" - if [ ${adb_whitelist_mode} -eq 1 ] + adb_dnsdeny="awk '{print \"server=/\"\$0\"/\"}'" + if [ ${adb_jail} -eq 1 ] then - adb_dnsformat="awk '{print \"server=/\"\$0\"/#\"}'" - adb_dnsblock="server=/#/" + adb_dnsallow="awk '{print \"server=/\"\$0\"/#\"}'" + adb_dnshalt="server=/#/" fi ;; unbound) @@ -135,11 +137,11 @@ f_envload() adb_dnsuser="${adb_dnsuser:-"unbound"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" adb_dnsheader="" - adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" - if [ ${adb_whitelist_mode} -eq 1 ] + adb_dnsdeny="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" + if [ ${adb_jail} -eq 1 ] then - adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 transparent\"}'" - adb_dnsblock="local-zone: \".\" static" + adb_dnsallow="awk '{print \"local-zone: \042\"\$0\"\042 transparent\"}'" + adb_dnshalt="local-zone: \".\" static" fi ;; named) @@ -147,11 +149,11 @@ f_envload() adb_dnsuser="${adb_dnsuser:-"bind"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." - adb_dnsformat="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" - if [ ${adb_whitelist_mode} -eq 1 ] + adb_dnsdeny="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" + if [ ${adb_jail} -eq 1 ] then - adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" - adb_dnsblock="* CNAME ." + adb_dnsallow="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" + adb_dnshalt="* CNAME ." fi ;; kresd) @@ -159,11 +161,11 @@ f_envload() adb_dnsuser="${adb_dnsuser:-"root"}" adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." - adb_dnsformat="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" - if [ ${adb_whitelist_mode} -eq 1 ] + adb_dnsdeny="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" + if [ ${adb_jail} -eq 1 ] then - adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" - adb_dnsblock="* CNAME ." + adb_dnsallow="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" + adb_dnshalt="* CNAME ." fi ;; dnscrypt-proxy) @@ -171,7 +173,7 @@ f_envload() adb_dnsuser="${adb_dnsuser:-"nobody"}" adb_dnsdir="${adb_dnsdir:-"/tmp"}" adb_dnsheader="" - adb_dnsformat="awk '{print \$0}'" + adb_dnsdeny="awk '{print \$0}'" ;; esac @@ -187,11 +189,6 @@ f_envload() exit 0 fi - if [ "${adb_dns}" = "dnsmasq" ] && [ ${adb_dnsinstance} -ne 0 ] - then - adb_dnsfile="${adb_dnsprefix}.overall.${adb_dnsinstance}" - fi - if [ -d "${adb_dnsdir}" ] && [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] then printf '%s\n' "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" @@ -213,7 +210,7 @@ f_envload() cnt=$((cnt+1)) done - if [ -z "${adb_dns}" ] || [ -z "${adb_dnsformat}" ] || [ ! -x "$(command -v ${adb_dns})" ] || [ ! -d "${adb_dnsdir}" ] + if [ -z "${adb_dns}" ] || [ -z "${adb_dnsdeny}" ] || [ ! -x "$(command -v ${adb_dns})" ] || [ ! -d "${adb_dnsdir}" ] then f_log "err" "'${adb_dns}' not running, DNS backend not found" fi @@ -280,8 +277,8 @@ f_temp() if [ -z "${adb_tmpdir}" ] then adb_tmpdir="$(mktemp -p /tmp -d)" - adb_tmpload="$(mktemp -tu)" - adb_tmpfile="$(mktemp -tu)" + adb_tmpload="$(mktemp -p ${adb_tmpdir} -tu)" + adb_tmpfile="$(mktemp -p ${adb_tmpdir} -tu)" fi if [ ! -s "${adb_pidfile}" ] then @@ -296,8 +293,6 @@ f_rmtemp() if [ -d "${adb_tmpdir}" ] then rm -rf "${adb_tmpdir}" - rm -f "${adb_tmpload}" - rm -f "${adb_tmpfile}" fi > "${adb_pidfile}" } @@ -363,6 +358,9 @@ f_count() else adb_cnt="$(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}")" fi + elif [ "${mode}" = "whitelist" ] && [ -s "${adb_tmpdir}/tmp.whitelist" ] + then + adb_cnt="$(wc -l 2>/dev/null < "${adb_tmpdir}/tmp.whitelist")" elif [ -s "${adb_tmpfile}" ] then adb_cnt="$(wc -l 2>/dev/null < "${adb_tmpfile}")" @@ -499,27 +497,28 @@ f_dnsup() cnt=$((cnt+1)) sleep 1 done - f_log "debug" "f_dnsup::: cache_util: ${cache_util:-"-"}, cache_rc: ${cache_rc:-"-"}, cache_flush: ${adb_dnsflush}, cache_cnt: ${cnt}, out_rc: ${adb_rc}" + f_log "debug" "f_dnsup::: cache_util: ${cache_util:-"-"}, cache_rc: ${cache_rc:-"-"}, cache_flush: ${adb_dnsflush}, cache_cnt: ${cnt}, rc: ${adb_rc}" + return ${adb_rc} } # backup/restore/remove blocklists # f_list() { - local mode="${1}" in_rc="${adb_rc}" + local file mode="${1}" in_rc="${adb_rc}" case "${mode}" in backup) if [ -d "${adb_backupdir}" ] then - gzip -cf "${adb_tmpfile}" > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null + gzip -cf "${adb_tmpfile}" 2>/dev/null > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" adb_rc=${?} fi ;; restore) if [ -d "${adb_backupdir}" ] && [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ] then - gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" 2>/dev/null + gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null > "${adb_tmpfile}" adb_rc=${?} fi ;; @@ -531,18 +530,24 @@ f_list() adb_rc=${?} ;; merge) - if [ -s "${adb_tmpfile}" ] - then - cat "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}" - adb_rc=${?} - fi + for file in "${adb_tmpfile}".* + do + cat "${file}" 2>/dev/null >> "${adb_tmpdir}/${adb_dnsfile}" + if [ ${?} -ne 0 ] + then + adb_rc=${?} + break + fi + rm -f "${file}" + done + adb_tmpfile="${adb_tmpdir}/${adb_dnsfile}" ;; final) if [ -s "${adb_tmpdir}/tmp.whitelist" ] then - grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpdir}/${adb_dnsfile}" | eval "${adb_dnsformat}" > "${adb_dnsdir}/${adb_dnsfile}" + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpdir}/${adb_dnsfile}" | eval "${adb_dnsdeny}" > "${adb_dnsdir}/${adb_dnsfile}" else - eval "${adb_dnsformat}" "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" + eval "${adb_dnsdeny}" "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" fi if [ ${?} -eq 0 ] && [ -n "${adb_dnsheader}" ] then @@ -553,22 +558,50 @@ f_list() ;; esac f_count "${mode}" - f_log "debug" "f_list ::: mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" + f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" } # top level domain compression # f_tld() { - local cnt cnt_srt cnt_tld source="${1}" temp="${adb_tmpload}" + local cnt cnt_srt cnt_tld source="${1}" temp="${1}.tld" cnt="$(wc -l 2>/dev/null < "${source}")" - awk -F "." '{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${source}" 2>/dev/null | sort -u > "${temp}" - cnt_srt="$(wc -l 2>/dev/null < "${temp}")" - awk '{if(NR==1){tld=$NF};while(getline){if($NF!~tld"\\."){print tld;tld=$NF}}print tld}' "${temp}" 2>/dev/null |\ - awk -F "." '{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' 2>/dev/null | sort > "${source}" - cnt_tld="$(wc -l 2>/dev/null < "${source}")" - f_log "debug" "f_tld ::: source: ${source}, cnt: ${cnt}, cnt_srt: ${cnt_srt}, cnt_tld: ${cnt_tld}" + sort -u "${source}" > "${temp}" + if [ ${?} -eq 0 ] + then + cnt_srt="$(wc -l 2>/dev/null < "${temp}")" + awk -F "." '{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${temp}" > "${source}" + if [ ${?} -eq 0 ] + then + sort "${source}" > "${temp}" + if [ ${?} -eq 0 ] + then + awk '{if(NR==1){tld=$NF};while(getline){if($NF!~tld"\\."){print tld;tld=$NF}}print tld}' "${temp}" > "${source}" + if [ ${?} -eq 0 ] + then + awk -F "." '{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${source}" > "${temp}" + if [ ${?} -eq 0 ] + then + sort "${temp}" > "${source}" + if [ ${?} -eq 0 ] + then + cnt_tld="$(wc -l 2>/dev/null < "${source}")" + else + cat "${temp}" > "${source}" + fi + fi + else + cat "${temp}" > "${source}" + fi + fi + else + cat "${temp}" > "${source}" + fi + fi + rm -f "${temp}" + f_log "debug" "f_tld ::: source: ${source}, cnt: ${cnt:-"-"}, cnt_srt: ${cnt_srt:-"-"}, cnt_tld: ${cnt_tld:-"-"}" } # blocklist hash compare @@ -635,25 +668,25 @@ f_switch() # f_query() { - local search result cnt - local domain="${1}" - local tld="${domain#*.}" + local search result field=1 domain="${1}" tld="${1#*.}" if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'" else - cd "${adb_dnsdir}" + case "${adb_dns}" in + dnsmasq) + field=2 + ;; + unbound) + field=3 + ;; + esac while [ "${domain}" != "${tld}" ] do search="${domain//./\.}" - if [ "${adb_dns}" = "dnsmasq" ] || [ "${adb_dns}" = "unbound" ] - then - result="$(awk -F '/|\"' "/[\/\"\.]${search}/{i++;{printf(\" + %s\n\",\$2)};if(i>9){exit}}" "${adb_dnsfile}")" - else - result="$(awk "/(^[^\*][[:alpha:]]*[\.]+${search}|^${search})/{i++;{printf(\" + %s\n\",\$1)};if(i>9){exit}}" "${adb_dnsfile}")" - fi - printf "%s\n" "::: results for domain '${domain}' (max. 10)" + result="$(awk -F '/|\"| ' "/^($search|[^\*].*[\/\"\. ]+${search})/{i++;{printf(\" + %s\n\",\$${field})};if(i>9){printf(\" + %s\n\",\"[...]\");exit}}" "${adb_dnsdir}/${adb_dnsfile}")" + printf "%s\n" "::: results for domain '${domain}'" printf "%s\n" "${result:-" - no match"}" domain="${tld}" tld="${domain#*.}" @@ -688,9 +721,6 @@ f_jsnup() if [ ${adb_backup_mode} -eq 1 ] then mode="normal/backup mode" - elif [ ${adb_whitelist_mode} -eq 1 ] - then - mode="whitelist mode" fi if [ -z "${adb_fetchinfo}" ] && [ -s "${adb_rtfile}" ] @@ -713,7 +743,7 @@ f_jsnup() json_dump > "${adb_rtfile}" if [ ${adb_notify} -eq 1 ] && [ -x /etc/adblock/adblock.notify ] && ([ "${status}" = "error" ] ||\ - ([ "${status}" = "enabled" ] && [ ${adb_whitelist_mode} -eq 0 ] && [ ${adb_cnt} -le ${adb_notifycnt} ])) + ([ "${status}" = "enabled" ] && [ ${adb_cnt} -le ${adb_notifycnt} ])) then (/etc/adblock/adblock.notify >/dev/null 2>&1) & bg_pid=${!} @@ -744,63 +774,53 @@ f_log() # f_main() { - local src_name src_rset src_log src_arc enabled url mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" + local tmp_load tmp_file src_name src_rset src_arc src_log mem_total mem_free enabled url cnt=1 - f_log "debug" "f_main ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, whitelist_mode: ${adb_whitelist_mode}, force_srt: ${adb_forcesrt}, force_dns: ${adb_forcedns}, mem_total: ${mem_total}" + mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)" + mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)" + tmp_load="${adb_tmpload}" + tmp_file="${adb_tmpfile}" > "${adb_dnsdir}/.${adb_dnsfile}" > "${adb_tmpdir}/tmp.whitelist" + f_log "debug" "f_main ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, dns_jail: ${adb_jail}, force_srt: ${adb_forcesrt}, force_dns: ${adb_forcedns}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${adb_maxqueue}" # prepare whitelist entries # if [ -s "${adb_whitelist}" ] then - if [ ${adb_whitelist_mode} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] - then - adb_whitelist_rset="\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - else - adb_whitelist_rset="\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\.\",\"\\\.\",\$1);print tolower(\"^\"\$1\"\\\|\\\.\"\$1)}" - fi + adb_whitelist_rset="/^([^([:space:]|\#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\.\",\"\\\.\",\$1);print tolower(\"^\"\$1\"\\\|\\\.\"\$1)}" awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" + f_list whitelist + if [ ${adb_jail} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] + then + adb_whitelist_rset="/^([^([:space:]|\#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" + awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.dnsjail" + fi fi - # whitelist mode processing + # build 'dnsjail' list # - if [ ${adb_whitelist_mode} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] + if [ ${adb_jail} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] then - f_hash - f_tld "${adb_tmpdir}/tmp.whitelist" - eval "${adb_dnsformat}" "${adb_tmpdir}/tmp.whitelist" > "${adb_dnsdir}/${adb_dnsfile}" - printf '%s\n' "${adb_dnsblock}" >> "${adb_dnsdir}/${adb_dnsfile}" + f_tld "${adb_tmpdir}/tmp.dnsjail" + eval "${adb_dnsallow}" "${adb_tmpdir}/tmp.dnsjail" > "/tmp/${adb_dnsjail}" + printf '%s\n' "${adb_dnshalt}" >> "/tmp/${adb_dnsjail}" if [ -n "${adb_dnsheader}" ] then - printf '%s\n' "${adb_dnsheader}" | cat - "${adb_dnsdir}/${adb_dnsfile}" > "${adb_tmpdir}/${adb_dnsfile}" - cat "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" - fi - f_hash - if [ ${?} -eq 1 ] - then - f_dnsup - fi - f_jsnup - if [ ${adb_rc} -eq 0 ] - then - f_log "info" "whitelist with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" - return 0 - else - f_log "err" "dns backend restart with active whitelist failed" + printf '%s\n' "${adb_dnsheader}" | cat - "/tmp/${adb_dnsjail}" > "${adb_tmpdir}/tmp.dnsjail" + cat "${adb_tmpdir}/tmp.dnsjail" > "/tmp/${adb_dnsjail}" fi fi - # normal & backup mode processing + # main loop # for src_name in ${adb_sources} do eval "enabled=\"\${enabled_${src_name}}\"" eval "url=\"\${adb_src_${src_name}}\"" eval "src_rset=\"\${adb_src_rset_${src_name}}\"" - > "${adb_tmpload}" - > "${adb_tmpfile}" - adb_rc=1 + adb_tmpload="${tmp_load}.${src_name}" + adb_tmpfile="${tmp_file}.${src_name}" # basic pre-checks # @@ -818,92 +838,146 @@ f_main() f_list restore if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] then - f_list merge + if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ] + then + f_tld "${adb_tmpfile}" + fi continue fi fi - # download blocklist + # download queue processing # if [ "${src_name}" = "blacklist" ] then if [ -s "${url}" ] then - cat "${url}" > "${adb_tmpload}" - adb_rc=${?} + ( + src_log="$(cat "${url}" > "${adb_tmpload}" 2>&1)" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] + then + awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] + then + rm -f "${adb_tmpload}" + f_list download + if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ] + then + f_tld "${adb_tmpfile}" + fi + fi + else + src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')" + f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}" + fi + ) & + else + continue fi elif [ "${src_name}" = "shalla" ] then - src_arc="${adb_tmpdir}/shallalist.tar.gz" - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${url}" 2>&1)" - adb_rc=${?} - if [ ${adb_rc} -eq 0 ] - then - for category in ${adb_src_cat_shalla} - do - tar -xOzf "${src_arc}" "BL/${category}/domains" >> "${adb_tmpload}" - adb_rc=${?} - if [ ${adb_rc} -ne 0 ] - then - break - fi - done - fi - rm -f "${src_arc}" - rm -rf "${adb_tmpdir}/BL" + ( + src_arc="${adb_tmpdir}"/shallalist.tar.gz + src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${url}" 2>&1)" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${src_arc}" ] + then + for category in ${adb_src_cat_shalla} + do + tar -xOzf "${src_arc}" "BL/${category}/domains" >> "${adb_tmpload}" + adb_rc=${?} + if [ ${adb_rc} -ne 0 ] + then + break + fi + done + else + src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')" + f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}" + fi + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] + then + rm -f "${src_arc}" + awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] + then + rm -f "${adb_tmpload}" + f_list download + if [ ${adb_backup} -eq 1 ] + then + f_list backup + fi + if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ] + then + f_tld "${adb_tmpfile}" + fi + elif [ ${adb_backup} -eq 1 ] + then + f_list restore + fi + elif [ ${adb_backup} -eq 1 ] + then + f_list restore + fi + ) & else - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>&1)" - adb_rc=${?} + ( + src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>&1)" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] + then + awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" + adb_rc=${?} + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] + then + rm -f "${adb_tmpload}" + f_list download + if [ ${adb_backup} -eq 1 ] + then + f_list backup + fi + if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ] + then + f_tld "${adb_tmpfile}" + fi + elif [ ${adb_backup} -eq 1 ] + then + f_list restore + fi + else + src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')" + f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}" + if [ ${adb_backup} -eq 1 ] + then + f_list restore + fi + fi + ) & fi - if [ ${adb_rc} -ne 0 ] && [ "${src_name}" != "blacklist" ] + hold=$(( cnt % adb_maxqueue )) + if [ ${hold} -eq 0 ] then - f_log "debug" "f_main ::: url: ${url}, rc: ${adb_rc}, src_log: ${src_log:-"-"}" - fi - - # check download result and prepare list output - # - if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] - then - awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" - if [ -s "${adb_tmpfile}" ] - then - if [ ${mem_total} -lt 64 ] && [ ${adb_forcesrt} -eq 0 ] - then - f_tld "${adb_tmpfile}" - fi - if [ ${adb_backup} -eq 1 ] && [ "${src_name}" != "blacklist" ] - then - f_list backup - fi - elif [ ${adb_backup} -eq 1 ] - then - f_list restore - fi - elif [ ${adb_backup} -eq 1 ] && [ "${src_name}" != "blacklist" ] - then - f_list restore - fi - - # list merge - # - if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] - then - f_list merge - if [ ${adb_rc} -ne 0 ] - then - f_list remove - fi - else - f_list remove + wait fi + cnt=$(( cnt + 1 )) done + # list merge + # + wait + src_name="overall" + adb_tmpfile="${tmp_file}" + f_list merge + # overall sort and conditional dns restart # f_hash if [ -s "${adb_tmpdir}/${adb_dnsfile}" ] then - if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] + if ([ ${mem_total} -ge 64 ] && [ ${mem_free} -ge 40 ]) || [ ${adb_forcesrt} -eq 1 ] then f_tld "${adb_tmpdir}/${adb_dnsfile}" fi @@ -912,14 +986,13 @@ f_main() > "${adb_dnsdir}/${adb_dnsfile}" fi chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null - f_hash if [ ${?} -eq 1 ] then f_dnsup fi f_jsnup - if [ ${adb_rc} -eq 0 ] + if [ ${?} -eq 0 ] then f_log "info" "blocklist with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" else