Merge pull request #1946 from dibdot/master

adblock: uci support, dynamic uhttpd instance support plus various fixes
This commit is contained in:
Ted Hess 2015-11-11 17:00:32 -05:00
commit 25c6b0818f
15 changed files with 790 additions and 622 deletions

View File

@ -7,7 +7,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=adblock
PKG_VERSION:=0.11.0
PKG_VERSION:=0.20.0
PKG_RELEASE:=1
PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
@ -40,7 +40,7 @@ The script supports the following domain blacklist sites:
endef
define Package/$(PKG_NAME)/conffiles
/etc/adblock/adblock.conf
/etc/config/adblock
/etc/adblock/adblock.whitelist
/etc/adblock/adblock.blacklist
endef
@ -56,15 +56,19 @@ endef
define Package/$(PKG_NAME)/install
$(INSTALL_DIR) $(1)/usr/bin
$(INSTALL_BIN) ./files/usr/bin/adblock-update.sh $(1)/usr/bin/
$(INSTALL_DATA) ./files/usr/bin/adblock-helper.sh $(1)/usr/bin/
$(INSTALL_BIN) ./files/adblock-update.sh $(1)/usr/bin/
$(INSTALL_DATA) ./files/adblock-helper.sh $(1)/usr/bin/
$(INSTALL_DIR) $(1)/etc/config
$(INSTALL_CONF) ./files/adblock.conf $(1)/etc/config/adblock
$(INSTALL_DIR) $(1)/etc/adblock
$(INSTALL_CONF) ./files/etc/adblock/adblock.* $(1)/etc/adblock/
$(INSTALL_CONF) ./files/etc/adblock/README.md $(1)/etc/adblock/
$(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock/
$(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/
$(INSTALL_CONF) ./files/README.md $(1)/etc/adblock/
$(INSTALL_DIR) $(1)/etc/adblock/samples
$(INSTALL_CONF) ./files/etc/adblock/samples/*.sample $(1)/etc/adblock/samples/
$(INSTALL_CONF) ./files/samples/*.sample $(1)/etc/adblock/samples/
$(INSTALL_DIR) $(1)/www/adblock
$(INSTALL_DATA) ./files/www/adblock/* $(1)/www/adblock/

View File

@ -5,7 +5,7 @@ A lot of people already use adblocker plugins within their desktop browsers,
but what if you are using your (smart) phone, tablet, watch or any other wlan gadget...
...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router.
When the dns server on your router receives dns requests, well sort out queries that ask for the [A] resource records of ad servers
When the dns server on your router receives dns requests, youll sort out queries that ask for the [A] resource records of ad servers
and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
## Main Features
@ -21,34 +21,35 @@ and return the local ip address of your router and the internal web server deliv
* [winhelp2002.mvps.org](http://winhelp2002.mvps.org)
* blocklist parsing by fast & flexible regex rulesets
* additional white- and blacklist support for manual overrides
* separate adblock loopback network interface (auto-install)
* separate uhttpd instance as pixel server (auto-install)
* separate dynamic adblock network interface
* separate dynamic uhttpd instance as pixel server
* optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service
* optional: adblock updates only on pre-defined interfaces
* optional: adblock updates only on pre-defined wan interfaces
* optional: domain query logging as a background service to easily identify free and already blocked domains
* optional: ntp time sync
* optional: status & error logging (req. ntp time sync)
## Prerequisites
* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work (please adjust *min_release* accordingly)
* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work
* additional software packages:
* curl
* wget (due to an openwrt bug still needed for certain https requests - see ticket #19621)
* busybox find with *-mtime* support (needed only for query logging/housekeeping, will be disabled if not found)
* busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found)
* optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices
* the above dependencies will be checked during package installation & script runtime, please check *logread -e "adblock"* for errors
## Usage
* select & install adblock package (*opkg install adblock*)
* configure /etc/adblock/adblock.conf to your needs
* configure /etc/config/adblock to your needs
* start /usr/bin/adblock-update.sh and check *logread -e "adblock"* for errors
## Distributed samples
* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see */etc/adblock/samples/dhcp.config.sample*.
* for script autostart via rc.local and /tmp resizing on the fly see */etc/adblock/samples/rc.local.sample*.
* for scheduled call of *adblock-update.sh* see */etc/adblock/samples/root.crontab.sample*.
* to redirect/force all dns queries to your router see */etc/adblock/samples/firwall.user.sample*.
* for further dnsmasq tweaks see */etc/adblock/samples/dnsmasq.conf.sample*.
* all sample configuration files stored in */etc/adblock/samples*.
* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*.
* for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*.
* for scheduled call of *adblock-update.sh* see *root.crontab.sample*.
* to redirect/force all dns queries to your router see *firwall.user.sample*.
* for further dnsmasq tweaks see *dnsmasq.conf.sample*.
Have fun!
Dirk

View File

@ -0,0 +1,566 @@
#!/bin/sh
##############################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
##############################################
#####################################
# f_envload: load adblock environment
#
f_envload()
{
# source in openwrt function library
#
if [ -r "/lib/functions.sh" ]
then
. /lib/functions.sh
else
/usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found"
f_deltemp
exit 10
fi
# source in openwrt json helpers library
#
if [ -r "/usr/share/libubox/jshn.sh" ]
then
. "/usr/share/libubox/jshn.sh"
else
/usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found"
f_deltemp
exit 15
fi
# get list with all installed openwrt packages
#
pkg_list="$(opkg list-installed 2>/dev/null)"
if [ -z "${pkg_list}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list"
f_deltemp
exit 20
fi
}
######################################################
# f_envparse: parse adblock config and set environment
#
f_envparse()
{
# function to read/set global options by callback,
# prepare list items and build option list for all others
#
config_cb()
{
local type="${1}"
local name="${2}"
if [ "${type}" = "adblock" ]
then
option_cb()
{
local option="${1}"
local value="${2}"
eval "${option}=\"${value}\""
}
else
option_cb()
{
local option="${1}"
local value="${2}"
local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p')"
if [ -z "${opt_out}" ]
then
all_options="${all_options} ${option}"
fi
}
list_cb()
{
local list="${1}"
local value="${2}"
if [ "${list}" = "adb_wanlist" ]
then
adb_wandev="${adb_wandev} ${value}"
elif [ "${list}" = "adb_ntplist" ]
then
adb_ntpsrv="${adb_ntpsrv} ${value}"
elif [ "${list}" = "adb_catlist" ]
then
adb_cat_shalla="${adb_cat_shalla} ${value}"
fi
}
fi
}
# function to iterate through option list, read/set all options in "enabled" sections
#
parse_config()
{
local config="${1}"
config_get switch "${config}" "enabled"
if [ "${switch}" = "1" ]
then
for option in ${all_options}
do
config_get value "${config}" "${option}"
if [ -n "${value}" ]
then
local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p')"
if [ -n "${opt_src}" ]
then
adb_sources="${adb_sources} ${value}"
else
eval "${option}=\"${value}\""
fi
fi
done
elif [ "${config}" = "wancheck" ]
then
unset adb_wandev
elif [ "${config}" = "ntpcheck" ]
then
unset adb_ntpsrv
elif [ "${config}" = "shalla" ]
then
unset adb_cat_shalla
fi
}
# load adblock config and start parsing functions
#
config_load adblock
config_foreach parse_config service
config_foreach parse_config source
# set temp variables and counter
#
adb_tmpfile="$(mktemp -tu)"
adb_tmpdir="$(mktemp -d)"
cnt=0
max_cnt=30
max_time=60
# set adblock source ruleset definitions
#
rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
rset_end="sed '/^[#/:;_\s]*$/d'"
rset_default="${rset_start} | ${rset_end}"
rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
# set adblock/dnsmasq destination file and format
#
adb_dnsfile="/tmp/dnsmasq.d/adlist.conf"
adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
}
#############################################
# f_envcheck: check environment prerequisites
#
f_envcheck()
{
# check adblock network device configuration
#
if [ ! -d "/sys/class/net/${adb_dev}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})"
f_deltemp
exit 25
fi
# check adblock network interface configuration
#
check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')"
banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})"
f_deltemp
exit 30
fi
# check adblock ip address configuration
#
check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
if [ -z "${check_ip}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})"
f_deltemp
exit 35
fi
# check adblock blacklist/whitelist configuration
#
if [ ! -r "${adb_blacklist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found"
f_deltemp
exit 40
elif [ ! -r "${adb_whitelist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found"
f_deltemp
exit 45
fi
# check wan update configuration
#
if [ -n "${adb_wandev}" ]
then
wan_ok="true"
else
wan_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled"
fi
# check ntp sync configuration
#
if [ -n "${adb_ntpsrv}" ]
then
ntp_ok="true"
else
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled"
fi
# check backup configuration
#
adb_backupdir="${adb_backupfile%/*}"
if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
then
backup_ok="true"
adb_mounts="${adb_backupdir} ${adb_tmpdir}"
else
backup_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled"
fi
# check error log configuration
#
adb_logdir="${adb_logfile%/*}"
if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ]
then
log_ok="true"
adb_logfile="/proc/self/fd/1"
elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ]
then
log_ok="true"
adb_mounts="${adb_mounts} ${adb_logdir}"
else
log_ok="false"
adb_logfile="/dev/null"
/usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled"
fi
# check dns query log configuration
#
adb_querydir="${adb_queryfile%/*}"
if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
then
# check find capabilities
#
check="$(find --help 2>&1 | grep "mtime")"
if [ -z "${check}" ]
then
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
else
query_ok="true"
query_name="${adb_queryfile##*/}"
query_ip="${adb_ip//./\\.}"
query_pid="/var/run/query.pid"
adb_mounts="${adb_mounts} ${adb_querydir}"
fi
else
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled"
fi
# check mount points & space requirements
#
adb_mounts="${adb_mounts} ${adb_tmpdir}"
for mp in ${adb_mounts}
do
df "${mp}" 2>/dev/null |\
tail -n1 |\
while read filesystem overall used available scrap
do
av_space="${available}"
if [ $((av_space)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})"
exit 50
elif [ $((av_space)) -lt $((adb_minspace)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: not enough space on device (${mp})"
exit 55
fi
done
# subshell return code handling
#
rc=$?
if [ $((rc)) -ne 0 ]
then
f_deltemp
exit ${rc}
fi
done
# check curl package dependency
#
check="$(printf "${pkg_list}" | grep "^curl")"
if [ -z "${check}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: curl package not found"
f_deltemp
exit 60
fi
# check wget package dependency
#
check="$(printf "${pkg_list}" | grep "^wget")"
if [ -z "${check}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: wget package not found"
f_deltemp
exit 65
fi
# check dynamic/volatile adblock network interface configuration
#
rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
json_init
json_add_string name "${adb_if}"
json_add_string ifname "${adb_dev}"
json_add_string proto "static"
json_add_array ipaddr
json_add_string "" "${adb_ip}"
json_close_array
json_close_object
ubus call network add_dynamic "$(json_dump)"
/usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
fi
# check dynamic/volatile adblock uhttpd instance configuration
#
rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80"
/usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
fi
}
###################################################
# f_deltemp: delete temporary files and directories
#
f_deltemp()
{
if [ -f "${adb_tmpfile}" ]
then
rm -f "${adb_tmpfile}" 2>/dev/null
fi
if [ -d "${adb_tmpdir}" ]
then
rm -rf "${adb_tmpdir}" 2>/dev/null
fi
}
################################################################
# f_remove: remove temporary files, start and maintain query log
#
f_remove()
{
# delete temporary files and directories
#
f_deltemp
# kill existing domain query log background process,
# housekeeping and start of a new process on daily basis
#
if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
query_date="$(date "+%Y%m%d")"
if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
then
kill -9 $(< "${query_pid}") 2>/dev/null
> "${query_pid}"
find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "info: kill old query log background process and do logfile housekeeping"
fi
if [ ! -s "${query_pid}" ]
then
( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
/usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process"
fi
fi
# final log entry
#
/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})"
}
#####################################################
# f_restore: if available, restore last adlist backup
#
f_restore()
{
if [ -z "${restore_msg}" ]
then
restore_msg="unknown"
fi
if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ]
then
cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}"
else
> "${adb_dnsfile}"
/usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}"
fi
# restart dnsmasq
#
/etc/init.d/dnsmasq restart >/dev/null 2>&1
# remove files and exit
#
f_remove
exit 100
}
#######################################################
# f_wancheck: check for usable adblock update interface
#
f_wancheck()
{
if [ "${wan_ok}" = "true" ]
then
# wait for wan update interface(s)
#
while [ $((cnt)) -le $((max_cnt)) ]
do
for dev in ${adb_wandev}
do
if [ -d "/sys/class/net/${dev}" ]
then
dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null)
if [ "${dev_out}" = "up" ]
then
/usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops"
break 2
fi
fi
if [ $((cnt)) -eq $((max_cnt)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}"
restore_msg="no wan/update interface(s)"
f_restore
fi
done
sleep 1
cnt=$((cnt + 1))
done
fi
}
#####################################
# f_ntpcheck: check/get ntp time sync
#
f_ntpcheck()
{
if [ "${ntp_ok}" = "true" ]
then
# prepare ntp server pool
#
unset ntp_pool
for srv in ${adb_ntpsrv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
# wait for ntp time sync
#
while [ $((cnt)) -le $((max_cnt)) ]
do
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops"
break
fi
if [ $((cnt)) -eq $((max_cnt)) ]
then
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}"
restore_msg="time sync failed"
f_restore
fi
sleep 1
cnt=$((cnt + 1))
done
fi
}
#################################################################
# f_dnscheck: dnsmasq health check with newly generated blocklist
#
f_dnscheck()
{
# check 1: dnsmasq startup
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
if [ -z "${dns_status}" ]
then
# check 2: nslookup probe
#
dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")"
if [ -z "${dns_status}" ]
then
# create backup of new block list only, if both checks are OK and backup enabled
#
if [ "${backup_ok}" = "true" ]
then
cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated"
else
/usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup"
fi
else
restore_msg="nslookup probe failed"
f_restore
fi
else
restore_msg="dnsmasq probe failed"
f_restore
fi
}
##########################################################
# f_footer: write footer with a few statistics to dns file
#
f_footer()
{
adb_count="$(wc -l < "${adb_dnsfile}")"
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}"
printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}"
for src in ${adb_sources}
do
url="${src//\&ruleset=*/}"
printf "%s\n" "# ${url}" >> "${adb_dnsfile}"
done
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}"
printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}"
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
}

View File

@ -25,38 +25,21 @@
# set script version
#
script_ver="0.11.0"
adb_version="0.20.0"
# get current pid and script directory
#
pid=$$
script_dir="$(printf "${0}" | sed 's/\(.*\)\/.*/\1/')"
# set temp variables
#
tmp_file="$(mktemp -tu)"
tmp_dir="$(mktemp -d)"
# source in adblock configuration
#
if [ -r "/etc/adblock/adblock.conf" ]
then
. "/etc/adblock/adblock.conf"
else
/usr/bin/logger -t "adblock[${pid}]" "adblock configuration not found"
rm -rf "${tmp_dir}" 2>/dev/null
exit 200
fi
adb_scriptdir="${0%/*}"
# source in adblock function library
#
if [ -r "${script_dir}/adblock-helper.sh" ]
if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
then
. "${script_dir}/adblock-helper.sh"
. "${adb_scriptdir}/adblock-helper.sh"
else
/usr/bin/logger -t "adblock[${pid}]" "adblock function library not found"
rm -rf "${tmp_dir}" 2>/dev/null
exit 210
/usr/bin/logger -t "adblock[${pid}]" "error: adblock function library not found"
exit 200
fi
################
@ -69,7 +52,15 @@ trap "restore_msg='trap error'; f_restore" 1 2 3 10 11 15
# start logging
#
/usr/bin/logger -t "adblock[${pid}]" "domain adblock processing started (${script_ver})"
/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing started (${adb_version})"
# load environment
#
f_envload
# parse environment
#
f_envparse
# check environment
#
@ -83,32 +74,36 @@ f_wancheck
#
f_ntpcheck
# download shallalist archive
# check/start shallalist (pre-)processing
#
if [ "${shalla_ok}" = "true" ]
if [ -n "${adb_arc_shalla}" ]
then
curl --insecure --max-time "${max_time}" "${shalla_url}" -o "${shalla_archive}" 2>/dev/null
# download shallalist archive
#
shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
shalla_file="${adb_tmpdir}/shallalist.txt"
curl --insecure --max-time "${max_time}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "shallalist archive download finished"
/usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive download finished"
else
/usr/bin/logger -t "adblock[${pid}]" "shallalist archive download failed (${shalla_url})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${shalla_url})" >> "${log_file}"
/usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive download failed (${adb_arc_shalla})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${adb_arc_shalla})" >> "${adb_logfile}"
restore_msg="archive download failed"
f_restore
fi
# extract shallalist archive
#
tar -xzf "${shalla_archive}" -C "${tmp_dir}" 2>/dev/null
tar -xzf "${shalla_archive}" -C "${adb_tmpdir}" 2>/dev/null
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction finished"
/usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive extraction finished"
else
/usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction failed"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${log_file}"
/usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive extraction failed"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${adb_logfile}"
restore_msg="archive extraction failed"
f_restore
fi
@ -116,11 +111,11 @@ then
# merge selected shallalist categories
#
> "${shalla_file}"
for category in ${shalla_cat}
for category in ${adb_cat_shalla}
do
if [ -f "${tmp_dir}/BL/${category}/domains" ]
if [ -f "${adb_tmpdir}/BL/${category}/domains" ]
then
cat "${tmp_dir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null
cat "${adb_tmpdir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null
rc=$?
else
rc=220
@ -135,23 +130,25 @@ then
#
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "shallalist (pre-)processing finished (${shalla_cat})"
adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla"
/usr/bin/logger -t "adblock[${pid}]" "info: shallalist (pre-)processing finished (${adb_cat_shalla})"
else
/usr/bin/logger -t "adblock[${pid}]" "shallalist category merge failed (${rc}, ${shalla_cat})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist category merge failed (${rc}, ${shalla_cat})" >> "${log_file}"
/usr/bin/logger -t "adblock[${pid}]" "error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" >> "${adb_logfile}"
restore_msg="shallalist merge failed"
f_restore
fi
fi
# loop through domain source list,
# loop through active adblock domain sources,
# prepare output and store all extracted domains in temp file
#
for src in ${adb_source}
adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_default"
for src in ${adb_sources}
do
# download selected adblock sources
#
url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')"
url="${src//\&ruleset=*/}"
check_url="$(printf "${url}" | sed -n '/^https:/p')"
if [ -n "${check_url}" ]
then
@ -166,16 +163,16 @@ do
#
if [ $((rc)) -eq 0 ] && [ -n "${tmp_var}" ]
then
eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$rset_/g')"
eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[[:upper:]]' '[[:lower:]]')"
adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${tmp_file}" | wc -l)"
/usr/bin/logger -t "adblock[${pid}]" "source download finished (${url}, ${adb_count} entries)"
adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)"
/usr/bin/logger -t "adblock[${pid}]" "info: source download finished (${url}, ${adb_count} entries)"
elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "empty source download finished (${url})"
/usr/bin/logger -t "adblock[${pid}]" "info: empty source download finished (${url})"
else
/usr/bin/logger -t "adblock[${pid}]" "source download failed (${url})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${log_file}"
/usr/bin/logger -t "adblock[${pid}]" "error: source download failed (${url})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${adb_logfile}"
restore_msg="download failed"
f_restore
fi
@ -183,12 +180,12 @@ done
# create empty destination file
#
> "${dns_file}"
> "${adb_dnsfile}"
# rewrite ad/abuse domain information to dns file,
# remove duplicates and whitelist entries
#
grep -vxf "${adb_whitelist}" < "${tmp_file}" | eval "${dns_format}" | sort -u 2>/dev/null >> "${dns_file}"
grep -vxf "${adb_whitelist}" < "${adb_tmpfile}" | eval "${adb_dnsformat}" | sort -u 2>/dev/null >> "${adb_dnsfile}"
# write dns file footer
#

View File

@ -0,0 +1,76 @@
# adblock configuration
#
config adblock "global"
option adb_ip "192.168.2.1"
option adb_dev "eth0"
option adb_if "adblock"
option adb_domain "heise.de"
option adb_minspace "100000"
option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
list adb_ntplist "0.pool.ntp.org"
list adb_ntplist "1.pool.ntp.org"
list adb_ntplist "2.pool.ntp.org"
list adb_ntplist "3.pool.ntp.org"
config service "backup"
option enabled "0"
option adb_backupfile "/tmp/adlist.backup"
config service "errorlog"
option enabled "0"
option adb_logfile "/tmp/error.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/query.log"
option adb_queryhistory "1"
config source "yoyo"
option enabled "1"
option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
config source "zeus"
option enabled "0"
option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
config source "feodo"
option enabled "0"
option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
config source "palevo"
option enabled "0"
option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
config source "dshield"
option enabled "0"
option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
config source "spam404"
option enabled "0"
option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
config source "winhelp"
option enabled "0"
option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
config source "shalla"
option enabled "0"
option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
list adb_catlist "adv"
list adb_catlist "costtraps"
list adb_catlist "downloads"
list adb_catlist "spyware"
list adb_catlist "tracker"
list adb_catlist "warez"

View File

@ -1,118 +0,0 @@
###################################################
# adblock configuration used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
###################################################
# adblock ip address and device information
# adb_ip => ip address of the local adblock loopback interface/uhttpd instance
# adb_dev => name of the physical adblock network device
# adb_if => name of the adblock network interface and uhttpd instance
#
adb_ip="192.168.2.1"
adb_dev="eth0"
adb_if="adblock"
# backup file and default restore message
# backup_dir => full path to directory for logging and backups, an empty entry disables this feature
# backup_file => full path to backup file
# restore_msg => default restore message
#
backup_dir=""
backup_file="${backup_dir}/adlist.backup"
restore_msg="unknown error"
# wan/update interface(s)
# wan_dev => space separated list of devices that are allowed for adblock updates (check /sys/class/net/<dev>),
# an empty entry disables this feature
#
wan_dev=""
# local adblock black & white list for manual overrides
# adb_blacklist => full path to domain blacklist file
# adb_whitelist => full path to domain whitelist file
#
adb_blacklist="/etc/adblock/adblock.blacklist"
adb_whitelist="/etc/adblock/adblock.whitelist"
# error logfile
# log_file => full path to additional logfile for error logging,
# "/dev/stdout" redirect errors to console, an empty entry disables this feature
# this feature requires ntp time sync too (see below)
#
log_file="${backup_dir}/error.log"
# domain query logfile
# query_file => full path to additional domain query logfile,
# an empty entry or an empty backup_dir entry disables this feature
# query_history => delete query logfiles older than n days (req. busybox find with mtime support)
#
query_file="${backup_dir}/query.log"
query_history=1
query_name="$(printf "${query_file}" | sed 's/.*\///g')"
query_ip="$(printf "${adb_ip}" | sed 's/\./\\./g')"
query_pid="/var/run/query.pid"
# ntp time server pool
# ntp_srv => space separated list of ntp serverpools, an empty entry disables this feature
#
ntp_srv="ntp0.fau.de ntp1.fau.de ntp2.fau.de ntp3.fau.de"
# domain for nslookup probe
# check_domain => check the newly generated domain blocklist with this domain
#
check_domain="heise.de"
# set startup counter and download timeout defaults
# max_cnt => wait n seconds/loops for wan & ntp check in sum
# max_time => wait n seconds for every source download
#
cnt=0
max_cnt=30
max_time=60
# minimum values for environment checks
# min_release => minimum required openwrt release number
# min_space => minimum required space for backups & logfiles (in kbyte)
#
min_release=47025
min_space=100000
# shallalist url, categories and local naming
# shalla_cat => space separated list of categories to use from shallalist archive
#
shalla_cat="adv costtraps downloads spyware tracker warez"
shalla_url="http://www.shallalist.de/Downloads/shallalist.tar.gz"
shalla_archive="${tmp_dir}/shallalist.tar.gz"
shalla_file="${tmp_dir}/shallalist.txt"
# remote and local domain block list sources
# adb_source => comment out sources you don't want to use
#
unset adb_source
adb_source="${adb_source} http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=yoyo"
#adb_source="${adb_source} http://mirror1.malwaredomains.com/files/justdomains&ruleset=default"
#adb_source="${adb_source} https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=default"
#adb_source="${adb_source} file:///${shalla_file}&ruleset=shalla"
#adb_source="${adb_source} http://spam404bl.com/spam404scamlist.txt&ruleset=spam404"
#adb_source="${adb_source} http://winhelp2002.mvps.org/hosts.txt&ruleset=winhelp"
adb_source="${adb_source} file://${adb_blacklist}&ruleset=default"
# adblock source ruleset definitions
# note: please keep the default settings below
#
rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
rset_end="sed '/^[#/:;_\s]*$/d'"
rset_default="${rset_start} | ${rset_end}"
rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
# dnsmasq destination file and format
# note: please keep the default settings below
#
dns_file="/tmp/dnsmasq.d/adlist.conf"
dns_format="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"

View File

@ -0,0 +1,76 @@
# adblock configuration
#
config adblock "global"
option adb_ip "192.168.2.1"
option adb_dev "eth0"
option adb_if "adblock"
option adb_domain "heise.de"
option adb_minspace "100000"
option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
list adb_ntplist "0.pool.ntp.org"
list adb_ntplist "1.pool.ntp.org"
list adb_ntplist "2.pool.ntp.org"
list adb_ntplist "3.pool.ntp.org"
config service "backup"
option enabled "0"
option adb_backupfile "/tmp/adlist.backup"
config service "errorlog"
option enabled "0"
option adb_logfile "/tmp/error.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/query.log"
option adb_queryhistory "1"
config source "yoyo"
option enabled "1"
option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
config source "zeus"
option enabled "0"
option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
config source "feodo"
option enabled "0"
option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
config source "palevo"
option enabled "0"
option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
config source "dshield"
option enabled "0"
option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
config source "spam404"
option enabled "0"
option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
config source "winhelp"
option enabled "0"
option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
config source "shalla"
option enabled "0"
option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
list adb_catlist "adv"
list adb_catlist "costtraps"
list adb_catlist "downloads"
list adb_catlist "spyware"
list adb_catlist "tracker"
list adb_catlist "warez"

View File

@ -1,434 +0,0 @@
##############################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
##############################################
#############################################
# f_envcheck: check environment prerequisites
#
f_envcheck()
{
# source in json helpers library
#
if [ -r "/usr/share/libubox/jshn.sh" ]
then
. "/usr/share/libubox/jshn.sh"
else
/usr/bin/logger -t "adblock[${pid}]" "json helpers library not found"
f_deltemp
exit 10
fi
# check adblock network device configuration
#
if [ ! -d "/sys/class/net/${adb_dev}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock network device input (${adb_dev})"
f_deltemp
exit 15
fi
# check adblock network interface configuration
#
check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')"
banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock network interface input (${adb_if})"
f_deltemp
exit 20
fi
# check adblock ip address configuration
#
check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
if [ -z "${check_ip}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock ip address input (${adb_ip})"
f_deltemp
exit 25
fi
# check adblock blacklist/whitelist configuration
#
if [ ! -r "${adb_blacklist}" ] || [ ! -r "${adb_whitelist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "adblock blacklist or whitelist not found"
f_deltemp
exit 30
fi
# check wan update configuration
#
if [ -n "${wan_dev}" ]
then
wan_ok="true"
else
wan_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "wan update check will be disabled"
fi
# check ntp sync configuration
#
if [ -n "${ntp_srv}" ]
then
ntp_ok="true"
else
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "ntp time sync will be disabled"
fi
# check backup configuration
#
if [ -n "${backup_dir}" ] && [ -d "${backup_dir}" ]
then
backup_ok="true"
mounts="${backup_dir} ${tmp_dir}"
else
backup_ok="false"
mounts="${tmp_dir}"
/usr/bin/logger -t "adblock[${pid}]" "backup/restore will be disabled"
fi
# check error log configuration
#
if [ "${log_file}" = "/dev/stdout" ]
then
log_ok="true"
log_file="/proc/self/fd/1"
elif [ -n "${log_file}" ] && [ "${backup_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
log_ok="true"
else
log_ok="false"
log_file="/dev/null"
/usr/bin/logger -t "adblock[${pid}]" "error logging will be disabled"
fi
# check dns query log configuration
#
if [ -n "${query_file}" ] && [ "${backup_ok}" = "true" ]
then
# check find capabilities
#
base="$(find --help 2>&1 | grep "mtime")"
if [[ -z "${base}" ]]
then
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "no 'find/mtime' support, dns query logging will be disabled"
else
query_ok="true"
fi
else
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "dns query logging will be disabled"
fi
# check shallalist configuration
#
check_shalla="$(printf "${adb_source}" | sed -n '/.*shallalist.txt.*/p')"
if [ -n "${check_shalla}" ]
then
shalla_ok="true"
else
shalla_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "shallalist processing will be disabled"
fi
# check mount points & space requirements
#
for mp in ${mounts}
do
df "${mp}" 2>/dev/null |\
tail -n1 |\
while read filesystem overall used available scrap
do
av_space="${available}"
if [ $((av_space)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "no space left on device, not mounted (${mp})"
exit 35
elif [ $((av_space)) -lt $((min_space)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "not enough space on device (${mp})"
exit 40
fi
done
# subshell return code handling
#
rc=$?
if [ $((rc)) -ne 0 ]
then
f_deltemp
exit ${rc}
fi
done
# get list with all installed packages
#
pkg_list="$(opkg list-installed 2>/dev/null)"
# check openwrt release
#
base="$(printf "${pkg_list}" | grep "^base-files" | sed 's/\(.*r\)//g')"
if [ $((base)) -lt $((min_release)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "openwrt (r${wrt_release}) seems to be too old"
f_deltemp
exit 45
fi
# check curl package dependency
#
base="$(printf "${pkg_list}" | grep "^curl")"
if [ -z "${base}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "curl package not found"
f_deltemp
exit 50
fi
# check wget package dependency
#
base="$(printf "${pkg_list}" | grep "^wget")"
if [ -z "${base}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "wget package not found"
f_deltemp
exit 55
fi
# check dynamic/volatile adblock network interface configuration
#
rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
json_init
json_add_string name "${adb_if}"
json_add_string ifname "${adb_dev}"
json_add_string proto "static"
json_add_array ipaddr
json_add_string "" "${adb_ip}"
json_close_array
json_close_object
ubus call network add_dynamic "$(json_dump)"
/usr/bin/logger -t "adblock[${pid}]" "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
fi
# check adblock uhttpd instance configuration
#
if [ -z "$(uci -q get uhttpd.${adb_if} 2>/dev/null)" ]
then
uci -q set uhttpd.${adb_if}="uhttpd"
uci -q set uhttpd.${adb_if}.listen_http="${adb_ip}:80"
uci -q set uhttpd.${adb_if}.home="/www/adblock"
uci -q set uhttpd.${adb_if}.error_page="/adblock.html"
uci -q commit uhttpd
/etc/init.d/uhttpd reload
/usr/bin/logger -t "adblock[${pid}]" "created new uhttpd instance (${adb_if}, ${adb_ip}) in /etc/config/uhttpd"
fi
}
###################################################
# f_deltemp: delete temporary files and directories
f_deltemp()
{
if [ -f "${tmp_file}" ]
then
rm -f "${tmp_file}" 2>/dev/null
fi
if [ -d "${tmp_dir}" ]
then
rm -rf "${tmp_dir}" 2>/dev/null
fi
}
################################################################
# f_remove: remove temporary files, start and maintain query log
#
f_remove()
{
# delete temporary files and directories
#
f_deltemp
# kill existing domain query log background process,
# housekeeping and start of a new process on daily basis
#
if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
query_date="$(date "+%Y%m%d")"
if [ -s "${query_pid}" ] && [ ! -f "${query_file}.${query_date}" ]
then
kill -9 $(< "${query_pid}") 2>/dev/null
> "${query_pid}"
find "${backup_dir}" -maxdepth 1 -type f -mtime +${query_history} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "kill old query log background process and do logfile housekeeping"
fi
if [ ! -s "${query_pid}" ]
then
( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${query_file}.${query_date}" &
/usr/bin/logger -t "adblock[${pid}]" "start new domain query log background process"
fi
fi
# final log entry
#
/usr/bin/logger -t "adblock[${pid}]" "domain adblock processing finished (${script_ver})"
}
#####################################################
# f_restore: if available, restore last adlist backup
#
f_restore()
{
if [ "${backup_ok}" = "true" ] && [ -f "${backup_file}" ]
then
cp -f "${backup_file}" "${dns_file}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, adlist backup restored"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${log_file}"
else
> "${dns_file}"
/usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, empty adlist generated"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${log_file}"
fi
# restart dnsmasq
#
/etc/init.d/dnsmasq restart >/dev/null 2>&1
# remove files and exit
#
f_remove
exit 100
}
#######################################################
# f_wancheck: check for usable adblock update interface
#
f_wancheck()
{
if [ "${wan_ok}" = "true" ]
then
# wait for wan update interface(s)
#
while [ $((cnt)) -le $((max_cnt)) ]
do
for dev in ${wan_dev}
do
dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null)
if [[ "${dev_out}" = "up" ]]
then
/usr/bin/logger -t "adblock[${pid}]" "get wan/update interface: ${dev}, after ${cnt} loops"
break 2
elif [ $((cnt)) -eq $((max_cnt)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "no wan/update interface(s) found (${wan_dev})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${wan_dev})" >> "${log_file}"
restore_msg="no wan/update interface(s)"
f_restore
fi
done
sleep 1
cnt=$((cnt + 1))
done
fi
}
#####################################
# f_ntpcheck: check/get ntp time sync
#
f_ntpcheck()
{
if [ "${ntp_ok}" = "true" ]
then
# prepare ntp server pool
#
unset ntp_pool
for srv in ${ntp_srv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
# wait for ntp time sync
#
while [ $((cnt)) -le $((max_cnt)) ]
do
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "get ntp time sync (${ntp_srv}), after ${cnt} loops"
break
elif [ $((cnt)) -eq $((max_cnt)) ]
then
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "ntp time sync failed (${ntp_srv})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${ntp_srv})" >> "${log_file}"
restore_msg="time sync failed"
f_restore
fi
sleep 1
cnt=$((cnt + 1))
done
fi
}
#################################################################
# f_dnscheck: dnsmasq health check with newly generated blocklist
#
f_dnscheck()
{
# check 1: dnsmasq startup
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
if [ -z "${dns_status}" ]
then
# check 2: nslookup probe
#
dns_status="$(nslookup "${check_domain}" 2>/dev/null | grep "${adb_ip}")"
if [ -z "${dns_status}" ]
then
# create backup of new block list only, if both checks are OK and backup enabled
#
if [ "${backup_ok}" = "true" ]
then
cp -f "${dns_file}" "${backup_file}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded, backup generated"
else
/usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded"
fi
else
restore_msg="nslookup probe failed"
f_restore
fi
else
restore_msg="dnsmasq probe failed"
f_restore
fi
}
##########################################################
# f_footer: write footer with a few statistics to dns file
#
f_footer()
{
# count result of merged domain entries
#
adb_count="$(wc -l < "${dns_file}")"
# write file footer with timestamp and merged ad count sum
#
printf "%s\n" "###################################################" >> "${dns_file}"
printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${dns_file}"
printf "%s\n" "# ${0##*/} (${script_ver}) - ${adb_count} ad/abuse domains blocked" >> "${dns_file}"
printf "%s\n" "# domain blacklist sources:" >> "${dns_file}"
for src in ${adb_source}
do
url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')"
printf "%s\n" "# ${url}" >> "${dns_file}"
done
printf "%s\n" "###################################################" >> "${dns_file}"
printf "%s\n" "# domain whitelist source:" >> "${dns_file}"
printf "%s\n" "# ${adb_whitelist}" >> "${dns_file}"
printf "%s\n" "###################################################" >> "${dns_file}"
}