mwan3: Fix packet routing when WAN interface is partially up

This introduces a new concept of "unknown_wan" to mwan3.  The action for
this can be configured in the globals section the default of which is
'none'.  This can be set to 'none', 'default', 'unreachable' or 'blacklist'
switching out the matching ip rule for this match.  This assignment for
a connection is temporary and is re-resolved for each additional
original direction packet through the firewall allowing the unknown WAN
to start resolving once the ifup has finished for the given interface.

An example configuration:

	config globals 'globals'
			option unknown_wan_action 'unreachable'

Prior to this commit, mwan3 had multiple hit spots for packets in the
following order:

 1. Packets are checked to see if they originate from known WAN interfaces
 2. Packets are checked to see if they destined for ipsets defined
 3. Packets are checked against default WAN policies

The WAN list is maintained via hotplug 'ifup'/'ifdown' events and the local
route ipset list is maintained via monitoring the routing table.  This
means that while a WAN interface is brought up, the list for 2 is
updated before the list for 1, since an interface is fully brought up
before the ifup event is fired off.  Additionally, we want to make sure we
don't apply a WAN policy for incoming packets from a WAN interface that
is in the process of being brought up.

We can identify packets that are presumably coming from a WAN interface
we don't recognize yet by eliminating all packets that the source comes
from networks we don't know about in the ipsets that mwan3 manages.  We
have to be careful here to only match the original direction of the
packet flow (e.g. for instance with ICMP, the ping request is in the
ORIGINAL direction, and the response is in the REPLY direction) or else
we might match something we didn't intend to.

By modifying the rule set to the following:

 1. Packets are checked to see if they are in a REPLY direction of flow
 2. Packets are checked to see if they originate from known WAN interfaces
 3. Packets are checked to see if they not sourced from ipsets defined
 4. Packets are checked to see if they destined for ipsets defined
 5. Packets are checked against default WAN policies

If a packet is in the REPLY direction of flow, we definitely don't want
to do any routing table assignments - we only want to do this for the
original direction of traffic flow.  This reduces the amount of rules
parsed within mwan3.

If a packet is not sourced from a defined ipset, this should match any
packet originating from a "default route" upstream.  We do this post the
known WAN interface check since we don't know what mask to apply to this
packet at this time until the 'ifup' has completed.  It's also setup to
reevaluate this decision by clearing this specific mark when a new
packet comes in in the REPLY direction of flow before any subsequent
evaluations.  This allows additional packets for the same connection to
eventually be assigned the appropriate mask once the 'ifup' has
finished.

One easy way to test this out before and after this change is to:

 - Bring down wan (e.g. ifdown wan)
 - Manually bring up WAN
    - This mitigates the firewall rules being added for 1 above, but 2
      is still added since this is monitoring the routing interface
 - Ping the device from a non-local subnet via the WAN interface; leave
   running
 - Observe mark set to ICMP session via conntrack
 - Bring up wan (e.g. ifup wan)
 - Observe mark set to ICMP session from above

Signed-off-by: Tim Nordell <tnordell@airgain.com>

net/mwan3/files/etc/config/mwan3

@@ -3,6 +3,7 @@
 
 config globals 'globals'
 	option mmx_mask '0x3F00'
+	option unknown_wan_action 'none'
 
 config interface 'wan'
 	option enabled '1'

net/mwan3/files/lib/mwan3/common.sh

@@ -17,6 +17,9 @@ MM_BLACKHOLE=""
 
 MMX_UNREACHABLE=""
 MM_UNREACHABLE=""
+
+MMX_UNKNOWN_WAN=""
+MM_UNKNOWN_WAN=""
 MAX_SLEEP=$(((1<<31)-1))
 
 command -v ip6tables > /dev/null
@@ -147,11 +150,13 @@ mwan3_init()
 	mmdefault=$(((1<<bitcnt)-1))
 	MM_BLACKHOLE=$((mmdefault-2))
 	MM_UNREACHABLE=$((mmdefault-1))
+	MM_UNKNOWN_WAN=$((mmdefault-3))
 
 	# MMX_DEFAULT should equal MMX_MASK
 	MMX_DEFAULT=$(mwan3_id2mask mmdefault MMX_MASK)
 	MMX_BLACKHOLE=$(mwan3_id2mask MM_BLACKHOLE MMX_MASK)
 	MMX_UNREACHABLE=$(mwan3_id2mask MM_UNREACHABLE MMX_MASK)
+	MMX_UNKNOWN_WAN=$(mwan3_id2mask MM_UNKNOWN_WAN MMX_MASK)
 }
 
 # maps the 1st parameter so it only uses the bits allowed by the bitmask (2nd parameter)

net/mwan3/files/lib/mwan3/mwan3.sh

@@ -237,7 +237,9 @@ mwan3_set_dynamic_ipset()
 
 mwan3_set_general_rules()
 {
-	local IP
+	local IP unknown_wan_action
+
+	config_get unknown_wan_action globals unknown_wan_action "none"
 
 	for IP in "$IP4" "$IP6"; do
 		[ "$IP" = "$IP6" ] && [ $NO_IPV6 -ne 0 ] && continue
@@ -250,12 +252,21 @@ mwan3_set_general_rules()
 		if [ -z "$($IP rule list | awk -v var="$RULE_NO:" '$1 == var')" ]; then
 			$IP rule add pref $RULE_NO fwmark $MMX_UNREACHABLE/$MMX_MASK unreachable
 		fi
+
+		if [ $unknown_wan_action != "none" ]; then
+			RULE_NO=$((MM_UNKNOWN_WAN+2000))
+			if [ -z "$($IP rule list | awk -v var="$RULE_NO:" '$1 == var')" ]; then
+				$IP rule add pref $RULE_NO fwmark $MMX_UNKNOWN_WAN/$MMX_MASK "$unknown_wan_action"
+			fi
+		fi
 	done
 }
 
 mwan3_set_general_iptables()
 {
-	local IPT current update error family
+	local IPT current update error family unknown_wan_action
+
+	config_get unknown_wan_action globals unknown_wan_action "none"
 
 	for IPT in "$IPT4" "$IPT6"; do
 		[ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
@@ -278,10 +289,24 @@ mwan3_set_general_iptables()
 				mwan3_push_update -N mwan3_${chain}_${family}
 				mwan3_push_update -A mwan3_${chain}_${family} \
 					-m set --match-set mwan3_${chain}_${family} dst \
+					-m set --match-set mwan3_${chain}_${family} src \
 					-j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
 			fi
 		done
 
+		if [ $unknown_wan_action != "none" ]; then
+			if [ -n "${current##*-N mwan3_unknown_wan_${family}*}" ]; then
+				mwan3_push_update -N mwan3_unknown_wan_${family}
+				for chain in custom connected dynamic; do
+					mwan3_push_update -A mwan3_unknown_wan_${family} \
+						-m set --match-set mwan3_${chain}_${family} src \
+						-j RETURN
+				done
+				mwan3_push_update -A mwan3_unknown_wan_${family} \
+					-j MARK --set-xmark $MMX_UNKNOWN_WAN/$MMX_MASK
+			fi
+		fi
+
 		if [ -n "${current##*-N mwan3_rules*}" ]; then
 			mwan3_push_update -N mwan3_rules
 		fi
@@ -315,9 +340,22 @@ mwan3_set_general_iptables()
 			mwan3_push_update -A mwan3_hook \
 					  -m mark --mark 0x0/$MMX_MASK \
 					  -j CONNMARK --restore-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK"
+			if [ $unknown_wan_action != "none" ]; then
+				mwan3_push_update -A mwan3_hook \
+						  -m conntrack --ctdir REPLY \
+						  -j RETURN
+				mwan3_push_update -A mwan3_hook \
+						  -m mark --mark $MMX_UNKNOWN_WAN/$MMX_MASK \
+						  -j MARK --set-xmark 0/$MMX_MASK
+			fi
 			mwan3_push_update -A mwan3_hook \
 					  -m mark --mark 0x0/$MMX_MASK \
 					  -j mwan3_ifaces_in
+			if [ $unknown_wan_action != "none" ]; then
+				mwan3_push_update -A mwan3_hook \
+						  -m mark --mark 0x0/$MMX_MASK \
+						  -j mwan3_unknown_wan_${family}
+			fi
 
 			for chain in custom connected dynamic; do
 				mwan3_push_update -A mwan3_hook \