api/alfred and router.html: Collect and show gateway information

This requires changes to the MySQL database!

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

ffmap/db/routers.py

@@ -109,6 +109,23 @@ mysql.execute("""
 		ADD PRIMARY KEY (`router`,`time`,`type`)
 """)
 
+mysql.execute("""
+	CREATE TABLE router_gw (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` char(17) COLLATE utf8_unicode_ci NOT NULL,
+		`quality` smallint(6) NOT NULL,
+		`nexthop` char(17) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`gw_class` varchar(25) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`selected` tinyint(1) NOT NULL DEFAULT '0'
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE router_gw
+		ADD PRIMARY KEY (`router`,`mac`)
+""")
+
 mysql.execute("""
 	CREATE TABLE router_ipv6 (
 		`router` mediumint(8) UNSIGNED NOT NULL,
@@ -183,6 +200,21 @@ mysql.execute("""
 		ADD KEY `router` (`router`)
 """)
 
+mysql.execute("""
+	CREATE TABLE router_stats_gw (
+		`time` int(11) NOT NULL,
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` char(17) COLLATE utf8_unicode_ci NOT NULL,
+		`quality` smallint(6) NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE router_stats_gw
+		ADD PRIMARY KEY (`time`,`router`,`mac`),
+		ADD KEY `router` (`router`)
+""")
+
 mysql.execute("""
 	CREATE TABLE router_stats_neighbor (
 		`time` int(11) NOT NULL,

ffmap/routertools.py

@@ -164,6 +164,14 @@ def import_nodewatcher_xml(mysql, mac, xml, banned, netifdict):
 			else:
 				mysql.execute("DELETE FROM router_neighbor WHERE router = %s",(router_id,))
 			
+			gwkeys = []
+			for g in router_update["gws"]:
+				gwkeys.append(g["mac"])
+			if gwkeys:
+				mysql.execute("DELETE FROM router_gw WHERE router = %s AND mac NOT IN ({})".format(','.join(['%s'] * len(gwkeys))),tuple([router_id] + gwkeys))
+			else:
+				mysql.execute("DELETE FROM router_gw WHERE router = %s",(router_id,))
+			
 		else:
 			# insert new router
 			created = mysql.utcnow()
@@ -232,6 +240,21 @@ def import_nodewatcher_xml(mysql, mac, xml, banned, netifdict):
 				type=VALUES(type)
 		""",nbdata)
 		
+		gwdata = []
+		for g in router_update["gws"]:
+			gwdata.append((router_id,g["mac"],g["quality"],g["nexthop"],g["netif"],g["gw_class"],g["selected"],))
+		
+		mysql.executemany("""
+			INSERT INTO router_gw (router, mac, quality, nexthop, netif, gw_class, selected)
+			VALUES (%s, %s, %s, %s, %s, %s, %s)
+			ON DUPLICATE KEY UPDATE
+				quality=VALUES(quality),
+				nexthop=VALUES(nexthop),
+				netif=VALUES(netif),
+				gw_class=VALUES(gw_class),
+				selected=VALUES(selected)
+		""",gwdata)
+		
 		if router_id:
 			new_router_stats(mysql, router_id, uptime, router_update, netifdict)
 		
@@ -354,6 +377,17 @@ def delete_old_stats(mysql):
 	writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete stats: %.3f seconds" % (time.time() - start_time))
 	print("--- Delete stats: %.3f seconds ---" % (time.time() - start_time))
 
+	time.sleep(10)
+	start_time = time.time()
+	mysql.execute("""
+		DELETE s FROM router_stats_gw AS s
+		LEFT JOIN router AS r ON s.router = r.id
+		WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
+	""",(threshold,))
+	mysql.commit()
+	writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete gw-stats: %.3f seconds" % (time.time() - start_time))
+	print("--- Delete gw-stats: %.3f seconds ---" % (time.time() - start_time))
+
 	time.sleep(10)
 	start_time = time.time()
 	mysql.execute("""
@@ -494,6 +528,15 @@ def new_router_stats(mysql, router_id, uptime, router_update, netifdict):
 			INSERT INTO router_stats_neighbor (time, router, mac, quality)
 			VALUES (%s, %s, %s, %s)
 		""",nbdata)
+		
+		gwdata = []
+		for gw in router_update["gws"]:
+			with suppress(KeyError):
+				gwdata.append((time,router_id,gw["mac"],gw["quality"],))
+		mysql.executemany("""
+			INSERT INTO router_stats_gw (time, router, mac, quality)
+			VALUES (%s, %s, %s, %s)
+		""",gwdata)
 
 def calculate_network_io(mysql, router_id, uptime, router_update):
 	"""
@@ -538,6 +581,14 @@ def evalxpathint(tree,p,default=0):
 		tmp = int(tree.xpath(p)[0])
 	return tmp
 
+def evalxpathbool(tree,p):
+	tmp = False
+	with suppress(IndexError):
+		tmp = tree.xpath(p)[0]
+	if tmp:
+		return (tmp.lower()=="true" or tmp=="1")
+	return False
+
 def parse_nodewatcher_xml(xml):
 	try:
 		assert xml != ""
@@ -547,6 +598,7 @@ def parse_nodewatcher_xml(xml):
 			"status": evalxpath(tree,"/data/system_data/status/text()"),
 			"hostname": evalxpath(tree,"/data/system_data/hostname/text()"),
 			"last_contact": utcnow().strftime('%Y-%m-%d %H:%M:%S'),
+			"gws": [],
 			"neighbours": [],
 			"netifs": [],
 			# hardware
@@ -680,6 +732,23 @@ def parse_nodewatcher_xml(xml):
 		visible_neighbours += len(l3_neighbours)
 		router_update["visible_neighbours"] = visible_neighbours
 		router_update["neighbours"] += l3_neighbours
+		
+		for gw in tree.xpath("/data/batman_adv_gateway_list/*"):
+			gw_mac = evalxpath(gw,"gateway/text()")
+			if (gw_mac and len(gw_mac)>12): # Throw away headline
+				gw = {
+					"mac": gw_mac.lower(),
+					"quality": evalxpathint(gw,"link_quality/text()"),
+					"nexthop": evalxpath(gw,"nexthop/text()",None),
+					"netif": evalxpath(gw,"outgoing_interface/text()",None),
+					"gw_class": evalxpath(gw,"gw_class/text()",None),
+					"selected": evalxpathbool(gw,"selected/text()")
+				}
+				if gw["netif"]=="false":
+					tmp = gw["gw_class"].split(None,1)
+					gw["netif"] = tmp[0]
+					gw["gw_class"] = tmp[1]
+				router_update["gws"].append(gw)
 
 		return router_update
 	except (AssertionError, lxml.etree.XMLSyntaxError, IndexError) as e:

ffmap/web/application.py

@@ -118,6 +118,12 @@ def router_info(dbid):
 			""",(dbid,))
 			# FIX SQL: only one from router_netif
 			
+			router["gws"] = mysql.fetchall("""
+				SELECT mac, quality, netif, gw_class, selected
+				FROM router_gw
+				WHERE router = %s
+			""",(dbid,))
+			
 			router["events"] = mysql.fetchall("""SELECT * FROM router_events WHERE router = %s""",(dbid,))
 			router["events"] = mysql.utcawaretuple(router["events"],"time")
 			
@@ -142,6 +148,13 @@ def router_info(dbid):
 			for ns in neighfetch:
 				ns["time"] = mysql.utcawareint(ns["time"])
 
+			gwfetch = mysql.fetchall("""
+				SELECT quality, mac, time FROM router_stats_gw WHERE router = %s
+			""",(dbid,))
+			
+			for ns in gwfetch:
+				ns["time"] = mysql.utcawareint(ns["time"])
+
 			if request.method == 'POST':
 				if request.form.get("act") == "delete":
 					# a router may not have a owner, but admin users still can delete it
@@ -193,6 +206,7 @@ def router_info(dbid):
 				tileurls = tileurls,
 				netifstats = netiffetch,
 				neighstats = neighfetch,
+				gwstats = gwfetch,
 				authuser = is_authorized(router["user"], session),
 				authadmin = session.get('admin')
 				)

ffmap/web/static/js/graph.js

@@ -142,6 +142,49 @@ function neighbour_graph(neighbours) {
 	setup_plot_zoom(plot, pdata, len);
 }
 
+function gw_graph(gws) {
+	var gwstat = $("#gwstat");
+	var pdata = [];
+	for (j=0; j<gws.length; j++) {
+		var label = gws[j].name;
+
+		// add network interface when there are multiple links to same node
+		var k;
+		for(k=0; k<gws.length; k++) {
+			if(label == gws[k].name && k != j) {
+				label += "@" + gws[j].netif;
+			}
+		}
+
+		var mac = gws[j].mac;
+		var data = [];
+		var len, i;
+		for (len=gw_stats.length, i=0; i<len; i++) {
+			if (gw_stats[i].mac != mac) { continue; }
+			try {
+				var quality = gw_stats[i].quality;
+				var date_value = gw_stats[i].time.$date;
+				if(quality == null) {
+					quality = 0;
+				}
+				data.push([date_value, quality]);
+			}
+			catch(TypeError) {
+				// pass
+			}
+		}
+		pdata.push({"label": label, "data": data});
+	}
+	var plot = $.plot(gwstat, pdata, {
+		xaxis: {mode: "time", timezone: "browser"},
+		selection: {mode: "x"},
+		yaxis: {min: 0, max: 350},
+		legend: {noColumns: 2, hideable: true},
+		series: {downsample: {threshold: Math.floor(gwstat.width() * points_per_px)}}
+	});
+	setup_plot_zoom(plot, pdata, len);
+}
+
 function memory_graph() {
 	var memstat = $("#memstat");
 	var free = [], caching = [], buffering = [];

ffmap/web/templates/router.html

@@ -315,6 +315,42 @@
 				{%- endfor %}
 				</ul>
 			</div>
+			{%- if router.gws|length > 0 %}
+			<div class="panel panel-default" style="flex: 1 1 auto;">
+				<div class="panel-heading">Gateways</div>
+				<div class="panel-body" style="height: 100%;">
+					<div class="table-responsive">
+						<table class="neighbours" style="width: 100%; margin-bottom: 6px;">
+							<tr>
+								<th>MAC (fastd/l2tp)</th>
+								<th>Qual</th>
+								<th>Netif</th>
+								<th>Class</th>
+							</tr>
+							{%- for gw in router.gws %}
+							{%- if gw.selected %}
+							<tr style="background-color:#04ff0a">
+							{%- else %}
+							<tr>
+							{%- endif %}
+								<td>{{ gw.mac }}</td>
+								<td>{{ gw.quality }}</td>
+								<td>{{ gw.netif }}</td>
+								<td>{{ gw.gw_class }}</td>
+							</tr>
+							{%- endfor %}
+						</table>
+					</div>
+					{# hack for graph vertical align #}
+					{%- if  router.gws|length < 3 %}
+					{%- for n in range(3- router.gws|length) %}
+					<br />
+					{%- endfor %}
+					{%- endif %}
+					<div id="gwstat" class="graph" style="height: 150px;"></div>
+				</div>
+			</div>
+			{%- endif %}
 		</div>
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
@@ -347,15 +383,24 @@
 		var router_stats = {{ router.stats|statbson2json|safe }};
 		var netif_stats = {{ netifstats|statbson2json|safe }};
 		var neigh_stats = {{ neighstats|statbson2json|safe }};
+		var gw_stats = {{ gwstats|statbson2json|safe }};
 		var neighbours = [
 		{%- for neighbour in router.neighbours %}
 			{"name": "{{ neighbour.hostname or neighbour.mac }}", "mac": "{{ neighbour.mac }}", "netif": "{{ neighbour.netif }}"},
 		{%- endfor %}
 		];
+		var gws = [
+		{%- for gw in router.gws %}
+			{"name": "{{ gw.mac }}", "mac": "{{ gw.mac }}", "netif": "{{ gw.netif }}"},
+		{%- endfor %}
+		];
 		$(document).ready(function() {
 			{%- if router.neighbours|length > 0 %}
 			neighbour_graph(neighbours);
 			{%- endif %}
+			{%- if router.gws|length > 0 %}
+			gw_graph(gws);
+			{%- endif %}
 			memory_graph();
 			process_graph();
 			client_graph();