Add gateway-specific statistics

This adds gateway stats which work similar to the detailed hood
statistics.

This requires changes to the MySQL database!

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

ffmap/db/stats.py

@@ -24,6 +24,24 @@ mysql.execute("""
 		ADD PRIMARY KEY (`time`)
 """)
 
+mysql.execute("""
+	CREATE TABLE stats_gw (
+		`time` int(11) NOT NULL,
+		`mac` char(17) COLLATE utf8_unicode_ci NOT NULL,
+		`clients` mediumint(9) NOT NULL,
+		`online` smallint(6) NOT NULL,
+		`offline` smallint(6) NOT NULL,
+		`unknown` smallint(6) NOT NULL,
+		`orphaned` smallint(6) NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE stats_gw
+		ADD PRIMARY KEY (`time`,`mac`),
+		ADD KEY `mac` (`mac`)
+""")
+
 mysql.execute("""
 	CREATE TABLE stats_hood (
 		`time` int(11) NOT NULL,

ffmap/stattools.py

@@ -58,7 +58,31 @@ def router_status_hood(mysql):
 		dict[d["hood"]][d["status"]] = d["count"]
 	return dict
 
-def router_models(mysql,selecthood=None):
+def total_clients_gw(mysql):
+	return mysql.fetchdict("""
+		SELECT router_gw.mac, SUM(clients) AS clients
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE
+		GROUP BY router_gw.mac
+	""",(),"mac","clients")
+
+def router_status_gw(mysql):
+	data = mysql.fetchall("""
+		SELECT router_gw.mac, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE
+		GROUP BY router_gw.mac, router.status
+	""")
+	dict = {}
+	for d in data:
+		if not d["mac"] in dict:
+			dict[d["mac"]] = {}
+		dict[d["mac"]][d["status"]] = d["count"]
+	return dict
+
+def router_models(mysql,selecthood=None,selectgw=None):
 	if selecthood:
 		return mysql.fetchdict("""
 			SELECT hardware, COUNT(id) AS count
@@ -66,6 +90,14 @@ def router_models(mysql,selecthood=None):
 			WHERE hood = %s
 			GROUP BY hardware
 		""",(selecthood,),"hardware","count")
+	elif selectgw:
+		return mysql.fetchdict("""
+			SELECT hardware, COUNT(router_gw.router) AS count
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE mac = %s
+			GROUP BY hardware
+		""",(selectgw,),"hardware","count")
 	else:
 		return mysql.fetchdict("""
 			SELECT hardware, COUNT(id) AS count
@@ -73,7 +105,7 @@ def router_models(mysql,selecthood=None):
 			GROUP BY hardware
 		""",(),"hardware","count")
 
-def router_firmwares(mysql,selecthood=None):
+def router_firmwares(mysql,selecthood=None,selectgw=None):
 	if selecthood:
 		return mysql.fetchdict("""
 			SELECT firmware, COUNT(id) AS count
@@ -81,6 +113,14 @@ def router_firmwares(mysql,selecthood=None):
 			WHERE hood = %s
 			GROUP BY firmware
 		""",(selecthood,),"firmware","count")
+	elif selectgw:
+		return mysql.fetchdict("""
+			SELECT firmware, COUNT(router_gw.router) AS count
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE mac = %s
+			GROUP BY firmware
+		""",(selectgw,),"firmware","count")
 	else:
 		return mysql.fetchdict("""
 			SELECT firmware, COUNT(id) AS count
@@ -88,7 +128,7 @@ def router_firmwares(mysql,selecthood=None):
 			GROUP BY firmware
 		""",(),"firmware","count")
 
-def hoods(mysql):
+def hoods(mysql,selectgw=None):
 	data = mysql.fetchall("""
 		SELECT hood, status, COUNT(id) AS count
 		FROM router
@@ -103,7 +143,7 @@ def hoods(mysql):
 		result[rs["hood"]][rs["status"]] = rs["count"]
 	return result
 
-def hoods_sum(mysql):
+def hoods_sum(mysql,selectgw=None):
 	data = mysql.fetchall("""
 		SELECT hood, COUNT(id) AS count, SUM(clients) AS clients
 		FROM router
@@ -226,6 +266,34 @@ def record_hood_stats(mysql):
 
 	mysql.commit()
 
+def record_gw_stats(mysql):
+	threshold=(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"])).timestamp()
+	time = mysql.utctimestamp()
+	status = router_status_gw(mysql)
+	clients = total_clients_gw(mysql)
+	
+	for mac in clients.keys():
+		old = mysql.findone("SELECT time FROM stats_gw WHERE time = %s AND mac = %s LIMIT 1",(time,mac,))
+		
+		if old:
+			mysql.execute("""
+				UPDATE stats_gw
+				SET clients = %s, online = %s, offline = %s, unknown = %s, orphaned = %s
+				WHERE time = %s AND mac = %s
+			""",(clients[mac],status[mac].get("online",0),status[mac].get("offline",0),status[mac].get("unknown",0),status[mac].get("orphaned",0),time,mac,))
+		else:
+			mysql.execute("""
+				INSERT INTO stats_gw (time, mac, clients, online, offline, unknown, orphaned)
+				VALUES (%s, %s, %s, %s, %s, %s, %s)
+			""",(time,mac,clients[mac],status[mac].get("online",0),status[mac].get("offline",0),status[mac].get("unknown",0),status[mac].get("orphaned",0),))
+	
+	mysql.execute("""
+		DELETE FROM stats_gw
+		WHERE time < %s
+	""",(threshold,))
+
+	mysql.commit()
+
 def router_user_sum(mysql):
 	data = mysql.fetchall("""
 		SELECT contact, COUNT(id) AS count, SUM(clients) AS clients

ffmap/web/application.py

@@ -326,16 +326,22 @@ def user_info(nickname):
 def global_statistics():
 	mysql = FreifunkMySQL()
 	stats = mysql.fetchall("SELECT * FROM stats_global")
-	return helper_statistics(mysql,stats,None)
+	return helper_statistics(mysql,stats,None,None)
 
 @app.route('/hoodstatistics/<selecthood>')
 def global_hoodstatistics(selecthood):
 	mysql = FreifunkMySQL()
 	stats = mysql.fetchall("SELECT * FROM stats_hood WHERE hood = %s",(selecthood,))
-	return helper_statistics(mysql,stats,selecthood)
+	return helper_statistics(mysql,stats,selecthood,None)
 
-def helper_statistics(mysql,stats,selecthood):
-	hoods = stattools.hoods(mysql)
+@app.route('/gwstatistics/<selectgw>')
+def global_gwstatistics(selectgw):
+	mysql = FreifunkMySQL()
+	stats = mysql.fetchall("SELECT * FROM stats_gw WHERE mac = %s",(selectgw,))
+	return helper_statistics(mysql,stats,None,selectgw)
+
+def helper_statistics(mysql,stats,selecthood,selectgw):
+	hoods = stattools.hoods(mysql,selectgw)
 	
 	stats = mysql.utcawaretupleint(stats,"time")
 	
@@ -343,33 +349,43 @@ def helper_statistics(mysql,stats,selecthood):
 	if numnew < 1:
 		numnew = 1
 	
-	if selecthood:
-		where = " AND hood = %s"
-		tup = (selecthood,numnew,)
+	if selectgw:
+		newest_routers = mysql.fetchall("""
+			SELECT id, hostname, hood, created
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE hardware <> 'Legacy' AND mac = %s
+			ORDER BY created DESC
+			LIMIT %s
+		""",(selectgw,numnew,))
 	else:
-		where = ""
-		tup = (numnew,)
-	
-	newest_routers = mysql.fetchall("""
-		SELECT id, hostname, hood, created
-		FROM router
-		WHERE hardware <> 'Legacy' {}
-		ORDER BY created DESC
-		LIMIT %s
-	""".format(where),tup)
+		if selecthood:
+			where = " AND hood = %s"
+			tup = (selecthood,numnew,)
+		else:
+			where = ""
+			tup = (numnew,)
+		newest_routers = mysql.fetchall("""
+			SELECT id, hostname, hood, created
+			FROM router
+			WHERE hardware <> 'Legacy' {}
+			ORDER BY created DESC
+			LIMIT %s
+		""".format(where),tup)
 	newest_routers = mysql.utcawaretuple(newest_routers,"created")
 	
 	clients = stattools.total_clients(mysql)
 	router_status = stattools.router_status(mysql)
-	router_models = stattools.router_models(mysql,selecthood)
-	router_firmwares = stattools.router_firmwares(mysql,selecthood)
-	hoods_sum = stattools.hoods_sum(mysql)
+	router_models = stattools.router_models(mysql,selecthood,selectgw)
+	router_firmwares = stattools.router_firmwares(mysql,selecthood,selectgw)
+	hoods_sum = stattools.hoods_sum(mysql,selectgw)
 	gws = stattools.gws(mysql,selecthood)
 	gws_sum = stattools.gws_sum(mysql,selecthood)
 	mysql.close()
 	
 	return render_template("statistics.html",
 		selecthood = selecthood,
+		selectgw = selectgw,
 		stats = stats,
 		clients = clients,
 		router_status = router_status,

ffmap/web/templates/statistics.html

@@ -1,5 +1,5 @@
 {% extends "bootstrap.html" %}
-{% block title %}{{super()}} :: Statistics{%- if selecthood %} for {{ selecthood }}{%- endif -%}{% endblock %}
+{% block title %}{{super()}} :: Statistics{%- if selecthood %} for {{ selecthood }}{%- endif -%}{%- if selectgw %} for GW {{ selectgw }}{%- endif -%}{% endblock %}
 {% block head %}{{super()}}
 	<script src="{{ url_for('static', filename='js/graph/date.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/graph/jquery.flot.js') }}"></script>
@@ -87,19 +87,19 @@
 		</div>
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Routers{%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
+				<div class="panel-heading">Routers{%- if selecthood %} @ {{ selecthood }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }} (selected only){%- endif -%}</div>
 				<div class="panel-body">
 					<div id="globrouterstat" class="graph"></div>
 				</div>
 			</div>
 			<div class="panel panel-default">
-				<div class="panel-heading">Clients{%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
+				<div class="panel-heading">Clients{%- if selecthood %} @ {{ selecthood }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }} (selected only){%- endif -%}</div>
 				<div class="panel-body">
 					<div id="globclientstat" class="graph"></div>
 				</div>
 			</div>
 			<div class="panel panel-default">
-				<div class="panel-heading">Newest Routers{%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
+				<div class="panel-heading">Newest Routers{%- if selecthood %} @ {{ selecthood }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
 				<div class="panel-body" style="padding-bottom:34px">
 					<div class="table-responsive">
 						<table class="table table-condensed">
@@ -124,7 +124,7 @@
 	<div class="row">
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Gateways (selected / others)</div>
+				<div class="panel-heading">Gateways (selected / others){%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
 				<div class="panel-body">
 					<table class="table table-condensed table-hoods">
 						<tr>
@@ -134,6 +134,7 @@
 							<th class="warning" title="Unknown Routers">Unknown</th>
 							<th class="active" title="Total Routers">Total</th>
 							<th class="info">Clients</th>
+							<th class="stats">Stats</th>
 						</tr>
 						{%- for mac, value in gws|dictsort %}
 						<tr>
@@ -143,6 +144,7 @@
 							<td class="warning"><span style="font-weight:bold">{{ value["selected"]["unknown"] or 0 }}</span> / {{ value["others"]["unknown"] or 0 }}</td>
 							<td class="active"><span style="font-weight:bold">{{ gws_sum[mac]["routers"] if gws_sum[mac] else 0 }}</span> / {{ value["others"]|sumdict if value["others"] else 0 }}</td>
 							<td class="info">{{ gws_sum[mac]["clients"] if gws_sum[mac] }}</td>
+							<td class="stats"><a href="{{ url_for('global_gwstatistics', selectgw='%s' % mac) }}">Stats</a></td>
 						</tr>
 						{%- endfor %}
 					</table>
@@ -151,13 +153,13 @@
 		</div>
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Router Firmwares{%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
+				<div class="panel-heading">Router Firmwares{%- if selecthood %} @ {{ selecthood }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
 				<div class="panel-body">
 					<div id="globrouterfwstat" class="graph-pie"></div>
 				</div>
 			</div>
 			<div class="panel panel-default">
-				<div class="panel-heading">Router Models{%- if selecthood %} @ {{ selecthood }}{%- endif -%}</div>
+				<div class="panel-heading">Router Models{%- if selecthood %} @ {{ selecthood }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
 				<div class="panel-body">
 					<div id="globroutermodelsstat" class="graph-pie"></div>
 				</div>

scripts/calcglobalstats.py

@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
 from ffmap.routertools import *
 from ffmap.maptools import *
 from ffmap.mysqltools import FreifunkMySQL
-from ffmap.stattools import record_global_stats, record_hood_stats
+from ffmap.stattools import record_global_stats, record_hood_stats, record_gw_stats
 
 import time
 start_time = time.time()
@@ -21,6 +21,7 @@ delete_orphaned_routers(mysql)
 #delete_old_stats(mysql) # Only execute once daily, takes 2 minutes
 record_global_stats(mysql)
 record_hood_stats(mysql)
+record_gw_stats(mysql)
 update_mapnik_csv(mysql)
 mysql.close()