Change router_stats_netif to use ids for netifs

This introduces a serious of changes to code and database.

This patch requires changes to the MySQL database.

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

ffmap/db/routers.py

@@ -12,12 +12,30 @@ mysql.execute("""
 	CREATE TABLE banned (
 		`mac` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
 		`added` datetime NOT NULL
-	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
 """)
 
 mysql.execute("""
 	ALTER TABLE `banned`
-		ADD PRIMARY KEY (`mac`);
+		ADD PRIMARY KEY (`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE netifs (
+		`id` smallint(6) UNSIGNED NOT NULL,
+		`name` varchar(15) COLLATE utf8_unicode_ci NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE netifs
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`)
+""")
+
+mysql.execute("""
+	ALTER TABLE netifs
+		MODIFY `id` smallint(6) UNSIGNED NOT NULL AUTO_INCREMENT
 """)
 
 mysql.execute("""
@@ -178,7 +196,7 @@ mysql.execute("""
 mysql.execute("""
 	CREATE TABLE router_stats_netif (
 		`router` int(11) NOT NULL,
-		`netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+		`netif` smallint(6) UNSIGNED NOT NULL,
 		`rx` bigint(20) NOT NULL,
 		`tx` bigint(20) NOT NULL,
 		`time` int(11) NOT NULL,

ffmap/routertools.py

@@ -40,7 +40,7 @@ def ban_router(mysql,dbid):
 		mysql.execute("INSERT INTO banned (mac, added) VALUES (%s, %s)",(mac,added,))
 		mysql.commit()
 
-def import_nodewatcher_xml(mysql, mac, xml, banned):
+def import_nodewatcher_xml(mysql, mac, xml, banned, netifdict):
 	global router_rate_limit_list
 
 	t = utcnow()
@@ -181,7 +181,7 @@ def import_nodewatcher_xml(mysql, mac, xml, banned):
 		mysql.executemany("INSERT INTO router_neighbor (router, mac, quality, net_if, type) VALUES (%s, %s, %s, %s, %s)",nbdata)
 		
 		if router_id:
-			new_router_stats(mysql, router_id, uptime, router_update)
+			new_router_stats(mysql, router_id, uptime, router_update, netifdict)
 		
 	except ValueError as e:
 		import traceback
@@ -383,7 +383,7 @@ def set_status(mysql,router_id,status):
 		mysql.utcnow(),
 		router_id,))
 
-def new_router_stats(mysql, router_id, uptime, router_update):
+def new_router_stats(mysql, router_id, uptime, router_update, netifdict):
 	if (uptime + CONFIG["router_stat_mindiff_secs"]) < router_update["sys_uptime"]:
 		time = mysql.utctimestamp()
 		
@@ -402,11 +402,32 @@ def new_router_stats(mysql, router_id, uptime, router_update):
 			router_update["clients"],))
 		
 		ndata = []
+		nkeys = []
 		for netif in router_update["netifs"]:
 			# sanitize name
 			name = netif["name"].replace(".", "").replace("$", "")
 			with suppress(KeyError):
-				ndata.append((router_id,name,time,netif["traffic"]["rx"],netif["traffic"]["tx"],))
+				if name in netifdict.keys():
+					ndata.append((router_id,netifdict[name],time,netif["traffic"]["rx"],netif["traffic"]["tx"],))
+				else:
+					nkeys.append((name,))
+		
+		# 99.9 % of the routers will NOT enter this, so the doubled code is not a problem
+		if nkeys:
+			mysql.executemany("""
+				INSERT INTO netifs (name)
+				VALUES (%s)
+				ON DUPLICATE KEY UPDATE name=name
+			""",nkeys)
+			netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id")
+			
+			ndata = []
+			for netif in router_update["netifs"]:
+				# sanitize name
+				name = netif["name"].replace(".", "").replace("$", "")
+				with suppress(KeyError):
+					ndata.append((router_id,netifdict[name],time,netif["traffic"]["rx"],netif["traffic"]["tx"],))
+		
 		mysql.executemany("""
 			INSERT INTO router_stats_netif (router, netif, time, rx, tx)
 			VALUES (%s, %s, %s, %s, %s)

ffmap/web/api.py

@@ -98,6 +98,7 @@ def alfred():
 		banned = mysql.fetchall("""
 			SELECT mac FROM banned
 		""",(),"mac")
+		netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id")
 		if request.method == 'POST':
 			alfred_data = request.get_json()
 			
@@ -105,7 +106,7 @@ def alfred():
 				# load router status xml data
 				i = 1
 				for mac, xml in alfred_data.get("64", {}).items():
-					import_nodewatcher_xml(mysql, mac, xml, banned)
+					import_nodewatcher_xml(mysql, mac, xml, banned, netifdict)
 					if (i%500 == 0):
 						mysql.commit()
 					i += 1

ffmap/web/application.py

@@ -126,7 +126,10 @@ def router_info(dbid):
 				s["time"] = mysql.utcawareint(s["time"])
 			
 			netiffetch = mysql.fetchall("""
-				SELECT netif, rx, tx, time FROM router_stats_netif WHERE router = %s
+				SELECT netifs.name AS netif, rx, tx, time
+				FROM router_stats_netif
+				INNER JOIN netifs ON router_stats_netif.netif = netifs.id
+				WHERE router = %s
 			""",(dbid,))
 			
 			for ns in netiffetch: