diff --git a/ffmap/db/routers.py b/ffmap/db/routers.py index e915e38..d5890f2 100755 --- a/ffmap/db/routers.py +++ b/ffmap/db/routers.py @@ -12,12 +12,30 @@ mysql.execute(""" CREATE TABLE banned ( `mac` varchar(20) COLLATE utf8_unicode_ci NOT NULL, `added` datetime NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci """) mysql.execute(""" ALTER TABLE `banned` - ADD PRIMARY KEY (`mac`); + ADD PRIMARY KEY (`mac`) +""") + +mysql.execute(""" + CREATE TABLE netifs ( + `id` smallint(6) UNSIGNED NOT NULL, + `name` varchar(15) COLLATE utf8_unicode_ci NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + ALTER TABLE netifs + ADD PRIMARY KEY (`id`), + ADD UNIQUE KEY `name` (`name`) +""") + +mysql.execute(""" + ALTER TABLE netifs + MODIFY `id` smallint(6) UNSIGNED NOT NULL AUTO_INCREMENT """) mysql.execute(""" @@ -178,7 +196,7 @@ mysql.execute(""" mysql.execute(""" CREATE TABLE router_stats_netif ( `router` int(11) NOT NULL, - `netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL, + `netif` smallint(6) UNSIGNED NOT NULL, `rx` bigint(20) NOT NULL, `tx` bigint(20) NOT NULL, `time` int(11) NOT NULL, diff --git a/ffmap/routertools.py b/ffmap/routertools.py index 1d10b90..d055c01 100644 --- a/ffmap/routertools.py +++ b/ffmap/routertools.py @@ -40,7 +40,7 @@ def ban_router(mysql,dbid): mysql.execute("INSERT INTO banned (mac, added) VALUES (%s, %s)",(mac,added,)) mysql.commit() -def import_nodewatcher_xml(mysql, mac, xml, banned): +def import_nodewatcher_xml(mysql, mac, xml, banned, netifdict): global router_rate_limit_list t = utcnow() @@ -181,7 +181,7 @@ def import_nodewatcher_xml(mysql, mac, xml, banned): mysql.executemany("INSERT INTO router_neighbor (router, mac, quality, net_if, type) VALUES (%s, %s, %s, %s, %s)",nbdata) if router_id: - new_router_stats(mysql, router_id, uptime, router_update) + new_router_stats(mysql, router_id, uptime, router_update, netifdict) except ValueError as e: import traceback @@ -383,7 +383,7 @@ def set_status(mysql,router_id,status): mysql.utcnow(), router_id,)) -def new_router_stats(mysql, router_id, uptime, router_update): +def new_router_stats(mysql, router_id, uptime, router_update, netifdict): if (uptime + CONFIG["router_stat_mindiff_secs"]) < router_update["sys_uptime"]: time = mysql.utctimestamp() @@ -402,11 +402,32 @@ def new_router_stats(mysql, router_id, uptime, router_update): router_update["clients"],)) ndata = [] + nkeys = [] for netif in router_update["netifs"]: # sanitize name name = netif["name"].replace(".", "").replace("$", "") with suppress(KeyError): - ndata.append((router_id,name,time,netif["traffic"]["rx"],netif["traffic"]["tx"],)) + if name in netifdict.keys(): + ndata.append((router_id,netifdict[name],time,netif["traffic"]["rx"],netif["traffic"]["tx"],)) + else: + nkeys.append((name,)) + + # 99.9 % of the routers will NOT enter this, so the doubled code is not a problem + if nkeys: + mysql.executemany(""" + INSERT INTO netifs (name) + VALUES (%s) + ON DUPLICATE KEY UPDATE name=name + """,nkeys) + netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id") + + ndata = [] + for netif in router_update["netifs"]: + # sanitize name + name = netif["name"].replace(".", "").replace("$", "") + with suppress(KeyError): + ndata.append((router_id,netifdict[name],time,netif["traffic"]["rx"],netif["traffic"]["tx"],)) + mysql.executemany(""" INSERT INTO router_stats_netif (router, netif, time, rx, tx) VALUES (%s, %s, %s, %s, %s) diff --git a/ffmap/web/api.py b/ffmap/web/api.py index 10ec8b7..f0cec07 100755 --- a/ffmap/web/api.py +++ b/ffmap/web/api.py @@ -98,6 +98,7 @@ def alfred(): banned = mysql.fetchall(""" SELECT mac FROM banned """,(),"mac") + netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id") if request.method == 'POST': alfred_data = request.get_json() @@ -105,7 +106,7 @@ def alfred(): # load router status xml data i = 1 for mac, xml in alfred_data.get("64", {}).items(): - import_nodewatcher_xml(mysql, mac, xml, banned) + import_nodewatcher_xml(mysql, mac, xml, banned, netifdict) if (i%500 == 0): mysql.commit() i += 1 diff --git a/ffmap/web/application.py b/ffmap/web/application.py index 23f0b23..e40b2f9 100755 --- a/ffmap/web/application.py +++ b/ffmap/web/application.py @@ -126,7 +126,10 @@ def router_info(dbid): s["time"] = mysql.utcawareint(s["time"]) netiffetch = mysql.fetchall(""" - SELECT netif, rx, tx, time FROM router_stats_netif WHERE router = %s + SELECT netifs.name AS netif, rx, tx, time + FROM router_stats_netif + INNER JOIN netifs ON router_stats_netif.netif = netifs.id + WHERE router = %s """,(dbid,)) for ns in netiffetch: