From 2a515b94c727c37840c21764caf094d6fc090a16 Mon Sep 17 00:00:00 2001 From: Adrian Schmutzler Date: Thu, 11 Jan 2018 14:44:10 +0100 Subject: [PATCH] config/routertools: Set minimum distance for data in router stats This implement to different minimum distances in seconds for the router stats in general and the netif stats in particular. The values are chosen so that they are 30 secs. shorter than the desired timespans of 5 and 10 minutes, to allow for fluctuation in when data arrives. This fixes the data density increase caused by V2 Hoods with two gateways. Signed-off-by: Adrian Schmutzler --- ffmap/config.py | 4 +- ffmap/routertools.py | 118 +++++++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 56 deletions(-) diff --git a/ffmap/config.py b/ffmap/config.py index 9db15c6..e2606b2 100644 --- a/ffmap/config.py +++ b/ffmap/config.py @@ -8,7 +8,9 @@ CONFIG = { "orphan_threshold_days": 7, # Router switches to orphaned state after X days "delete_threshold_days": 180, # Router is deleted after X days "router_stat_days": 30, # Router stats are collected for X days (if online) - "router_stat_mindiff_secs": 10, # Time difference (uptime) required for a new entry in router stats + "router_stat_mindiff_secs": 10, # Time difference (uptime) in seconds required for a new entry in router stats + "router_stat_mindiff_default": 270, # Time difference (router stats tables) in seconds required for a new entry in router stats + "router_stat_mindiff_netif": 570, # Time difference (router netif stats) in seconds required for a new entry in router stats "event_num_entries": 30, # Number of events stored per router "global_stat_days": 365, # Global/hood stats are collected for X days "csv_dir": "/var/lib/ffmap/csv", # Directory where the .csv files for TileStache/mapnik are stored diff --git a/ffmap/routertools.py b/ffmap/routertools.py index dee9cb8..c60a56c 100644 --- a/ffmap/routertools.py +++ b/ffmap/routertools.py @@ -474,69 +474,77 @@ def new_router_stats(mysql, router_id, uptime, router_update, netifdict): if (uptime + CONFIG["router_stat_mindiff_secs"]) < router_update["sys_uptime"]: time = mysql.utctimestamp() - mysql.execute(""" - INSERT INTO router_stats (time, router, sys_memfree, sys_membuff, sys_memcache, loadavg, sys_procrun, sys_proctot, clients) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) - """,( - time, - router_id, - router_update["memory"]['free'], - router_update["memory"]['buffering'], - router_update["memory"]['caching'], - router_update["sys_loadavg"], - router_update["processes"]['runnable'], - router_update["processes"]['total'], - router_update["clients"],)) + stattime = mysql.findone("SELECT time FROM router_stats WHERE router = %s ORDER BY time DESC LIMIT 1",(router_id,),"time") + if not stattime or (stattime + CONFIG["router_stat_mindiff_default"]) < time: + mysql.execute(""" + INSERT INTO router_stats (time, router, sys_memfree, sys_membuff, sys_memcache, loadavg, sys_procrun, sys_proctot, clients) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """,( + time, + router_id, + router_update["memory"]['free'], + router_update["memory"]['buffering'], + router_update["memory"]['caching'], + router_update["sys_loadavg"], + router_update["processes"]['runnable'], + router_update["processes"]['total'], + router_update["clients"],)) - ndata = [] - nkeys = [] - for netif in router_update["netifs"]: - # sanitize name - name = netif["name"].replace(".", "").replace("$", "") - with suppress(KeyError): - if name in netifdict.keys(): - ndata.append((time,router_id,netifdict[name],netif["traffic"]["rx"],netif["traffic"]["tx"],)) - else: - nkeys.append((name,)) - - # 99.9 % of the routers will NOT enter this, so the doubled code is not a problem - if nkeys: - mysql.executemany(""" - INSERT INTO netifs (name) - VALUES (%s) - ON DUPLICATE KEY UPDATE name=name - """,nkeys) - netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id") - + netiftime = mysql.findone("SELECT time FROM router_stats_netif WHERE router = %s ORDER BY time DESC LIMIT 1",(router_id,),"time") + if not netiftime or (netiftime + CONFIG["router_stat_mindiff_netif"]) < time: ndata = [] + nkeys = [] for netif in router_update["netifs"]: # sanitize name name = netif["name"].replace(".", "").replace("$", "") with suppress(KeyError): - ndata.append((time,router_id,netifdict[name],netif["traffic"]["rx"],netif["traffic"]["tx"],)) + if name in netifdict.keys(): + ndata.append((time,router_id,netifdict[name],netif["traffic"]["rx"],netif["traffic"]["tx"],)) + else: + nkeys.append((name,)) + + # 99.9 % of the routers will NOT enter this, so the doubled code is not a problem + if nkeys: + mysql.executemany(""" + INSERT INTO netifs (name) + VALUES (%s) + ON DUPLICATE KEY UPDATE name=name + """,nkeys) + netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id") + + ndata = [] + for netif in router_update["netifs"]: + # sanitize name + name = netif["name"].replace(".", "").replace("$", "") + with suppress(KeyError): + ndata.append((time,router_id,netifdict[name],netif["traffic"]["rx"],netif["traffic"]["tx"],)) + + mysql.executemany(""" + INSERT INTO router_stats_netif (time, router, netif, rx, tx) + VALUES (%s, %s, %s, %s, %s) + """,ndata) - mysql.executemany(""" - INSERT INTO router_stats_netif (time, router, netif, rx, tx) - VALUES (%s, %s, %s, %s, %s) - """,ndata) + # reuse timestamp from router_stats to avoid additional queries + if not stattime or (stattime + CONFIG["router_stat_mindiff_default"]) < time: + nbdata = [] + for neighbour in router_update["neighbours"]: + with suppress(KeyError): + nbdata.append((time,router_id,neighbour["mac"],neighbour["quality"],)) + mysql.executemany(""" + INSERT INTO router_stats_neighbor (time, router, mac, quality) + VALUES (%s, %s, %s, %s) + """,nbdata) - nbdata = [] - for neighbour in router_update["neighbours"]: - with suppress(KeyError): - nbdata.append((time,router_id,neighbour["mac"],neighbour["quality"],)) - mysql.executemany(""" - INSERT INTO router_stats_neighbor (time, router, mac, quality) - VALUES (%s, %s, %s, %s) - """,nbdata) - - gwdata = [] - for gw in router_update["gws"]: - with suppress(KeyError): - gwdata.append((time,router_id,gw["mac"],gw["quality"],)) - mysql.executemany(""" - INSERT INTO router_stats_gw (time, router, mac, quality) - VALUES (%s, %s, %s, %s) - """,gwdata) + # reuse timestamp from router_stats to avoid additional queries + if not stattime or (stattime + CONFIG["router_stat_mindiff_default"]) < time: + gwdata = [] + for gw in router_update["gws"]: + with suppress(KeyError): + gwdata.append((time,router_id,gw["mac"],gw["quality"],)) + mysql.executemany(""" + INSERT INTO router_stats_gw (time, router, mac, quality) + VALUES (%s, %s, %s, %s) + """,gwdata) def calculate_network_io(mysql, router_id, uptime, router_update): """