From 58cc2e167f4a4a11c60667b4421dd8c96bf0c009 Mon Sep 17 00:00:00 2001 From: Adrian Schmutzler Date: Mon, 15 Jan 2018 13:51:16 +0100 Subject: [PATCH] scripts/deletestats: Do not evaluate status in stats deletion Previously, old stats had been only deleted if the router was online. This required a join with the router table, which caused locking issues when writing to this table by alfred request. Now, we just delete old stats ignoring the router state, except for the router_stats table, which is smaller and thus remains as before. In addition, this patch logs the number of rows affected. Signed-off-by: Adrian Schmutzler --- ffmap/config.py | 4 +-- ffmap/routertools.py | 59 ++++++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/ffmap/config.py b/ffmap/config.py index 643bf0a..d89772f 100644 --- a/ffmap/config.py +++ b/ffmap/config.py @@ -7,8 +7,8 @@ CONFIG = { "offline_threshold_minutes": 15, # Router switches to offline after X minutes "orphan_threshold_days": 7, # Router switches to orphaned state after X days "delete_threshold_days": 180, # Router is deleted after X days - "router_stat_days": 30, # Router stats are collected for X days (if online) - "router_stat_netif": 21, # Router stats for netifs are collected for X days (if online) + "router_stat_days": 30, # Router stats are collected for X days + "router_stat_netif": 21, # Router stats for netifs are collected for X days "router_stat_mindiff_secs": 10, # Time difference (uptime) in seconds required for a new entry in router stats "router_stat_mindiff_default": 270, # Time difference (router stats tables) in seconds required for a new entry in router stats "router_stat_mindiff_netif": 570, # Time difference (router netif stats) in seconds required for a new entry in router stats diff --git a/ffmap/routertools.py b/ffmap/routertools.py index ae03011..5efa303 100644 --- a/ffmap/routertools.py +++ b/ffmap/routertools.py @@ -390,69 +390,58 @@ def delete_old_stats(mysql): threshold_netif=(utcnow() - datetime.timedelta(days=CONFIG["router_stat_netif"])).timestamp() start_time = time.time() - mysql.execute(""" + rowsaffected = mysql.execute(""" DELETE s FROM router_stats AS s LEFT JOIN router AS r ON s.router = r.id WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) """,(threshold,)) mysql.commit() - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete stats: %.3f seconds" % (time.time() - start_time)) - print("--- Delete stats: %.3f seconds ---" % (time.time() - start_time)) + writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from stats: %.3f seconds" % (rowsaffected,time.time() - start_time)) + print("--- Deleted %i rows from stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time)) time.sleep(10) start_time = time.time() - mysql.execute(""" - DELETE s FROM router_stats_gw AS s - LEFT JOIN router AS r ON s.router = r.id - WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) + rowsaffected = mysql.execute(""" + DELETE FROM router_stats_gw + WHERE router_stats_gw.time < %s """,(threshold,)) mysql.commit() - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete gw-stats: %.3f seconds" % (time.time() - start_time)) - print("--- Delete gw-stats: %.3f seconds ---" % (time.time() - start_time)) + writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from gw-stats: %.3f seconds" % (rowsaffected,time.time() - start_time)) + print("--- Deleted %i rows from gw-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time)) time.sleep(10) start_time = time.time() - mysql.execute(""" - DELETE s FROM router_stats_neighbor AS s - LEFT JOIN router AS r ON s.router = r.id - WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) + rowsaffected = mysql.execute(""" + DELETE FROM router_stats_neighbor + WHERE router_stats_neighbor.time < %s """,(threshold,)) mysql.commit() - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete neighbor-stats: %.3f seconds" % (time.time() - start_time)) - print("--- Delete neighbor-stats: %.3f seconds ---" % (time.time() - start_time)) + writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from neighbor-stats: %.3f seconds" % (rowsaffected,time.time() - start_time)) + print("--- Deleted %i rows from neighbor-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time)) - time.sleep(10) - start_time = time.time() - mysql.execute(""" - UPDATE router_stats_netif AS s - LEFT JOIN router AS r ON s.router = r.id - SET s.deletebit = 1 - WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) - """,(threshold_netif,)) - mysql.commit() - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Update netif stats: %.3f seconds" % (time.time() - start_time)) - print("--- Update netif stats: %.3f seconds ---" % (time.time() - start_time)) - time.sleep(30) minustime=0 rowsaffected=1 + allrows=0 start_time = time.time() while rowsaffected > 0: try: rowsaffected = mysql.execute(""" DELETE FROM router_stats_netif - WHERE deletebit = 1 - LIMIT 50000 - """) + WHERE router_stats_netif.time < %s + LIMIT 100000 + """,(threshold_netif,)) mysql.commit() + allrows += rowsaffected except my.OperationalError: rowsaffected = 1 time.sleep(10) minustime += 10 - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete netif stats: %.3f seconds" % (time.time() - start_time - minustime)) - print("--- Delete netif stats: %.3f seconds ---" % (time.time() - start_time - minustime)) + writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from netif stats: %.3f seconds" % (allrows,time.time() - start_time - minustime)) + print("--- Deleted %i rows from netif stats: %.3f seconds ---" % (allrows,time.time() - start_time - minustime)) start_time = time.time() + allrows=0 events = mysql.fetchall(""" SELECT router, COUNT(time) AS count FROM router_events GROUP BY router @@ -461,16 +450,16 @@ def delete_old_stats(mysql): for e in events: delnum = int(e["count"] - CONFIG["event_num_entries"]) if delnum > 0: - mysql.execute(""" + allrows += mysql.execute(""" DELETE FROM router_events WHERE router = %s ORDER BY time ASC LIMIT %s """,(e["router"],delnum,)) mysql.commit() - writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete events: %.3f seconds" % (time.time() - start_time)) + writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from events: %.3f seconds" % (allrows,time.time() - start_time)) writelog(CONFIG["debug_dir"] + "/deletetime.txt", "-------") - print("--- Delete events: %.3f seconds ---" % (time.time() - start_time)) + print("--- Deleted %i rows from events: %.3f seconds ---" % (allrows,time.time() - start_time)) def events_append(mysql,router_id,event,comment): mysql.execute("""