freifunk-franken
/
monitoring
mirror of https://github.com/FreifunkFranken/fff-monitoring.git
3
0
Fork 0
Code Issues Releases Wiki Activity

scripts/deletestats: Do not evaluate status in stats deletion 

Previously, old stats had been only deleted if the router was
online. This required a join with the router table, which caused
locking issues when writing to this table by alfred request.

Now, we just delete old stats ignoring the router state, except
for the router_stats table, which is smaller and thus remains
as before.

In addition, this patch logs the number of rows affected.

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>
This commit is contained in:
Adrian Schmutzler 2018-01-15 13:51:16 +01:00
parent 804ce80472
commit 58cc2e167f
2 changed files with 26 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ffmap/config.py
View File

@ -7,8 +7,8 @@ CONFIG = {
"offline_threshold_minutes": 15, # Router switches to offline after X minutes
"orphan_threshold_days": 7, # Router switches to orphaned state after X days
"delete_threshold_days": 180, # Router is deleted after X days
"router_stat_days": 30, # Router stats are collected for X days (if online)
"router_stat_netif": 21, # Router stats for netifs are collected for X days (if online)
"router_stat_days": 30, # Router stats are collected for X days
"router_stat_netif": 21, # Router stats for netifs are collected for X days
"router_stat_mindiff_secs": 10, # Time difference (uptime) in seconds required for a new entry in router stats
"router_stat_mindiff_default": 270, # Time difference (router stats tables) in seconds required for a new entry in router stats
"router_stat_mindiff_netif": 570, # Time difference (router netif stats) in seconds required for a new entry in router stats

59
ffmap/routertools.py
View File

@ -390,69 +390,58 @@ def delete_old_stats(mysql):
threshold_netif=(utcnow() - datetime.timedelta(days=CONFIG["router_stat_netif"])).timestamp()
start_time = time.time()
mysql.execute("""
rowsaffected = mysql.execute("""
DELETE s FROM router_stats AS s
LEFT JOIN router AS r ON s.router = r.id
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
""",(threshold,))
mysql.commit()
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete stats: %.3f seconds" % (time.time() - start_time))
print("--- Delete stats: %.3f seconds ---" % (time.time() - start_time))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
print("--- Deleted %i rows from stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
time.sleep(10)
start_time = time.time()
mysql.execute("""
DELETE s FROM router_stats_gw AS s
LEFT JOIN router AS r ON s.router = r.id
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
rowsaffected = mysql.execute("""
DELETE FROM router_stats_gw
WHERE router_stats_gw.time < %s
""",(threshold,))
mysql.commit()
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete gw-stats: %.3f seconds" % (time.time() - start_time))
print("--- Delete gw-stats: %.3f seconds ---" % (time.time() - start_time))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from gw-stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
print("--- Deleted %i rows from gw-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
time.sleep(10)
start_time = time.time()
mysql.execute("""
DELETE s FROM router_stats_neighbor AS s
LEFT JOIN router AS r ON s.router = r.id
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
rowsaffected = mysql.execute("""
DELETE FROM router_stats_neighbor
WHERE router_stats_neighbor.time < %s
""",(threshold,))
mysql.commit()
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete neighbor-stats: %.3f seconds" % (time.time() - start_time))
print("--- Delete neighbor-stats: %.3f seconds ---" % (time.time() - start_time))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from neighbor-stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
print("--- Deleted %i rows from neighbor-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
time.sleep(10)
start_time = time.time()
mysql.execute("""
UPDATE router_stats_netif AS s
LEFT JOIN router AS r ON s.router = r.id
SET s.deletebit = 1
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
""",(threshold_netif,))
mysql.commit()
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Update netif stats: %.3f seconds" % (time.time() - start_time))
print("--- Update netif stats: %.3f seconds ---" % (time.time() - start_time))
time.sleep(30)
minustime=0
rowsaffected=1
allrows=0
start_time = time.time()
while rowsaffected > 0:
try:
rowsaffected = mysql.execute("""
DELETE FROM router_stats_netif
WHERE deletebit = 1
LIMIT 50000
""")
WHERE router_stats_netif.time < %s
LIMIT 100000
""",(threshold_netif,))
mysql.commit()
allrows += rowsaffected
except my.OperationalError:
rowsaffected = 1
time.sleep(10)
minustime += 10
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete netif stats: %.3f seconds" % (time.time() - start_time - minustime))
print("--- Delete netif stats: %.3f seconds ---" % (time.time() - start_time - minustime))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from netif stats: %.3f seconds" % (allrows,time.time() - start_time - minustime))
print("--- Deleted %i rows from netif stats: %.3f seconds ---" % (allrows,time.time() - start_time - minustime))
start_time = time.time()
allrows=0
events = mysql.fetchall("""
SELECT router, COUNT(time) AS count FROM router_events
GROUP BY router
@ -461,16 +450,16 @@ def delete_old_stats(mysql):
for e in events:
delnum = int(e["count"] - CONFIG["event_num_entries"])
if delnum > 0:
mysql.execute("""
allrows += mysql.execute("""
DELETE FROM router_events
WHERE router = %s
ORDER BY time ASC
LIMIT %s
""",(e["router"],delnum,))
mysql.commit()
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete events: %.3f seconds" % (time.time() - start_time))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from events: %.3f seconds" % (allrows,time.time() - start_time))
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "-------")
print("--- Delete events: %.3f seconds ---" % (time.time() - start_time))
print("--- Deleted %i rows from events: %.3f seconds ---" % (allrows,time.time() - start_time))
def events_append(mysql,router_id,event,comment):
mysql.execute("""