mirror of
https://github.com/FreifunkFranken/fff-monitoring.git
synced 2024-06-15 20:04:03 +02:00
scripts/deletestats: Do not evaluate status in stats deletion
Previously, old stats had been only deleted if the router was online. This required a join with the router table, which caused locking issues when writing to this table by alfred request. Now, we just delete old stats ignoring the router state, except for the router_stats table, which is smaller and thus remains as before. In addition, this patch logs the number of rows affected. Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>
This commit is contained in:
parent
804ce80472
commit
58cc2e167f
|
@ -7,8 +7,8 @@ CONFIG = {
|
|||
"offline_threshold_minutes": 15, # Router switches to offline after X minutes
|
||||
"orphan_threshold_days": 7, # Router switches to orphaned state after X days
|
||||
"delete_threshold_days": 180, # Router is deleted after X days
|
||||
"router_stat_days": 30, # Router stats are collected for X days (if online)
|
||||
"router_stat_netif": 21, # Router stats for netifs are collected for X days (if online)
|
||||
"router_stat_days": 30, # Router stats are collected for X days
|
||||
"router_stat_netif": 21, # Router stats for netifs are collected for X days
|
||||
"router_stat_mindiff_secs": 10, # Time difference (uptime) in seconds required for a new entry in router stats
|
||||
"router_stat_mindiff_default": 270, # Time difference (router stats tables) in seconds required for a new entry in router stats
|
||||
"router_stat_mindiff_netif": 570, # Time difference (router netif stats) in seconds required for a new entry in router stats
|
||||
|
|
|
@ -390,69 +390,58 @@ def delete_old_stats(mysql):
|
|||
threshold_netif=(utcnow() - datetime.timedelta(days=CONFIG["router_stat_netif"])).timestamp()
|
||||
|
||||
start_time = time.time()
|
||||
mysql.execute("""
|
||||
rowsaffected = mysql.execute("""
|
||||
DELETE s FROM router_stats AS s
|
||||
LEFT JOIN router AS r ON s.router = r.id
|
||||
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
|
||||
""",(threshold,))
|
||||
mysql.commit()
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete stats: %.3f seconds" % (time.time() - start_time))
|
||||
print("--- Delete stats: %.3f seconds ---" % (time.time() - start_time))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
|
||||
print("--- Deleted %i rows from stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
|
||||
|
||||
time.sleep(10)
|
||||
start_time = time.time()
|
||||
mysql.execute("""
|
||||
DELETE s FROM router_stats_gw AS s
|
||||
LEFT JOIN router AS r ON s.router = r.id
|
||||
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
|
||||
rowsaffected = mysql.execute("""
|
||||
DELETE FROM router_stats_gw
|
||||
WHERE router_stats_gw.time < %s
|
||||
""",(threshold,))
|
||||
mysql.commit()
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete gw-stats: %.3f seconds" % (time.time() - start_time))
|
||||
print("--- Delete gw-stats: %.3f seconds ---" % (time.time() - start_time))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from gw-stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
|
||||
print("--- Deleted %i rows from gw-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
|
||||
|
||||
time.sleep(10)
|
||||
start_time = time.time()
|
||||
mysql.execute("""
|
||||
DELETE s FROM router_stats_neighbor AS s
|
||||
LEFT JOIN router AS r ON s.router = r.id
|
||||
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
|
||||
rowsaffected = mysql.execute("""
|
||||
DELETE FROM router_stats_neighbor
|
||||
WHERE router_stats_neighbor.time < %s
|
||||
""",(threshold,))
|
||||
mysql.commit()
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete neighbor-stats: %.3f seconds" % (time.time() - start_time))
|
||||
print("--- Delete neighbor-stats: %.3f seconds ---" % (time.time() - start_time))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from neighbor-stats: %.3f seconds" % (rowsaffected,time.time() - start_time))
|
||||
print("--- Deleted %i rows from neighbor-stats: %.3f seconds ---" % (rowsaffected,time.time() - start_time))
|
||||
|
||||
time.sleep(10)
|
||||
start_time = time.time()
|
||||
mysql.execute("""
|
||||
UPDATE router_stats_netif AS s
|
||||
LEFT JOIN router AS r ON s.router = r.id
|
||||
SET s.deletebit = 1
|
||||
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
|
||||
""",(threshold_netif,))
|
||||
mysql.commit()
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Update netif stats: %.3f seconds" % (time.time() - start_time))
|
||||
print("--- Update netif stats: %.3f seconds ---" % (time.time() - start_time))
|
||||
|
||||
time.sleep(30)
|
||||
minustime=0
|
||||
rowsaffected=1
|
||||
allrows=0
|
||||
start_time = time.time()
|
||||
while rowsaffected > 0:
|
||||
try:
|
||||
rowsaffected = mysql.execute("""
|
||||
DELETE FROM router_stats_netif
|
||||
WHERE deletebit = 1
|
||||
LIMIT 50000
|
||||
""")
|
||||
WHERE router_stats_netif.time < %s
|
||||
LIMIT 100000
|
||||
""",(threshold_netif,))
|
||||
mysql.commit()
|
||||
allrows += rowsaffected
|
||||
except my.OperationalError:
|
||||
rowsaffected = 1
|
||||
time.sleep(10)
|
||||
minustime += 10
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete netif stats: %.3f seconds" % (time.time() - start_time - minustime))
|
||||
print("--- Delete netif stats: %.3f seconds ---" % (time.time() - start_time - minustime))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from netif stats: %.3f seconds" % (allrows,time.time() - start_time - minustime))
|
||||
print("--- Deleted %i rows from netif stats: %.3f seconds ---" % (allrows,time.time() - start_time - minustime))
|
||||
|
||||
start_time = time.time()
|
||||
allrows=0
|
||||
events = mysql.fetchall("""
|
||||
SELECT router, COUNT(time) AS count FROM router_events
|
||||
GROUP BY router
|
||||
|
@ -461,16 +450,16 @@ def delete_old_stats(mysql):
|
|||
for e in events:
|
||||
delnum = int(e["count"] - CONFIG["event_num_entries"])
|
||||
if delnum > 0:
|
||||
mysql.execute("""
|
||||
allrows += mysql.execute("""
|
||||
DELETE FROM router_events
|
||||
WHERE router = %s
|
||||
ORDER BY time ASC
|
||||
LIMIT %s
|
||||
""",(e["router"],delnum,))
|
||||
mysql.commit()
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Delete events: %.3f seconds" % (time.time() - start_time))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Deleted %i rows from events: %.3f seconds" % (allrows,time.time() - start_time))
|
||||
writelog(CONFIG["debug_dir"] + "/deletetime.txt", "-------")
|
||||
print("--- Delete events: %.3f seconds ---" % (time.time() - start_time))
|
||||
print("--- Deleted %i rows from events: %.3f seconds ---" % (allrows,time.time() - start_time))
|
||||
|
||||
def events_append(mysql,router_id,event,comment):
|
||||
mysql.execute("""
|
||||
|
|
Loading…
Reference in New Issue
Block a user