routertools/deletestats: Split and improve netif stats deletion

This patch reorganizes the deletion of old stats:
- Commits are done after each step
- Netif stats deletion is split into UPDATE and DELETE
- Delays are added, to reduce locking

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>
This commit is contained in:
Adrian Schmutzler 2017-12-04 16:09:42 +01:00
parent 7e821b2ae7
commit 80f20f4d0c

View File

@ -7,10 +7,12 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from ffmap.mysqltools import FreifunkMySQL from ffmap.mysqltools import FreifunkMySQL
from ffmap.misc import * from ffmap.misc import *
from ffmap.config import CONFIG from ffmap.config import CONFIG
import MySQLdb as my
import lxml.etree import lxml.etree
import datetime import datetime
import requests import requests
import time
from bson import SON from bson import SON
from contextlib import suppress from contextlib import suppress
@ -271,18 +273,37 @@ def delete_old_stats(mysql):
LEFT JOIN router AS r ON s.router = r.id LEFT JOIN router AS r ON s.router = r.id
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
""",(threshold,)) """,(threshold,))
mysql.commit()
time.sleep(10)
mysql.execute(""" mysql.execute("""
DELETE s FROM router_stats_neighbor AS s DELETE s FROM router_stats_neighbor AS s
LEFT JOIN router AS r ON s.router = r.id LEFT JOIN router AS r ON s.router = r.id
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
""",(threshold,)) """,(threshold,))
mysql.commit()
time.sleep(10)
mysql.execute(""" mysql.execute("""
DELETE s FROM router_stats_netif AS s UPDATE router_stats_netif AS s
LEFT JOIN router AS r ON s.router = r.id LEFT JOIN router AS r ON s.router = r.id
SET s.deletebit = 1
WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL) WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
""",(threshold,)) """,(threshold,))
mysql.commit()
time.sleep(30)
rowsaffected=1
while rowsaffected > 0:
try:
rowsaffected = mysql.execute("""
DELETE FROM router_stats_netif
WHERE deletebit = 1
LIMIT 50000
""")
mysql.commit()
except my.OperationalError:
rowsaffected = 1
time.sleep(10)
events = mysql.fetchall(""" events = mysql.fetchall("""
SELECT router, COUNT(time) AS count FROM router_events SELECT router, COUNT(time) AS count FROM router_events