routertools/deletestats: Split and improve netif stats deletion

This patch reorganizes the deletion of old stats:
- Commits are done after each step
- Netif stats deletion is split into UPDATE and DELETE
- Delays are added, to reduce locking

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

ffmap/routertools.py

@@ -7,10 +7,12 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
 from ffmap.mysqltools import FreifunkMySQL
 from ffmap.misc import *
 from ffmap.config import CONFIG
+import MySQLdb as my
 
 import lxml.etree
 import datetime
 import requests
+import time
 from bson import SON
 from contextlib import suppress
 
@@ -271,18 +273,37 @@ def delete_old_stats(mysql):
 		LEFT JOIN router AS r ON s.router = r.id
 		WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
 	""",(threshold,))
+	mysql.commit()
 
+	time.sleep(10)
 	mysql.execute("""
 		DELETE s FROM router_stats_neighbor AS s
 		LEFT JOIN router AS r ON s.router = r.id
 		WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
 	""",(threshold,))
+	mysql.commit()
 
+	time.sleep(10)
 	mysql.execute("""
-		DELETE s FROM router_stats_netif AS s
+		UPDATE router_stats_netif AS s
 		LEFT JOIN router AS r ON s.router = r.id
+		SET s.deletebit = 1
 		WHERE s.time < %s AND (r.status = 'online' OR r.status IS NULL)
 	""",(threshold,))
+	mysql.commit()
+	time.sleep(30)
+	rowsaffected=1
+	while rowsaffected > 0:
+		try:
+			rowsaffected = mysql.execute("""
+				DELETE FROM router_stats_netif
+				WHERE deletebit = 1
+				LIMIT 50000
+			""")
+			mysql.commit()
+		except my.OperationalError:
+			rowsaffected = 1
+		time.sleep(10)
 
 	events = mysql.fetchall("""
 		SELECT router, COUNT(time) AS count FROM router_events