Global: Add V2 hoods to table

This requires changes to the MySQL database!

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

ffmap/db/hoods.py

@@ -30,6 +30,24 @@ mysql.execute("""
 		ADD KEY `sin_lat` (`sin_lat`)
 """)
 
+mysql.execute("""
+	CREATE TABLE hoodsv2 (
+		`id` int(11) NOT NULL,
+		`name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+		`net` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+		`lat` double DEFAULT NULL,
+		`lng` double DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE hoodsv2
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`),
+		ADD KEY `lat` (`lat`),
+		ADD KEY `lng` (`lng`)
+""")
+
 mysql.commit()
 
 mysql.close()

ffmap/hoodtools.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+
+import urllib.request, urllib.error, json
+
+def update_hoods_v2(mysql):
+	try:
+		with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
+			hoodskx = json.loads(url.read().decode())
+		
+		kx_keys = []
+		kx_data = []
+		for kx in hoodskx:
+			kx_keys.append(kx["id"])
+			kx_data.append((kx["id"],kx["name"],kx["net"],kx.get("lat",None),kx.get("lon",None),))
+
+		# Delete entries in DB where hood is missing in KeyXchange
+		db_keys = mysql.fetchall("SELECT id FROM hoodsv2",(),"id")
+		for n in db_keys:
+			if n in kx_keys:
+				continue
+			mysql.execute("DELETE FROM hoodsv2 WHERE id = %s",(n,))
+
+		# Create/update entries from KeyXchange to DB
+		mysql.executemany("""
+			INSERT INTO hoodsv2 (id, name, net, lat, lng)
+			VALUES (%s, %s, %s, %s, %s)
+			ON DUPLICATE KEY UPDATE
+				name=VALUES(name),
+				net=VALUES(net),
+				lat=VALUES(lat),
+				lng=VALUES(lng)
+		""",kx_data)
+
+	except urllib.error.HTTPError as e:
+		return

ffmap/maptools.py

@@ -219,17 +219,15 @@ def update_mapnik_csv(mysql):
 			hoods.append([x, y])
 		draw_voronoi_lines(csv, hoods)
 
-	with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
-		dbhoodsv2 = json.loads(url.read().decode())
-	
+	dbhoodsv2 = mysql.fetchall("""
+		SELECT name, lat, lng FROM hoodsv2
+		WHERE lat IS NOT NULL AND lng IS NOT NULL
+	""")
 	with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv:
 		csv.write("lng,lat,name\n")
-		
 		for hood in dbhoodsv2:
-			if not ( 'lon' in hood and 'lat' in hood ):
-				continue
 			csv.write("%f,%f,\"%s\"\n" % (
-				hood["lon"],
+				hood["lng"],
 				hood["lat"],
 				hood["name"]
 			))
@@ -239,10 +237,8 @@ def update_mapnik_csv(mysql):
 		hoods = []
 		
 		for hood in dbhoodsv2:
-			if not ( 'lon' in hood and 'lat' in hood ):
-				continue
 			# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
-			x, y = merc_sphere(hood["lon"], hood["lat"])
+			x, y = merc_sphere(hood["lng"], hood["lat"])
 			hoods.append([x, y])
 		draw_voronoi_lines(csv, hoods)
 

ffmap/misc.py

@@ -136,7 +136,7 @@ def defrag_table(mysql,table,sleep):
 	print("--- Defragmented table %s: %.3f seconds ---" % (table,end_time - start_time))
 
 def defrag_all(mysql,doall=False):
-	alltables = ('gw','gw_admin','gw_netif','hoods','netifs','router','router_events','router_gw','router_ipv6','router_neighbor','router_netif','users')
+	alltables = ('gw','gw_admin','gw_netif','hoods','hoodsv2','netifs','router','router_events','router_gw','router_ipv6','router_neighbor','router_netif','users')
 	stattables = ('router_stats','router_stats_gw','router_stats_neighbor','router_stats_netif','stats_global','stats_gw','stats_hood')
 
 	for t in alltables:

scripts/calcglobalstats.py

@@ -10,6 +10,7 @@ from ffmap.routertools import *
 from ffmap.maptools import *
 from ffmap.mysqltools import FreifunkMySQL
 from ffmap.stattools import record_global_stats, record_hood_stats, record_gw_stats
+from ffmap.hoodtools import update_hoods_v2
 
 import time
 start_time = time.time()
@@ -19,6 +20,7 @@ detect_offline_routers(mysql)
 detect_orphaned_routers(mysql)
 delete_orphaned_routers(mysql)
 #delete_old_stats(mysql) # Only execute once daily, takes 2 minutes
+update_hoods_v2(mysql)
 record_global_stats(mysql)
 record_hood_stats(mysql)
 record_gw_stats(mysql)