diff --git a/ffmap/db/hoods.py b/ffmap/db/hoods.py index 2dc0965..7c4cb12 100755 --- a/ffmap/db/hoods.py +++ b/ffmap/db/hoods.py @@ -30,6 +30,24 @@ mysql.execute(""" ADD KEY `sin_lat` (`sin_lat`) """) +mysql.execute(""" + CREATE TABLE hoodsv2 ( + `id` int(11) NOT NULL, + `name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `net` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `lat` double DEFAULT NULL, + `lng` double DEFAULT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + ALTER TABLE hoodsv2 + ADD PRIMARY KEY (`id`), + ADD UNIQUE KEY `name` (`name`), + ADD KEY `lat` (`lat`), + ADD KEY `lng` (`lng`) +""") + mysql.commit() mysql.close() diff --git a/ffmap/hoodtools.py b/ffmap/hoodtools.py new file mode 100644 index 0000000..753cf5e --- /dev/null +++ b/ffmap/hoodtools.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 + +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) + +from ffmap.mysqltools import FreifunkMySQL + +import urllib.request, urllib.error, json + +def update_hoods_v2(mysql): + try: + with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: + hoodskx = json.loads(url.read().decode()) + + kx_keys = [] + kx_data = [] + for kx in hoodskx: + kx_keys.append(kx["id"]) + kx_data.append((kx["id"],kx["name"],kx["net"],kx.get("lat",None),kx.get("lon",None),)) + + # Delete entries in DB where hood is missing in KeyXchange + db_keys = mysql.fetchall("SELECT id FROM hoodsv2",(),"id") + for n in db_keys: + if n in kx_keys: + continue + mysql.execute("DELETE FROM hoodsv2 WHERE id = %s",(n,)) + + # Create/update entries from KeyXchange to DB + mysql.executemany(""" + INSERT INTO hoodsv2 (id, name, net, lat, lng) + VALUES (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + name=VALUES(name), + net=VALUES(net), + lat=VALUES(lat), + lng=VALUES(lng) + """,kx_data) + + except urllib.error.HTTPError as e: + return diff --git a/ffmap/maptools.py b/ffmap/maptools.py index d5cd5f4..2d04da3 100644 --- a/ffmap/maptools.py +++ b/ffmap/maptools.py @@ -219,17 +219,15 @@ def update_mapnik_csv(mysql): hoods.append([x, y]) draw_voronoi_lines(csv, hoods) - with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: - dbhoodsv2 = json.loads(url.read().decode()) - + dbhoodsv2 = mysql.fetchall(""" + SELECT name, lat, lng FROM hoodsv2 + WHERE lat IS NOT NULL AND lng IS NOT NULL + """) with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv: csv.write("lng,lat,name\n") - for hood in dbhoodsv2: - if not ( 'lon' in hood and 'lat' in hood ): - continue csv.write("%f,%f,\"%s\"\n" % ( - hood["lon"], + hood["lng"], hood["lat"], hood["name"] )) @@ -239,10 +237,8 @@ def update_mapnik_csv(mysql): hoods = [] for hood in dbhoodsv2: - if not ( 'lon' in hood and 'lat' in hood ): - continue # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat - x, y = merc_sphere(hood["lon"], hood["lat"]) + x, y = merc_sphere(hood["lng"], hood["lat"]) hoods.append([x, y]) draw_voronoi_lines(csv, hoods) diff --git a/ffmap/misc.py b/ffmap/misc.py index 3081151..944a2e6 100644 --- a/ffmap/misc.py +++ b/ffmap/misc.py @@ -136,7 +136,7 @@ def defrag_table(mysql,table,sleep): print("--- Defragmented table %s: %.3f seconds ---" % (table,end_time - start_time)) def defrag_all(mysql,doall=False): - alltables = ('gw','gw_admin','gw_netif','hoods','netifs','router','router_events','router_gw','router_ipv6','router_neighbor','router_netif','users') + alltables = ('gw','gw_admin','gw_netif','hoods','hoodsv2','netifs','router','router_events','router_gw','router_ipv6','router_neighbor','router_netif','users') stattables = ('router_stats','router_stats_gw','router_stats_neighbor','router_stats_netif','stats_global','stats_gw','stats_hood') for t in alltables: diff --git a/scripts/calcglobalstats.py b/scripts/calcglobalstats.py index 7b632a3..f274034 100755 --- a/scripts/calcglobalstats.py +++ b/scripts/calcglobalstats.py @@ -10,6 +10,7 @@ from ffmap.routertools import * from ffmap.maptools import * from ffmap.mysqltools import FreifunkMySQL from ffmap.stattools import record_global_stats, record_hood_stats, record_gw_stats +from ffmap.hoodtools import update_hoods_v2 import time start_time = time.time() @@ -19,6 +20,7 @@ detect_offline_routers(mysql) detect_orphaned_routers(mysql) delete_orphaned_routers(mysql) #delete_old_stats(mysql) # Only execute once daily, takes 2 minutes +update_hoods_v2(mysql) record_global_stats(mysql) record_hood_stats(mysql) record_gw_stats(mysql)