From b38af9a74b6d49de58028893f5b593f3f63f948c Mon Sep 17 00:00:00 2001 From: Adrian Schmutzler Date: Mon, 27 Aug 2018 19:48:40 +0200 Subject: [PATCH] Global: Read V1 hoods from KeyXchange This obsoletes the hood initialization file. Hood are capitalized and "V1" is added at the end. This may require a rename of the existing hood stats. Signed-off-by: Adrian Schmutzler --- ffmap/db/hooddata.py | 201 ------------------------------------- ffmap/db/hoods.py | 5 + ffmap/db/init_db.py | 1 - ffmap/hoodtools.py | 44 ++++++++ scripts/calcglobalstats.py | 3 +- 5 files changed, 51 insertions(+), 203 deletions(-) delete mode 100755 ffmap/db/hooddata.py diff --git a/ffmap/db/hooddata.py b/ffmap/db/hooddata.py deleted file mode 100755 index 8666fce..0000000 --- a/ffmap/db/hooddata.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/python - -import os -import sys -sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) - -from ffmap.mysqltools import FreifunkMySQL -import math - -mysql = FreifunkMySQL() - -hoods = [ -{ - "keyxchange_id": 0, - "name": "NoCoordinates", - "net": "" -}, -{ - "keyxchange_id": 1, - "name": "Default", - "net": "10.50.16.0/20" -}, -{ - "keyxchange_id": 2, - "name": "Fuerth", - "net": "10.50.32.0/21", - "position": {"lng": 10.966, "lat": 49.4814} -}, -{ - "keyxchange_id": 3, - "name": "Nuernberg", - "net": "10.50.40.0/21", - "position": {"lng": 11.05, "lat": 49.444} -}, -{ - "keyxchange_id": 4, - "name": "Ansbach", - "net": "10.50.48.0/21", - "position": {"lng": 10.571667, "lat": 49.300833} -}, -{ - "keyxchange_id": 5, - "name": "Hassberge", - "net": "10.50.56.0/21", - "position": {"lng": 10.568013390003, "lat": 50.093555895082} -}, -{ - "keyxchange_id": 6, - "name": "Erlangen", - "net": "10.50.64.0/21", - "position": {"lng": 11.0019221, "lat": 49.6005981} -}, -{ - "keyxchange_id": 7, - "name": "Wuerzburg", - "net": "10.50.72.0/21", - "position": {"lng": 9.93489, "lat": 49.79688} -}, -{ - "keyxchange_id": 8, - "name": "Bamberg", - "net": "10.50.124.0/22", - "position": {"lng": 10.95, "lat": 49.89} -}, -{ - "keyxchange_id": 9, - "name": "BGL", - "net": "10.50.80.0/21", - "position": {"lng": 12.8825, "lat": 47.7314} -}, -{ - "keyxchange_id": 10, - "name": "HassbergeSued", - "net": "10.50.60.0/22", - "position": {"lng": 10.568013390003, "lat": 50.04501} -}, -{ - "keyxchange_id": 11, - "name": "NbgLand", - "net": "10.50.88.0/21", - "position": {"lng": 11.162796020507812, "lat": 49.39200496388418} -}, -{ - "keyxchange_id": 12, - "name": "Hof", - "net": "10.50.104.0/21", - "position": {"lng": 11.917545, "lat": 50.312209} -}, -{ - "keyxchange_id": 13, - "name": "Aschaffenburg", - "net": "10.50.96.0/22", - "position": {"lng": 9.146826, "lat": 49.975661} -}, -{ - "keyxchange_id": 14, - "name": "Marktredwitz", - "net": "10.50.112.0/22", - "position": {"lng": 12.084797, "lat": 50.002915} -}, -{ - "keyxchange_id": 15, - "name": "Forchheim", - "net": "10.50.116.0/22", - "position": {"lng": 11.059474, "lat": 49.718820} -}, -{ - "keyxchange_id": 16, - "name": "Muenchberg", - "net": "10.50.120.0/22", - "position": {"lng": 11.79, "lat": 50.19} -}, -{ - "keyxchange_id": 17, - "name": "Adelsdorf", - "net": "10.50.144.0/22", - "position": {"lng": 10.894235, "lat": 49.709945} -}, -{ - "keyxchange_id": 18, - "name": "Schweinfurt", - "net": "10.50.160.0/22", - "position": {"lng": 10.21267, "lat": 50.04683} -}, -{ - "keyxchange_id": 19, - "name": "ErlangenWest", - "net": "10.50.152.0/22", - "position": {"lng": 10.984488, "lat": 49.6035981} -}, -{ - "keyxchange_id": 20, - "name": "Ebermannstadt", - "net": "10.50.148.0/22", - "position": {"lng": 11.18538, "lat": 49.78173} -}, -{ - "keyxchange_id": 21, - "name": "Lauf", - "net": "10.50.156.0/22", - "position": {"lng": 11.278789, "lat": 49.509972} -}, -{ - "keyxchange_id": 22, - "name": "Bayreuth", - "net": "10.50.168.0/22", - "position": {"lng": 11.580566, "lat": 49.94814} -}, -{ - "keyxchange_id": 23, - "name": "Fichtelberg", - "net": "10.50.172.0/22", - "position": {"lng": 11.852292, "lat": 49.998920} -}, -{ - "keyxchange_id": 24, - "name": "Rehau", - "net": "10.50.176.0/22", - "position": {"lng": 12.035305, "lat": 50.247594} -}, - { - "keyxchange_id": 25, - "name": "Coburg", - "net": "10.50.180.0/22", - "position": {"lng": 10.964414, "lat": 50.259675} -}, -{ - "keyxchange_id": 26, - "name": "Ebern", - "net": "10.50.184.0/22", - "position": {"lng": 10.798395, "lat": 50.095572} -}, -{ - "keyxchange_id": 27, - "name": "Arnstein", - "net": "10.50.188.0/22", - "position": {"lng": 9.970957, "lat": 49.978117} -}, -{ - "keyxchange_id": 28, - "name": "Erlenbach", - "net": "10.50.192.0/22", - "position": {"lng": 9.157491, "lat": 49.803930} -}] - -for h in hoods: - coord = h.get("position",{}) - if coord.get("lat"): - cos_lat = math.cos(math.radians(coord.get("lat"))) - sin_lat = math.sin(math.radians(coord.get("lat"))) - else: - cos_lat = None - sin_lat = None - - mysql.execute(""" - INSERT INTO hoods (id, name, net, lat, lng, cos_lat, sin_lat) - VALUES (%s, %s, %s, %s, %s, %s, %s) - """,(h["keyxchange_id"],h["name"],h["net"],coord.get("lat"),coord.get("lng"),cos_lat,sin_lat,)) - -mysql.commit() -mysql.close() diff --git a/ffmap/db/hoods.py b/ffmap/db/hoods.py index 7c4cb12..ae82922 100755 --- a/ffmap/db/hoods.py +++ b/ffmap/db/hoods.py @@ -30,6 +30,11 @@ mysql.execute(""" ADD KEY `sin_lat` (`sin_lat`) """) +mysql.execute(""" + INSERT INTO hoods (id, name, net, lat, lng, cos_lat, sin_lat) + VALUES (%s, %s, %s, %s, %s, %s, %s) +""",(0,NoCoordinates,"",None,None,None,None,)) + mysql.execute(""" CREATE TABLE hoodsv2 ( `id` int(11) NOT NULL, diff --git a/ffmap/db/init_db.py b/ffmap/db/init_db.py index 8b0e975..b049a2a 100755 --- a/ffmap/db/init_db.py +++ b/ffmap/db/init_db.py @@ -5,4 +5,3 @@ import hoods import stats import gws import users -import hooddata diff --git a/ffmap/hoodtools.py b/ffmap/hoodtools.py index 753cf5e..936e492 100644 --- a/ffmap/hoodtools.py +++ b/ffmap/hoodtools.py @@ -7,6 +7,7 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) from ffmap.mysqltools import FreifunkMySQL import urllib.request, urllib.error, json +import math def update_hoods_v2(mysql): try: @@ -39,3 +40,46 @@ def update_hoods_v2(mysql): except urllib.error.HTTPError as e: return + +def update_hoods_v1(mysql): + try: + with urllib.request.urlopen("http://keyserver.freifunk-franken.de/fff/hoods.php") as url: + hoodskx = json.loads(url.read().decode()) + + kx_keys = [] + kx_data = [] + for kx in hoodskx: + if kx["id"]==0: + continue # Skip Trainstation/NoCoordinates + kx_keys.append(kx["id"]) + if kx.get("lat",None): + cos_lat = math.cos(math.radians(kx["lat"])) + sin_lat = math.sin(math.radians(kx["lat"])) + else: + cos_lat = None + sin_lat = None + kx["name"] = kx["name"][0].upper() + kx["name"][1:] + "V1" + kx_data.append((kx["id"],kx["name"],kx["net"],kx.get("lat",None),kx.get("lon",None),cos_lat,sin_lat,)) + + # Delete entries in DB where hood is missing in KeyXchange + db_keys = mysql.fetchall("SELECT id FROM hoods",(),"id") + for n in db_keys: + if n in kx_keys or n==0: + continue + mysql.execute("DELETE FROM hoods WHERE id = %s",(n,)) + + # Create/update entries from KeyXchange to DB + mysql.executemany(""" + INSERT INTO hoods (id, name, net, lat, lng, cos_lat, sin_lat) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + name=VALUES(name), + net=VALUES(net), + lat=VALUES(lat), + lng=VALUES(lng), + cos_lat=VALUES(cos_lat), + sin_lat=VALUES(sin_lat) + """,kx_data) + + except urllib.error.HTTPError as e: + return diff --git a/scripts/calcglobalstats.py b/scripts/calcglobalstats.py index f274034..c1fd760 100755 --- a/scripts/calcglobalstats.py +++ b/scripts/calcglobalstats.py @@ -10,7 +10,7 @@ from ffmap.routertools import * from ffmap.maptools import * from ffmap.mysqltools import FreifunkMySQL from ffmap.stattools import record_global_stats, record_hood_stats, record_gw_stats -from ffmap.hoodtools import update_hoods_v2 +from ffmap.hoodtools import update_hoods_v2, update_hoods_v1 import time start_time = time.time() @@ -21,6 +21,7 @@ detect_orphaned_routers(mysql) delete_orphaned_routers(mysql) #delete_old_stats(mysql) # Only execute once daily, takes 2 minutes update_hoods_v2(mysql) +update_hoods_v1(mysql) record_global_stats(mysql) record_hood_stats(mysql) record_gw_stats(mysql)