diff --git a/copyusers.py b/copyusers.py new file mode 100755 index 0000000..6d7e8b5 --- /dev/null +++ b/copyusers.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 + +from ffmap.mysqltools import FreifunkMySQL + +import pymongo +from bson.json_util import dumps as bson2json +from bson.objectid import ObjectId +import base64 +import datetime + +client = MongoClient(tz_aware=True, connect=False) +db = client.freifunk + +users = db.users.find({}, {"nickname": 1, "password":1, "email": 1, "token": 1, "created": 1, "admin": 1}) + +mysql = FreifunkMySQL() +cur = mysql.cursor() +for u in users: + #print(u) + cur.execute(""" + INSERT INTO users (nickname, password, token, email, created, admin) + VALUES (%s, %s, %s, %s, %s, %s) + """,(u.get("nickname"),u.get("password"),u.get("token"),u.get("email",""),u.get("created"),u.get("admin",0),)) +mysql.commit() +mysql.close() diff --git a/ffmap/db/hooddata.py b/ffmap/db/hooddata.py new file mode 100755 index 0000000..84440d7 --- /dev/null +++ b/ffmap/db/hooddata.py @@ -0,0 +1,196 @@ +#!/usr/bin/python + +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) + +from ffmap.mysqltools import FreifunkMySQL +import math + +mysql = FreifunkMySQL() + +hoods = [ +{ + "keyxchange_id": 1, + "name": "Default", + "net": "10.50.16.0/20" +}, +{ + "keyxchange_id": 2, + "name": "Fuerth", + "net": "10.50.32.0/21", + "position": {"lng": 10.966, "lat": 49.4814} +}, +{ + "keyxchange_id": 3, + "name": "Nuernberg", + "net": "10.50.40.0/21", + "position": {"lng": 11.05, "lat": 49.444} +}, +{ + "keyxchange_id": 4, + "name": "Ansbach", + "net": "10.50.48.0/21", + "position": {"lng": 10.571667, "lat": 49.300833} +}, +{ + "keyxchange_id": 5, + "name": "Hassberge", + "net": "10.50.56.0/21", + "position": {"lng": 10.568013390003, "lat": 50.093555895082} +}, +{ + "keyxchange_id": 6, + "name": "Erlangen", + "net": "10.50.64.0/21", + "position": {"lng": 11.0019221, "lat": 49.6005981} +}, +{ + "keyxchange_id": 7, + "name": "Wuerzburg", + "net": "10.50.72.0/21", + "position": {"lng": 9.93489, "lat": 49.79688} +}, +{ + "keyxchange_id": 8, + "name": "Bamberg", + "net": "10.50.124.0/22", + "position": {"lng": 10.95, "lat": 49.89} +}, +{ + "keyxchange_id": 9, + "name": "BGL", + "net": "10.50.80.0/21", + "position": {"lng": 12.8825, "lat": 47.7314} +}, +{ + "keyxchange_id": 10, + "name": "HassbergeSued", + "net": "10.50.60.0/22", + "position": {"lng": 10.568013390003, "lat": 50.04501} +}, +{ + "keyxchange_id": 11, + "name": "NbgLand", + "net": "10.50.88.0/21", + "position": {"lng": 11.162796020507812, "lat": 49.39200496388418} +}, +{ + "keyxchange_id": 12, + "name": "Hof", + "net": "10.50.104.0/21", + "position": {"lng": 11.917545, "lat": 50.312209} +}, +{ + "keyxchange_id": 13, + "name": "Aschaffenburg", + "net": "10.50.96.0/22", + "position": {"lng": 9.146826, "lat": 49.975661} +}, +{ + "keyxchange_id": 14, + "name": "Marktredwitz", + "net": "10.50.112.0/22", + "position": {"lng": 12.084797, "lat": 50.002915} +}, +{ + "keyxchange_id": 15, + "name": "Forchheim", + "net": "10.50.116.0/22", + "position": {"lng": 11.059474, "lat": 49.718820} +}, +{ + "keyxchange_id": 16, + "name": "Muenchberg", + "net": "10.50.120.0/22", + "position": {"lng": 11.79, "lat": 50.19} +}, +{ + "keyxchange_id": 17, + "name": "Adelsdorf", + "net": "10.50.144.0/22", + "position": {"lng": 10.894235, "lat": 49.709945} +}, +{ + "keyxchange_id": 18, + "name": "Schweinfurt", + "net": "10.50.160.0/22", + "position": {"lng": 10.21267, "lat": 50.04683} +}, +{ + "keyxchange_id": 19, + "name": "ErlangenWest", + "net": "10.50.152.0/22", + "position": {"lng": 10.984488, "lat": 49.6035981} +}, +{ + "keyxchange_id": 20, + "name": "Ebermannstadt", + "net": "10.50.148.0/22", + "position": {"lng": 11.18538, "lat": 49.78173} +}, +{ + "keyxchange_id": 21, + "name": "Lauf", + "net": "10.50.156.0/22", + "position": {"lng": 11.278789, "lat": 49.509972} +}, +{ + "keyxchange_id": 22, + "name": "Bayreuth", + "net": "10.50.168.0/22", + "position": {"lng": 11.580566, "lat": 49.94814} +}, +{ + "keyxchange_id": 23, + "name": "Fichtelberg", + "net": "10.50.172.0/22", + "position": {"lng": 11.852292, "lat": 49.998920} +}, +{ + "keyxchange_id": 24, + "name": "Rehau", + "net": "10.50.176.0/22", + "position": {"lng": 12.035305, "lat": 50.247594} +}, + { + "keyxchange_id": 25, + "name": "Coburg", + "net": "10.50.180.0/22", + "position": {"lng": 10.964414, "lat": 50.259675} +}, +{ + "keyxchange_id": 26, + "name": "Ebern", + "net": "10.50.184.0/22", + "position": {"lng": 10.798395, "lat": 50.095572} +}, +{ + "keyxchange_id": 27, + "name": "Arnstein", + "net": "10.50.188.0/22", + "position": {"lng": 9.970957, "lat": 49.978117} +}, +{ + "keyxchange_id": 28, + "name": "Erlenbach", + "net": "10.50.192.0/22", + "position": {"lng": 9.157491, "lat": 49.803930} +}] + +for h in hoods: + coord = h.get("position",{}) + if coord.get("lat"): + cos_lat = math.cos(math.radians(coord.get("lat"))) + sin_lat = math.sin(math.radians(coord.get("lat"))) + else: + cos_lat = None + sin_lat = None + + mysql.execute(""" + INSERT INTO hoods (id, name, net, lat, lng, cos_lat, sin_lat) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """,(h["keyxchange_id"],h["name"],h["net"],coord.get("lat"),coord.get("lng"),cos_lat,sin_lat,)) + +mysql.commit() +mysql.close() diff --git a/ffmap/db/hoods.py b/ffmap/db/hoods.py old mode 100644 new mode 100755 index 3bf5c7b..e4ebc53 --- a/ffmap/db/hoods.py +++ b/ffmap/db/hoods.py @@ -1,182 +1,33 @@ -#!/usr/bin/python +#!/usr/bin/python3 -from pymongo import MongoClient -client = MongoClient() +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) -db = client.freifunk +from ffmap.mysqltools import FreifunkMySQL -# create db indexes -db.hoods.delete_many({}) -db.hoods.create_index([("position", "2dsphere")]) +mysql = FreifunkMySQL() -hoods = [ -{ - "keyxchange_id": 1, - "name": "Default", - "net": "10.50.16.0/20" -}, -{ - "keyxchange_id": 2, - "name": "Fuerth", - "net": "10.50.32.0/21", - "position": {"type": "Point", "coordinates": [10.966, 49.4814]} -}, -{ - "keyxchange_id": 3, - "name": "Nuernberg", - "net": "10.50.40.0/21", - "position": {"type": "Point", "coordinates": [11.05, 49.444]} -}, -{ - "keyxchange_id": 4, - "name": "Ansbach", - "net": "10.50.48.0/21", - "position": {"type": "Point", "coordinates": [10.571667, 49.300833]} -}, -{ - "keyxchange_id": 5, - "name": "Hassberge", - "net": "10.50.56.0/21", - "position": {"type": "Point", "coordinates": [10.568013390003, 50.093555895082]} -}, -{ - "keyxchange_id": 6, - "name": "Erlangen", - "net": "10.50.64.0/21", - "position": {"type": "Point", "coordinates": [11.0019221, 49.6005981]} -}, -{ - "keyxchange_id": 7, - "name": "Wuerzburg", - "net": "10.50.72.0/21", - "position": {"type": "Point", "coordinates": [9.93489, 49.79688]} -}, -{ - "keyxchange_id": 8, - "name": "Bamberg", - "net": "10.50.124.0/22", - "position": {"type": "Point", "coordinates": [10.95, 49.89]} -}, -{ - "keyxchange_id": 9, - "name": "BGL", - "net": "10.50.80.0/21", - "position": {"type": "Point", "coordinates": [12.8825, 47.7314]} -}, -{ - "keyxchange_id": 10, - "name": "HassbergeSued", - "net": "10.50.60.0/22", - "position": {"type": "Point", "coordinates": [10.568013390003, 50.04501]} -}, -{ - "keyxchange_id": 11, - "name": "NbgLand", - "net": "10.50.88.0/21", - "position": {"type": "Point", "coordinates": [11.162796020507812, 49.39200496388418]} -}, -{ - "keyxchange_id": 12, - "name": "Hof", - "net": "10.50.104.0/21", - "position": {"type": "Point", "coordinates": [11.917545, 50.312209]} -}, -{ - "keyxchange_id": 13, - "name": "Aschaffenburg", - "net": "10.50.96.0/22", - "position": {"type": "Point", "coordinates": [9.146826, 49.975661]} -}, -{ - "keyxchange_id": 14, - "name": "Marktredwitz", - "net": "10.50.112.0/22", - "position": {"type": "Point", "coordinates": [12.084797, 50.002915]} -}, -{ - "keyxchange_id": 15, - "name": "Forchheim", - "net": "10.50.116.0/22", - "position": {"type": "Point", "coordinates": [11.059474, 49.718820]} -}, -{ - "keyxchange_id": 16, - "name": "Muenchberg", - "net": "10.50.120.0/22", - "position": {"type": "Point", "coordinates": [11.79, 50.19]} -}, -{ - "keyxchange_id": 17, - "name": "Adelsdorf", - "net": "10.50.144.0/22", - "position": {"type": "Point", "coordinates": [10.894235, 49.709945]} -}, -{ - "keyxchange_id": 18, - "name": "Schweinfurt", - "net": "10.50.160.0/22", - "position": {"type": "Point", "coordinates": [10.21267, 50.04683]} -}, -{ - "keyxchange_id": 19, - "name": "ErlangenWest", - "net": "10.50.152.0/22", - "position": {"type": "Point", "coordinates": [10.984488, 49.6035981]} -}, -{ - "keyxchange_id": 20, - "name": "Ebermannstadt", - "net": "10.50.148.0/22", - "position": {"type": "Point", "coordinates": [11.18538, 49.78173]} -}, -{ - "keyxchange_id": 21, - "name": "Lauf", - "net": "10.50.156.0/22", - "position": {"type": "Point", "coordinates": [11.278789, 49.509972]} -}, -{ - "keyxchange_id": 22, - "name": "Bayreuth", - "net": "10.50.168.0/22", - "position": {"type": "Point", "coordinates": [11.580566, 49.94814]} -}, -{ - "keyxchange_id": 23, - "name": "Fichtelberg", - "net": "10.50.172.0/22", - "position": {"type": "Point", "coordinates": [11.852292, 49.998920]} -}, -{ - "keyxchange_id": 24, - "name": "Rehau", - "net": "10.50.176.0/22", - "position": {"type": "Point", "coordinates": [12.035305, 50.247594]} -}, - { - "keyxchange_id": 25, - "name": "Coburg", - "net": "10.50.180.0/22", - "position": {"type": "Point", "coordinates": [10.964414, 50.259675]} -}, -{ - "keyxchange_id": 26, - "name": "Ebern", - "net": "10.50.184.0/22", - "position": {"type": "Point", "coordinates": [10.798395, 50.095572]} -}, -{ - "keyxchange_id": 27, - "name": "Arnstein", - "net": "10.50.188.0/22", - "position": {"type": "Point", "coordinates": [9.970957, 49.978117]} -}, -{ - "keyxchange_id": 28, - "name": "Erlenbach", - "net": "10.50.192.0/22", - "position": {"type": "Point", "coordinates": [9.157491, 49.803930]} -}] +mysql.execute(""" + CREATE TABLE hoods ( + `id` int(11) NOT NULL, + `name` varchar(50) COLLATE utf8_unicode_ci NOT NULL, + `net` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `lat` double DEFAULT NULL, + `lng` double DEFAULT NULL, + `cos_lat` double DEFAULT NULL, + `sin_lat` double DEFAULT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") -for hood in hoods: - db.hoods.insert_one(hood) +mysql.execute(""" + ALTER TABLE hoods + ADD PRIMARY KEY (`id`), + ADD KEY `name` (`name`), + ADD KEY `lat` (`lat`), + ADD KEY `lng` (`lng`), + ADD KEY `cos_lat` (`cos_lat`), + ADD KEY `sin_lat` (`sin_lat`) +""") + +mysql.close() diff --git a/ffmap/db/init_db.py b/ffmap/db/init_db.py index f9a92de..0d75f8b 100755 --- a/ffmap/db/init_db.py +++ b/ffmap/db/init_db.py @@ -3,3 +3,5 @@ import routers import hoods import stats +import users +import hooddata diff --git a/ffmap/db/routers.py b/ffmap/db/routers.py old mode 100644 new mode 100755 index b45da29..d0154c5 --- a/ffmap/db/routers.py +++ b/ffmap/db/routers.py @@ -1,14 +1,178 @@ #!/usr/bin/python3 -from pymongo import MongoClient -client = MongoClient() +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) -db = client.freifunk +from ffmap.mysqltools import FreifunkMySQL -# create db indexes -db.routers.create_index("hostname") -db.routers.create_index("status") -db.routers.create_index("created") -db.routers.create_index("last_contact") -db.routers.create_index("netifs.mac") -db.routers.create_index([("position", "2dsphere")]) +mysql = FreifunkMySQL() + +mysql.execute(""" + CREATE TABLE router ( + `id` int(11) NOT NULL, + `status` varchar(20) COLLATE utf8_unicode_ci NOT NULL, + `hostname` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `created` datetime NOT NULL, + `last_contact` datetime NOT NULL, + `sys_time` datetime NOT NULL, + `sys_uptime` int(11) NOT NULL, + `sys_memfree` int(11) NOT NULL, + `sys_membuff` int(11) NOT NULL, + `sys_memcache` int(11) NOT NULL, + `sys_loadavg` double NOT NULL, + `sys_procrun` smallint(6) NOT NULL, + `sys_proctot` smallint(6) NOT NULL, + `clients` smallint(6) NOT NULL, + `wan_uplink` tinyint(1) NOT NULL, + `cpu` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `chipset` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `hardware` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `os` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `batman` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `kernel` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `nodewatcher` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `firmware` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `firmware_rev` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `description` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `position_comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `community` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `hood` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL, + `status_text` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `contact` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `lng` double DEFAULT NULL, + `lat` double DEFAULT NULL, + `neighbors` smallint(6) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_events ( + `router` int(11) NOT NULL, + `time` datetime NOT NULL, + `type` varchar(100) COLLATE utf8_unicode_ci NOT NULL, + `comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_ipv6 ( + `router` int(11) NOT NULL, + `netif` varchar(100) COLLATE utf8_unicode_ci NOT NULL, + `ipv6` varchar(60) COLLATE utf8_unicode_ci NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_neighbor ( + `router` int(11) NOT NULL, + `mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `quality` smallint(6) NOT NULL, + `net_if` varchar(20) COLLATE utf8_unicode_ci NOT NULL, + `type` varchar(10) COLLATE utf8_unicode_ci DEFAULT 'l2' + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_netif ( + `router` int(11) NOT NULL, + `netif` varchar(100) COLLATE utf8_unicode_ci NOT NULL, + `mtu` smallint(6) NOT NULL, + `rx_bytes` bigint(20) NOT NULL, + `tx_bytes` bigint(20) NOT NULL, + `rx` bigint(20) NOT NULL, + `tx` bigint(20) NOT NULL, + `fe80_addr` varchar(60) COLLATE utf8_unicode_ci NOT NULL, + `ipv4_addr` varchar(20) COLLATE utf8_unicode_ci NOT NULL, + `mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_stats ( + `router` int(11) NOT NULL, + `time` datetime NOT NULL, + `sys_proctot` smallint(6) NOT NULL, + `sys_procrun` smallint(6) NOT NULL, + `sys_memcache` int(11) NOT NULL, + `sys_membuff` int(11) NOT NULL, + `sys_memfree` int(11) NOT NULL, + `loadavg` double NOT NULL, + `clients` smallint(6) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_stats_neighbor ( + `router` int(11) NOT NULL, + `mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `time` datetime NOT NULL, + `quality` smallint(6) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + CREATE TABLE router_stats_netif ( + `router` int(11) NOT NULL, + `netif` varchar(20) COLLATE utf8_unicode_ci NOT NULL, + `rx` bigint(20) NOT NULL, + `tx` bigint(20) NOT NULL, + `time` datetime NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + ALTER TABLE router + ADD PRIMARY KEY (`id`), + ADD KEY `created` (`created`), + ADD KEY `hostname` (`hostname`), + ADD KEY `status` (`status`), + ADD KEY `last_contact` (`last_contact`), + ADD KEY `lat` (`lat`), + ADD KEY `lng` (`lng`), + ADD KEY `contact` (`contact`), + ADD KEY `hood` (`hood`) +""") + +mysql.execute(""" + ALTER TABLE router_events + ADD PRIMARY KEY (`router`,`time`,`type`) +""") + +mysql.execute(""" + ALTER TABLE router_ipv6 + ADD PRIMARY KEY (`router`,`netif`,`ipv6`) +""") + +mysql.execute(""" + ALTER TABLE router_neighbor + ADD PRIMARY KEY (`router`,`mac`,`net_if`) +""") + +mysql.execute(""" + ALTER TABLE router_netif + ADD PRIMARY KEY (`router`,`netif`), + ADD KEY `mac` (`mac`) +""") + +mysql.execute(""" + ALTER TABLE router_stats + ADD PRIMARY KEY (`router`,`time`) +""") + +mysql.execute(""" + ALTER TABLE router_stats_neighbor + ADD PRIMARY KEY (`router`,`mac`,`time`) +""") + +mysql.execute(""" + ALTER TABLE router_stats_netif + ADD PRIMARY KEY (`router`,`netif`,`time`) +""") + +mysql.execute(""" + ALTER TABLE router + MODIFY `id` int(11) NOT NULL AUTO_INCREMENT +""") + +mysql.close() diff --git a/ffmap/db/stats.py b/ffmap/db/stats.py old mode 100644 new mode 100755 index 8376dfc..84b796f --- a/ffmap/db/stats.py +++ b/ffmap/db/stats.py @@ -1,9 +1,26 @@ #!/usr/bin/python3 -from pymongo import MongoClient -client = MongoClient() +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) -db = client.freifunk +from ffmap.mysqltools import FreifunkMySQL -# create capped collection -db.create_collection("stats", capped=True, size=10*1024*1024, max=4320) +mysql = FreifunkMySQL() + +mysql.execute(""" + CREATE TABLE `stats_global` ( + `time` datetime NOT NULL, + `clients` int(11) NOT NULL, + `online` int(11) NOT NULL, + `offline` int(11) NOT NULL, + `unknown` int(11) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + ALTER TABLE `stats_global` + ADD PRIMARY KEY (`time`) +""") + +mysql.close() diff --git a/ffmap/db/users.py b/ffmap/db/users.py old mode 100644 new mode 100755 index 757944c..0958dd6 --- a/ffmap/db/users.py +++ b/ffmap/db/users.py @@ -1,10 +1,36 @@ #!/usr/bin/python3 -from pymongo import MongoClient -client = MongoClient() +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) -db = client.freifunk +from ffmap.mysqltools import FreifunkMySQL -# create db indexes -db.users.create_index("email") -db.users.create_index("nickname") +mysql = FreifunkMySQL() + +mysql.execute(""" + CREATE TABLE `users` ( + `id` int(11) NOT NULL, + `nickname` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `password` varchar(250) COLLATE utf8_unicode_ci DEFAULT NULL, + `token` varchar(250) COLLATE utf8_unicode_ci DEFAULT NULL, + `email` varchar(200) COLLATE utf8_unicode_ci NOT NULL, + `created` datetime NOT NULL, + `admin` tinyint(4) NOT NULL DEFAULT '0' + ) + ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +""") + +mysql.execute(""" + ALTER TABLE `users` + ADD PRIMARY KEY (`id`), + ADD KEY `nickname` (`nickname`), + ADD KEY `email` (`email`) +""") + +mysql.execute(""" + ALTER TABLE `users` + MODIFY `id` int(11) NOT NULL AUTO_INCREMENT +""") + +mysql.close() diff --git a/ffmap/dbtools.py b/ffmap/dbtools.py deleted file mode 100644 index 06a2e67..0000000 --- a/ffmap/dbtools.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -from pymongo import MongoClient - -class FreifunkDB(object): - client = None - db = None - - @classmethod - def handle(cls): - if not cls.client: - cls.client = MongoClient(tz_aware=True, connect=False) - if not cls.db: - cls.db = cls.client.freifunk - return cls.db diff --git a/ffmap/maptools.py b/ffmap/maptools.py index bfd553c..09a9d50 100644 --- a/ffmap/maptools.py +++ b/ffmap/maptools.py @@ -4,7 +4,7 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL import math import numpy as np @@ -12,8 +12,6 @@ from scipy.spatial import Voronoi import urllib.request, json -db = FreifunkDB().handle() - CONFIG = { "csv_dir": "/var/lib/ffmap/csv" } @@ -73,83 +71,88 @@ def draw_voronoi_lines(csv, hoods): csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2)) -def update_mapnik_csv(): +def update_mapnik_csv(mysql): with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv: csv.write("lng,lat,status\n") - for router in db.routers.find({"position.coordinates": {"$exists": True}}, {"status": 1, "position": 1}): + routers = mysql.fetchall(""" + SELECT status, lat, lng FROM router + WHERE lat IS NOT NULL AND lng IS NOT NULL + """) + for router in routers: csv.write("%f,%f,%s\n" % ( - router["position"]["coordinates"][0], - router["position"]["coordinates"][1], + router["lng"], + router["lat"], router["status"] )) + dblinks = mysql.fetchall(""" + SELECT r1.lat AS rlat, r1.lng AS rlng, r2.lat AS nlat, r2.lng AS nlng, n.type AS type, quality + FROM router AS r1 + INNER JOIN router_neighbor AS n ON r1.id = n.router + INNER JOIN ( + SELECT router, mac FROM router_netif GROUP BY mac, router + ) AS net ON n.mac = net.mac + INNER JOIN router AS r2 ON net.router = r2.id + WHERE r1.lat IS NOT NULL AND r1.lng IS NOT NULL AND r2.lat IS NOT NULL AND r2.lng IS NOT NULL + AND r1.status = 'online' + """) + links = [] + linksl3 = [] + for row in dblinks: + if row.get("type")=="l3": + linksl3.append(( + row["rlng"], + row["rlat"], + row["nlng"], + row["nlat"] + )) + else: + links.append(( + row["rlng"], + row["rlat"], + row["nlng"], + row["nlat"], + row["quality"] + )) with open(os.path.join(CONFIG["csv_dir"], "links.csv"), "w") as csv: csv.write("WKT,quality\n") - links = [] - for router in db.routers.find( - { - "position.coordinates": {"$exists": True}, - "neighbours": {"$exists": True}, - "status": "online" - }, - {"position": 1, "neighbours": 1} - ): - for neighbour in router["neighbours"]: - if "position" in neighbour and not neighbour.get("type"): - links.append(( - router["position"]["coordinates"][0], - router["position"]["coordinates"][1], - neighbour["position"]["coordinates"][0], - neighbour["position"]["coordinates"][1], - neighbour["quality"] - )) for link in sorted(links, key=lambda l: l[4]): csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link) with open(os.path.join(CONFIG["csv_dir"], "l3_links.csv"), "w") as csv: csv.write("WKT\n") - for router in db.routers.find( - { - "position.coordinates": {"$exists": True}, - "neighbours": {"$exists": True}, - "status": "online" - }, - {"position": 1, "neighbours": 1} - ): - for neighbour in router["neighbours"]: - if "position" in neighbour and neighbour.get("type") and neighbour["type"] == "l3": - csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % ( - router["position"]["coordinates"][0], - router["position"]["coordinates"][1], - neighbour["position"]["coordinates"][0], - neighbour["position"]["coordinates"][1] - )) + for link in linksl3: + csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % link) + dbhoods = mysql.fetchall(""" + SELECT name, lat, lng FROM hoods + WHERE lat IS NOT NULL AND lng IS NOT NULL + """) with open(os.path.join(CONFIG["csv_dir"], "hood-points.csv"), "w", encoding="UTF-8") as csv: csv.write("lng,lat,name\n") - for hood in db.hoods.find({"position": {"$exists": True}}): + for hood in dbhoods: csv.write("%f,%f,\"%s\"\n" % ( - hood["position"]["coordinates"][0], - hood["position"]["coordinates"][1], + hood["lng"], + hood["lat"], hood["name"] )) with open(os.path.join(CONFIG["csv_dir"], "hoods.csv"), "w") as csv: csv.write("WKT\n") hoods = [] - for hood in db.hoods.find({"position": {"$exists": True}}): + for hood in dbhoods: # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat - x, y = merc_sphere(hood["position"]["coordinates"][0], hood["position"]["coordinates"][1]) + x, y = merc_sphere(hood["lng"], hood["lat"]) hoods.append([x, y]) draw_voronoi_lines(csv, hoods) + with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: + dbhoodsv2 = json.loads(url.read().decode()) + with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv: csv.write("lng,lat,name\n") - with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: - data = json.loads(url.read().decode()) - - for hood in data: + for hood in dbhoodsv2: if not ( 'lon' in hood and 'lat' in hood ): continue csv.write("%f,%f,\"%s\"\n" % ( @@ -161,10 +164,8 @@ def update_mapnik_csv(): with open(os.path.join(CONFIG["csv_dir"], "hoodsv2.csv"), "w") as csv: csv.write("WKT\n") hoods = [] - with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: - data = json.loads(url.read().decode()) - for hood in data: + for hood in dbhoodsv2: if not ( 'lon' in hood and 'lat' in hood ): continue # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat diff --git a/ffmap/mysqlconfig.example.py b/ffmap/mysqlconfig.example.py new file mode 100644 index 0000000..112528d --- /dev/null +++ b/ffmap/mysqlconfig.example.py @@ -0,0 +1,8 @@ +#!/usr/bin/python3 + +mysq = { + "host":"localhost", + "user":"root", + "passwd":"password", + "db":"dbname" + } diff --git a/ffmap/mysqltools.py b/ffmap/mysqltools.py new file mode 100644 index 0000000..525fc0c --- /dev/null +++ b/ffmap/mysqltools.py @@ -0,0 +1,98 @@ +#!/usr/bin/python3 + +import MySQLdb +from ffmap.mysqlconfig import mysq +from ffmap.misc import * +#import pytz + +class FreifunkMySQL: + + db = None + cur = None + + def __init__(self): + #global mysq + self.db = MySQLdb.connect(host=mysq["host"], user=mysq["user"], passwd=mysq["passwd"], db=mysq["db"]) + self.cur = self.db.cursor(MySQLdb.cursors.DictCursor) + + def close(self): + self.db.close() + + def cursor(self): + return self.cur + + def commit(self): + self.db.commit() + + def fetchall(self,str,tup=(),key=None): + self.cur.execute(str,tup) + result = self.cur.fetchall() + if len(result) > 0: + if key: + rnew = [] + for r in result: + rnew.append(r[key]) + return rnew + else: + return result + else: + return () + + def fetchdict(self,str,tup,key,value=None): + self.cur.execute(str,tup) + dict = {} + for d in self.cur.fetchall(): + if value: + dict[d[key]] = d[value] + else: + dict[d[key]] = d + return dict + + def findone(self,str,tup,sel=None): + self.cur.execute(str,tup) + result = self.cur.fetchall() + if len(result) > 0: + if sel: + return result[0][sel] + else: + return result[0] + else: + return False + + def execute(self,a,b=None): + if b: + return self.cur.execute(a,b) + else: + return self.cur.execute(a) + + def utcnow(self): + return utcnow().strftime('%Y-%m-%d %H:%M:%S') + + def formatdt(self,dt): + return dt.strftime('%Y-%m-%d %H:%M:%S') + + def utcaware(self,data,keys=None): + if keys: + for k in keys: + #self.utcaware(data[k]) + #data[k] = pytz.utc.localize(data[k]) + data[k] = data[k].replace(tzinfo=datetime.timezone.utc) + else: + #data = pytz.utc.localize(data) + data = data.replace(tzinfo=datetime.timezone.utc) + return data + + def utcawaretuple(self,data,index=None): + if index: + for r in data: + #self.utcaware(r[index]) + #r[index] = pytz.utc.localize(r[index]) + r[index] = r[index].replace(tzinfo=datetime.timezone.utc) + else: + for r in data: + #self.utcaware(r) + #r = pytz.utc.localize(r) + r = r.replace(tzinfo=datetime.timezone.utc) + return data + + diff --git a/ffmap/routertools.py b/ffmap/routertools.py index 4092249..8a4acc9 100644 --- a/ffmap/routertools.py +++ b/ffmap/routertools.py @@ -4,7 +4,7 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL from ffmap.misc import * import lxml.etree @@ -13,8 +13,6 @@ import requests from bson import SON from contextlib import suppress -db = FreifunkDB().handle() - CONFIG = { "vpn_netif": "fffVPN", "vpn_netif_l2tp": "l2tp", @@ -26,9 +24,23 @@ CONFIG = { router_rate_limit_list = {} -def import_nodewatcher_xml(mac, xml): +def delete_router(mysql,dbid): + cur = mysql.cursor() + cur.execute("DELETE FROM router WHERE id = %s",(dbid,)) + cur.execute("DELETE FROM router_netif WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_ipv6 WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_neighbor WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_events WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_stats WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_stats_neighbor WHERE router = %s",(dbid,)) + cur.execute("DELETE FROM router_stats_netif WHERE router = %s",(dbid,)) + mysql.commit() + +def import_nodewatcher_xml(mysql, mac, xml): global router_rate_limit_list + cur = mysql.cursor() + t = utcnow() if mac in router_rate_limit_list: if (t - router_rate_limit_list[mac]) < datetime.timedelta(minutes=5): @@ -36,188 +48,335 @@ def import_nodewatcher_xml(mac, xml): router_rate_limit_list[mac] = t router_id = None + olddata = [] + uptime = 0 events = [] + status_comment = "" + try: - router = db.routers.find_one({"netifs.mac": mac.lower()}, {"stats": 0, "events": 0}) - if router: - router_id = router["_id"] - + cur.execute("SELECT router FROM router_netif WHERE mac = %s LIMIT 1",(mac.lower(),)) + result = cur.fetchall() + if len(result)>0: + router_id = result[0]["router"] + cur.execute("SELECT sys_uptime AS uptime, firmware, hostname, hood, status, lat, lng FROM router WHERE id = %s LIMIT 1",(router_id,)) + result = cur.fetchall() + if len(result)>0: + olddata = result[0] + uptime = olddata["uptime"] router_update = parse_nodewatcher_xml(xml) # keep hood up to date - if not "hood" in router_update: + if not router_update["hood"]: # router didn't send his hood in XML - if "position" in router_update: - # router has new position info from netmon - router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router_update["position"]}}})["name"] - elif router and "position" in router: + lat = router_update.get("lat") + lng = router_update.get("lng") + if olddata and not lat and not lng: # hoods might change as well - router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router["position"]}}})["name"] - - if router: + lat = olddata.get("lat") + lng = olddata.get("lng") + if lat and lng: + router_update["hood"] = mysql.findone(""" + SELECT name, + ( acos( cos( radians(%s) ) + * cos_lat + * cos( radians( lng ) - radians(%s) ) + + sin( radians(%s) ) * sin_lat + ) + ) AS distance + FROM + hoods + WHERE lat IS NOT NULL AND lng IS NOT NULL + ORDER BY + distance ASC + LIMIT 1 + """,(lat,lng,lat,),"name") + else: + router_update["hood"] = None + + if router_id: # statistics - calculate_network_io(router, router_update) - db.routers.update_one({"_id": router_id}, { - "$set": router_update, - "$push": {"stats": SON([ - ("$each", new_router_stats(router, router_update)), - ("$slice", int(CONFIG["router_stat_days"] * -1 * 24 * (3600 / 300))) - ]) - }}) + calculate_network_io(cur, router_id, uptime, router_update) + ru = router_update + rus = router_update["system"] + ruh = router_update["hardware"] + ruso = router_update["software"] + cur.execute(""" + UPDATE router + SET status = %s, hostname = %s, last_contact = %s, sys_time = %s, sys_uptime = %s, sys_memfree = %s, sys_membuff = %s, sys_memcache = %s, + sys_loadavg = %s, sys_procrun = %s, sys_proctot = %s, clients = %s, wan_uplink = %s, cpu = %s, chipset = %s, hardware = %s, os = %s, + batman = %s, kernel = %s, nodewatcher = %s, firmware = %s, firmware_rev = %s, description = %s, position_comment = %s, community = %s, hood = %s, + status_text = %s, contact = %s, lng = %s, lat = %s, neighbors = %s + WHERE id = %s + """,( + ru["status"],ru["hostname"],ru["last_contact"],rus["time"],rus["uptime"],rus["memory"]["free"],rus["memory"]["buffering"],rus["memory"]["caching"], + rus["loadavg"],rus["processes"]["runnable"],rus["processes"]["total"],rus["clients"],rus["has_wan_uplink"],ruh["cpu"],ruh["chipset"],ruh["name"],ruso["os"], + ruso["batman_adv"],ruso["kernel"],ruso["nodewatcher"],ruso["firmware"],ruso["firmware_rev"],ru["description"],ru["position_comment"],ru["community"],ru["hood"], + ru["system"]["status_text"],ru["system"]["contact"],ru["lng"],ru["lat"],rus["visible_neighbours"],router_id,)) + + cur.execute("DELETE FROM router_netif WHERE router = %s",(router_id,)) + cur.execute("DELETE FROM router_ipv6 WHERE router = %s",(router_id,)) + cur.execute("DELETE FROM router_neighbor WHERE router = %s",(router_id,)) + + uptime = 0 + new_router_stats(mysql, router_id, uptime, router_update) else: # insert new router - router_update["created"] = utcnow() - router_update["stats"] = [] - events = [] # don't fire sub-events of created events - router_update["events"] = [{ - "time": utcnow(), - "type": "created", - }] - router_id = db.routers.insert_one(router_update).inserted_id + created = mysql.utcnow() + #events = [] # don't fire sub-events of created events + #router_update["events"] = [{ + # "time": utcnow(), + # "type": "created", + #}] + ru = router_update + rus = router_update["system"] + ruh = router_update["hardware"] + ruso = router_update["software"] + + cur.execute(""" + INSERT INTO router (status, hostname, created, last_contact, sys_time, sys_uptime, sys_memfree, sys_membuff, sys_memcache, + sys_loadavg, sys_procrun, sys_proctot, clients, wan_uplink, cpu, chipset, hardware, os, + batman, kernel, nodewatcher, firmware, firmware_rev, description, position_comment, community, hood, + status_text, contact, lng, lat, neighbors) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """,( + ru["status"],ru["hostname"],created,ru["last_contact"],rus["time"],rus["uptime"],rus["memory"]["free"],rus["memory"]["buffering"],rus["memory"]["caching"], + rus["loadavg"],rus["processes"]["runnable"],rus["processes"]["total"],rus["clients"],rus["has_wan_uplink"],ruh["cpu"],ruh["chipset"],ruh["name"],ruso["os"], + ruso["batman_adv"],ruso["kernel"],ruso["nodewatcher"],ruso["firmware"],ruso["firmware_rev"],ru["description"],ru["position_comment"],ru["community"],ru["hood"], + ru["system"]["status_text"],ru["system"]["contact"],ru["lng"],ru["lat"],rus["visible_neighbours"],)) + router_id = cur.lastrowid + + events_append(mysql,router_id,"created","") + + for n in router_update["netifs"]: + cur.execute(""" + INSERT INTO router_netif (router, netif, mtu, rx_bytes, tx_bytes, rx, tx, fe80_addr, ipv4_addr, mac) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """,( + router_id,n["name"],n["mtu"],n["traffic"]["rx_bytes"],n["traffic"]["tx_bytes"],n["traffic"]["rx"],n["traffic"]["tx"],n["ipv6_fe80_addr"],n["ipv4_addr"],n["mac"],)) + for a in n["ipv6_addrs"]: + cur.execute("INSERT INTO router_ipv6 (router, netif, ipv6) VALUES (%s, %s, %s)",(router_id,n["name"],a,)) + + for n in router_update["neighbours"]: + cur.execute("INSERT INTO router_neighbor (router, mac, quality, net_if, type) VALUES (%s, %s, %s, %s, %s)",(router_id,n["mac"],n["quality"],n["net_if"],n["type"],)) + status = router_update["status"] except ValueError as e: import traceback print("Warning: Unable to parse xml from %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) - if router: - db.routers.update_one({"_id": router_id}, {"$set": { - "status": "unknown", - "last_contact": utcnow() - }}) + if router_id: + set_status(mysql,router_id,"unknown") status = "unknown" status_comment = "Invalid XML" except OverflowError as e: import traceback print("Warning: Overflow Error when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) - if router: - db.routers.update_one({"_id": router_id}, {"$set": { - "status": "unknown", - "last_contact": utcnow() - }}) + if router_id: + set_status(mysql,router_id,"unknown") status = "unknown" status_comment = "Integer Overflow" except Exception as e: import traceback print("Warning: Exception occurred when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) - if router: - db.routers.update_one({"_id": router_id}, {"$set": { - "status": "unknown", - "last_contact": utcnow() - }}) + if router_id: + set_status(mysql,router_id,"unknown") status = "unknown" status_comment = "Exception occurred" - if router_id: + if olddata: # fire events with suppress(KeyError, TypeError, UnboundLocalError): - if router["system"]["uptime"] > router_update["system"]["uptime"]: - events.append({ - "time": utcnow(), - "type": "reboot", - }) + if olddata["uptime"] > router_update["system"]["uptime"]: + events_append(mysql,router_id,"reboot","") with suppress(KeyError, TypeError, UnboundLocalError): - if router["software"]["firmware"] != router_update["software"]["firmware"]: - events.append({ - "time": utcnow(), - "type": "update", - "comment": "%s -> %s" % (router["software"]["firmware"], router_update["software"]["firmware"]), - }) + if olddata["firmware"] != router_update["software"]["firmware"]: + events_append(mysql,router_id,"update", + "%s -> %s" % (olddata["firmware"], router_update["software"]["firmware"])) + #events.append({ + # "time": utcnow(), + # "type": "update", + # "comment": "%s -> %s" % (olddata["firmware"], router_update["software"]["firmware"]), + #}) with suppress(KeyError, TypeError, UnboundLocalError): - if router["hostname"] != router_update["hostname"]: - events.append({ - "time": utcnow(), - "type": "hostname", - "comment": "%s -> %s" % (router["hostname"], router_update["hostname"]), - }) + if olddata["hostname"] != router_update["hostname"]: + events_append(mysql,router_id,"hostname", + "%s -> %s" % (olddata["hostname"], router_update["hostname"])) + #events.append({ + # "time": utcnow(), + # "type": "hostname", + # "comment": "%s -> %s" % (olddata["hostname"], router_update["hostname"]), + #}) with suppress(KeyError, TypeError, UnboundLocalError): - if router["hood"] != router_update["hood"]: - events.append({ - "time": utcnow(), - "type": "hood", - "comment": "%s -> %s" % (router["hood"], router_update["hood"]), - }) + if olddata["hood"] != router_update["hood"]: + events_append(mysql,router_id,"hood", + "%s -> %s" % (olddata["hood"], router_update["hood"])) + #events.append({ + # "time": utcnow(), + # "type": "hood", + # "comment": "%s -> %s" % (olddata["hood"], router_update["hood"]), + #}) with suppress(KeyError, TypeError): - if router["status"] != status: - event = { - "time": utcnow(), - "type": status, - } - with suppress(NameError): - event["comment"] = status_comment - events.append(event) + if olddata["status"] != status: + events_append(mysql,router_id,status,status_comment) + #event = { + # "time": utcnow(), + # "type": status, + #} + #with suppress(NameError): + # event["comment"] = status_comment + #events.append(event) - if len(events) > 0: - db.routers.update_one({"_id": router_id}, {"$push": {"events": SON([ - ("$each", events), - ("$slice", -10), - ])}}) +def detect_offline_routers(mysql): + cur = mysql.cursor() + + threshold=mysql.formatdt(utcnow() - datetime.timedelta(minutes=CONFIG["offline_threshold_minutes"])) + now=mysql.utcnow() + + cur.execute(""" + SELECT id + FROM router + WHERE last_contact < %s AND status <> 'offline' + """,(threshold,)) + result = cur.fetchall() + for r in result: + cur.execute(""" + INSERT INTO router_events ( router, time, type, comment ) + VALUES (%s, %s, 'offline', '') + """,(r["id"],now,)) + + cur.execute(""" + UPDATE router + SET status = 'offline', clients = 0 + WHERE last_contact < %s AND status <> 'offline' + """,(threshold,)) + mysql.commit() -def detect_offline_routers(): - db.routers.update_many({ - "last_contact": {"$lt": utcnow() - datetime.timedelta(minutes=CONFIG["offline_threshold_minutes"])}, - "status": {"$ne": "offline"} - }, { - "$set": {"status": "offline", "system.clients": 0}, - "$push": {"events": { - "time": utcnow(), - "type": "offline" - } - }}) +def delete_orphaned_routers(mysql): + cur = mysql.cursor() + + threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["orphan_threshold_days"])) + + cur.execute(""" + DELETE r, e, i, nb, net FROM router AS r + INNER JOIN router_events AS e ON r.id = e.router + INNER JOIN router_ipv6 AS i ON r.id = i.router + INNER JOIN router_neighbor AS nb ON r.id = nb.router + INNER JOIN router_netif AS net ON r.id = net.router + WHERE r.last_contact < %s AND r.status <> 'offline' + """,(threshold,)) + + mysql.commit() -def delete_orphaned_routers(): - db.routers.delete_many({ - "last_contact": {"$lt": utcnow() - datetime.timedelta(days=CONFIG["orphan_threshold_days"])}, - "status": "offline" - }) +def delete_old_stats(mysql): + threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["router_stat_days"])) + + mysql.execute(""" + DELETE FROM router_stats + WHERE time < %s + """,(threshold,)) -def new_router_stats(router, router_update): - if router["system"]["uptime"] < router_update["system"]["uptime"]: - netifs = {} - neighbours = {} + mysql.execute(""" + DELETE FROM router_stats_neighbor + WHERE time < %s + """,(threshold,)) + + mysql.execute(""" + DELETE FROM router_stats_netif + WHERE time < %s + """,(threshold,)) + + mysql.commit() + +def events_append(mysql,router_id,event,comment): + mysql.execute(""" + INSERT INTO router_events (router, time, type, comment) + VALUES (%s, %s, %s, %s) + """,( + router_id, + mysql.utcnow(), + event, + comment,)) + +def set_status(mysql,router_id,status): + mysql.execute(""" + UPDATE router + SET status = %s, last_contact = %s + WHERE id = %s + """,( + status, + mysql.utcnow(), + router_id,)) + +def new_router_stats(mysql, router_id, uptime, router_update): + if uptime < router_update["system"]["uptime"]: + time = mysql.utcnow() + + mysql.execute(""" + INSERT INTO router_stats (router, time, sys_memfree, sys_membuff, sys_memcache, loadavg, sys_procrun, sys_proctot, clients) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """,( + router_id, + time, + router_update["system"]["memory"]['free'], + router_update["system"]["memory"]['buffering'], + router_update["system"]["memory"]['caching'], + router_update["system"]["loadavg"], + router_update["system"]["processes"]['runnable'], + router_update["system"]["processes"]['total'], + router_update["system"]["clients"],)) + for netif in router_update["netifs"]: # sanitize name name = netif["name"].replace(".", "").replace("$", "") with suppress(KeyError): - netifs[name] = {"rx": netif["traffic"]["rx"], "tx": netif["traffic"]["tx"]} + mysql.execute(""" + INSERT INTO router_stats_netif (router, netif, time, rx, tx) + VALUES (%s, %s, %s, %s, %s) + """,( + router_id, + name, + time, + netif["traffic"]["rx"], + netif["traffic"]["tx"],)) for neighbour in router_update["neighbours"]: with suppress(KeyError): - neighbours[neighbour["mac"]] = neighbour["quality"] - return [{ - "time": utcnow(), - "netifs": netifs, - "neighbours": neighbours, - "memory": router_update["system"]["memory"], - "loadavg": router_update["system"]["loadavg"], - "processes": router_update["system"]["processes"], - "clients": router_update["system"]["clients"], - }] - else: - # don't push old data - return [] + mysql.execute(""" + INSERT INTO router_stats_neighbor (router, mac, time, quality) + VALUES (%s, %s, %s, %s) + """,( + router_id, + neighbour["mac"], + time, + neighbour["quality"],)) -def calculate_network_io(router, router_update): +def calculate_network_io(cur, router_id, uptime, router_update): """ router: old router dict router_update: new router dict (which will be updated with new data) """ + cur.execute("SELECT netif, rx_bytes, tx_bytes, rx, tx FROM router_netif WHERE router = %s",(router_id,)); + results = cur.fetchall() + with suppress(KeyError, StopIteration): - if router["system"]["uptime"] < router_update["system"]["uptime"]: - timediff = router_update["system"]["uptime"] - router["system"]["uptime"] - for netif in router["netifs"]: - netif_update = next(filter(lambda n: n["name"] == netif["name"], router_update["netifs"])) - rx_diff = netif_update["traffic"]["rx_bytes"] - netif["traffic"]["rx_bytes"] - tx_diff = netif_update["traffic"]["tx_bytes"] - netif["traffic"]["tx_bytes"] + if uptime < router_update["system"]["uptime"]: + timediff = router_update["system"]["uptime"] - uptime + for row in results: + netif_update = next(filter(lambda n: n["name"] == row["netif"], router_update["netifs"])) + rx_diff = netif_update["traffic"]["rx_bytes"] - int(row["rx_bytes"]) + tx_diff = netif_update["traffic"]["tx_bytes"] - int(row["tx_bytes"]) if rx_diff >= 0 and tx_diff >= 0: netif_update["traffic"]["rx"] = int(rx_diff / timediff) netif_update["traffic"]["tx"] = int(tx_diff / timediff) else: - for netif in router["netifs"]: - netif_update = next(filter(lambda n: n["name"] == netif["name"], router_update["netifs"])) - netif_update["traffic"]["rx"] = netif["traffic"]["rx"] - netif_update["traffic"]["tx"] = netif["traffic"]["tx"] + for row in results: + netif_update = next(filter(lambda n: n["name"] == row["netif"], router_update["netifs"])) + netif_update["traffic"]["rx"] = int(row["rx"]) + netif_update["traffic"]["tx"] = int(row["tx"]) + + return uptime def parse_nodewatcher_xml(xml): try: @@ -227,11 +386,11 @@ def parse_nodewatcher_xml(xml): router_update = { "status": tree.xpath("/data/system_data/status/text()")[0], "hostname": tree.xpath("/data/system_data/hostname/text()")[0], - "last_contact": utcnow(), + "last_contact": utcnow().strftime('%Y-%m-%d %H:%M:%S'), "neighbours": [], "netifs": [], "system": { - "time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])), + "time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])).strftime('%Y-%m-%d %H:%M:%S'), "uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])), "memory": { "free": int(tree.xpath("/data/system_data/memory_free/text()")[0]), @@ -276,38 +435,41 @@ def parse_nodewatcher_xml(xml): router_update["hardware"]["name"] = "Legacy" router_update["hardware"]["name"] = tree.xpath("/data/system_data/model/text()")[0] - # data.system_data.chipset - with suppress(IndexError): - router_update["hardware"]["chipset"] = "Unknown" - router_update["hardware"]["chipset"] = tree.xpath("/data/system_data/chipset/text()")[0] - # data.system_data.description with suppress(IndexError): + router_update["description"] = "" router_update["description"] = tree.xpath("/data/system_data/description/text()")[0] # data.system_data.position_comment with suppress(IndexError): + router_update["position_comment"] = "" router_update["position_comment"] = tree.xpath("/data/system_data/position_comment/text()")[0] # data.system_data.firmware_community with suppress(IndexError): + router_update["community"] = "" router_update["community"] = tree.xpath("/data/system_data/firmware_community/text()")[0] # data.system_data.hood with suppress(IndexError): + router_update["hood"] = "" router_update["hood"] = tree.xpath("/data/system_data/hood/text()")[0].lower() # data.system_data.status_text with suppress(IndexError): + router_update["system"]["status_text"] = "" router_update["system"]["status_text"] = tree.xpath("/data/system_data/status_text/text()")[0] # data.system_data.contact with suppress(IndexError): + router_update["system"]["contact"] = "" + #router_update["user"] = "" router_update["system"]["contact"] = tree.xpath("/data/system_data/contact/text()")[0] - user = db.users.find_one({"email": router_update["system"]["contact"]}) - if user: - # post-netmon router gets its user assigned - router_update["user"] = {"nickname": user["nickname"], "_id": user["_id"]} + #user = db.users.find_one({"email": router_update["system"]["contact"]}) + #if user: + # # post-netmon router gets its user assigned + # #router_update["user"] = {"nickname": user["nickname"], "_id": user["_id"]} + # router_update["user"] = user["nickname"] # data.system_data.geo with suppress(AssertionError, IndexError): @@ -316,10 +478,8 @@ def parse_nodewatcher_xml(xml): assert lng != 0 assert lat != 0 - router_update["position"] = { - "type": "Point", - "coordinates": [lng, lat] - } + router_update["lng"] = lng + router_update["lat"] = lat #FIXME: tmp workaround to get similar hardware names router_update["hardware"]["name"] = router_update["hardware"]["name"].replace("nanostation-m", "Ubiquiti Nanostation M") @@ -341,15 +501,19 @@ def parse_nodewatcher_xml(xml): "traffic": { "rx_bytes": int(netif.xpath("traffic_rx/text()")[0]), "tx_bytes": int(netif.xpath("traffic_tx/text()")[0]), + "rx": 0, + "tx": 0, }, } with suppress(IndexError): + interface["ipv6_fe80_addr"] = "" interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0] + interface["ipv6_addrs"] = [] if len(netif.xpath("ipv6_addr/text()")) > 0: - interface["ipv6_addrs"] = [] for ipv6_addr in netif.xpath("ipv6_addr/text()"): interface["ipv6_addrs"].append(ipv6_addr.lower().split("/")[0]) with suppress(IndexError): + interface["ipv4_addr"] = "" interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0] with suppress(IndexError): @@ -379,8 +543,8 @@ def parse_nodewatcher_xml(xml): "mac": o_mac.lower(), "quality": int(o_link_quality), "net_if": o_out_if, + "type": "l2" } - set_hostname_and_pos_for_neighbour(neighbour) router_update["neighbours"].append(neighbour) l3_neighbours = get_l3_neighbours(tree) @@ -392,20 +556,6 @@ def parse_nodewatcher_xml(xml): except (AssertionError, lxml.etree.XMLSyntaxError, IndexError) as e: raise ValueError("%s: %s" % (e.__class__.__name__, str(e))) - -def set_hostname_and_pos_for_neighbour(neighbour): - with suppress(AssertionError, TypeError): - neighbour_router = db.routers.find_one( - {"netifs.mac": neighbour["mac"]}, {"hostname": 1, "position": 1}) - neighbour["_id"] = neighbour_router["_id"] - neighbour["hostname"] = neighbour_router["hostname"] - assert "position" in neighbour_router - assert "coordinates" in neighbour_router["position"] - assert neighbour_router["position"]["coordinates"][0] != 0 - assert neighbour_router["position"]["coordinates"][1] != 0 - neighbour["position"] = neighbour_router["position"] - - def get_l3_neighbours(tree): l3_neighbours = list() for neighbour in tree.xpath("/data/babel_neighbours/*"): @@ -417,7 +567,6 @@ def get_l3_neighbours(tree): "net_if": out_if, "type": "l3" } - set_hostname_and_pos_for_neighbour(neighbour) l3_neighbours.append(neighbour) return l3_neighbours diff --git a/ffmap/stattools.py b/ffmap/stattools.py index c5b2f3e..5fcfdc6 100644 --- a/ffmap/stattools.py +++ b/ffmap/stattools.py @@ -4,91 +4,102 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL from ffmap.misc import * -db = FreifunkDB().handle() +CONFIG = { + "global_stat_days": 30, +} -def total_clients(): - r = db.routers.aggregate([{"$group": { - "_id": None, - "clients": {"$sum": "$system.clients"} - }}]) - return next(r)["clients"] +def total_clients(mysql): + return mysql.findone(""" + SELECT SUM(clients) AS clients + FROM router + """,(),"clients") -def router_status(): - r = db.routers.aggregate([{"$group": { - "_id": "$status", - "count": {"$sum": 1} - }}]) +def router_status(mysql): + return mysql.fetchdict(""" + SELECT status, COUNT(id) AS count + FROM router + GROUP BY status + """,(),"status","count") + +def router_models(mysql): + return mysql.fetchdict(""" + SELECT hardware, COUNT(id) AS count + FROM router + GROUP BY hardware + """,(),"hardware","count") + +def router_firmwares(mysql): + return mysql.fetchdict(""" + SELECT firmware, COUNT(id) AS count + FROM router + GROUP BY firmware + """,(),"firmware","count") + +def hoods(mysql): + data = mysql.fetchall(""" + SELECT hood, status, COUNT(id) AS count + FROM router + GROUP BY hood, status + """) result = {} - for rs in r: - result[rs["_id"]] = rs["count"] + for rs in data: + if not rs["hood"]: + rs["hood"] = "default" + if not rs["hood"] in result: + result[rs["hood"]] = {} + result[rs["hood"]][rs["status"]] = rs["count"] return result -def router_models(): - r = db.routers.aggregate([{"$group": { - "_id": "$hardware.name", - "count": {"$sum": 1} - }}]) +def hoods_sum(mysql): + data = mysql.fetchall(""" + SELECT hood, COUNT(id) AS count, SUM(clients) AS clients + FROM router + GROUP BY hood + """) result = {} - for rs in r: - result[rs["_id"]] = rs["count"] + for rs in data: + if not rs["hood"]: + rs["hood"] = "default" + result[rs["hood"]] = {"routers": rs["count"], "clients": rs["clients"]} return result -def router_firmwares(): - r = db.routers.aggregate([{"$group": { - "_id": "$software.firmware", - "count": {"$sum": 1} - }}]) +def record_global_stats(mysql): + threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"])) + time = mysql.utcnow() + status = router_status(mysql) + + old = mysql.findone("SELECT time FROM stats_global WHERE time = %s LIMIT 1",(time,)) + + if old: + mysql.execute(""" + UPDATE stats_global + SET clients = %s, online = %s, offline = %s, unknown = %s + WHERE time = %s + """,(total_clients(mysql),status.get("online",0),status.get("offline",0),status.get("unknown",0),time,)) + else: + mysql.execute(""" + INSERT INTO stats_global (time, clients, online, offline, unknown) + VALUES (%s, %s, %s, %s, %s) + """,(time,total_clients(mysql),status.get("online",0),status.get("offline",0),status.get("unknown",0),)) + + mysql.execute(""" + DELETE FROM stats_global + WHERE time < %s + """,(threshold,)) + + mysql.commit() + +def router_user_sum(mysql): + data = mysql.fetchall(""" + SELECT contact, COUNT(id) AS count, SUM(clients) AS clients + FROM router + GROUP BY contact + """) result = {} - for rs in r: - result[rs["_id"]] = rs["count"] - return result - -def hoods(): - r = db.routers.aggregate([{"$group": { - "_id": {"hood": "$hood", "status": "$status"}, - "count": {"$sum": 1}, - }}]) - result = {} - for rs in r: - if not "hood" in rs["_id"]: - rs["_id"]["hood"] = "default" - if not rs["_id"]["hood"] in result: - result[rs["_id"]["hood"]] = {} - result[rs["_id"]["hood"]][rs["_id"]["status"]] = rs["count"] - return result - -def hoods_sum(): - r = db.routers.aggregate([{"$group": { - "_id": "$hood", - "count": {"$sum": 1}, - "clients": {"$sum": "$system.clients"} - }}]) - result = {} - for rs in r: - if not rs["_id"]: - rs["_id"] = "default" - result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]} - return result - -def record_global_stats(): - db.stats.insert_one({ - "time": utcnow(), - "router_status": router_status(), - "total_clients": total_clients() - }) - - -def router_user_sum(): - r = db.routers.aggregate([{"$group": { - "_id": "$user.nickname", - "count": {"$sum": 1}, - "clients": {"$sum": "$system.clients"} - }}]) - result = {} - for rs in r: - if rs["_id"]: - result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]} + for rs in data: + if rs["contact"]: + result[rs["contact"]] = {"routers": rs["count"], "clients": rs["clients"]} return result diff --git a/ffmap/usertools.py b/ffmap/usertools.py index 5058645..be7c9a7 100644 --- a/ffmap/usertools.py +++ b/ffmap/usertools.py @@ -4,13 +4,11 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL from ffmap.misc import * from werkzeug.security import generate_password_hash, check_password_hash -db = FreifunkDB().handle() - class AccountWithEmailExists(Exception): pass @@ -24,67 +22,107 @@ class InvalidToken(Exception): pass def register_user(nickname, email, password): - user_with_nick = db.users.find_one({"nickname": nickname}) - user_with_email = db.users.find_one({"email": email}) + mysql = FreifunkMySQL() + + user_with_nick = mysql.findone("SELECT id, email FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + user_with_email = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id") + pw = generate_password_hash(password) if user_with_email: + mysql.close() raise AccountWithEmailExists() - elif user_with_nick and "email" in user_with_nick: + elif user_with_nick and user_with_nick["email"]: + mysql.close() raise AccountWithNicknameExists() else: - user_update = { - "nickname": nickname, - "password": generate_password_hash(password), - "email": email, - "created": utcnow() - } + time = mysql.utcnow() if user_with_nick: - db.users.update_one({"_id": user_with_nick["_id"]}, {"$set": user_update}) - return user_with_nick["_id"] + mysql.execute(""" + UPDATE users + SET password = %s, email = %s, created = %s, token = NULL + WHERE id = %s + LIMIT 1 + """,(pw,email,time,user_with_nick["id"],)) + mysql.commit() + mysql.close() + return user_with_nick["id"] else: - return db.users.insert_one(user_update).inserted_id + mysql.execute(""" + INSERT INTO users (nickname, password, email, created, token) + VALUES (%s, %s, %s, %s, NULL) + """,(nickname,pw,email,time,)) + userid = mysql.cursor().lastrowid + mysql.commit() + mysql.close() + return userid def check_login_details(nickname, password): - user = db.users.find_one({"nickname": nickname}) + mysql = FreifunkMySQL() + + user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + mysql.close() if user and check_password_hash(user.get('password', ''), password): return user else: return False -def reset_user_password(email, token=None, password=None): - user = db.users.find_one({"email": email}) +def reset_user_password(mysql, email, token=None, password=None): + userid = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id") if not user: raise AccountNotExisting() elif password: if user.get("token") == token: - db.users.update_one({"_id": user["_id"]}, { - "$set": {"password": generate_password_hash(password)}, - "$unset": {"token": 1}, - }) + mysql.execute(""" + UPDATE users + SET password = %s, token = NULL + WHERE id = %s + LIMIT 1 + """,(generate_password_hash(password),userid,)) + mysql.commit() else: raise InvalidToken() elif token: - db.users.update_one({"_id": user["_id"]}, {"$set": {"token": token}}) + mysql.execute(""" + UPDATE users + SET token = %s + WHERE id = %s + LIMIT 1 + """,(token,userid,)) + mysql.commit() -def set_user_password(nickname, password): - user = db.users.find_one({"nickname": nickname}) - if not user: +def set_user_password(mysql, nickname, password): + userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id") + if not userid: raise AccountNotExisting() elif password: - db.users.update_one({"_id": user["_id"]}, { - "$set": {"password": generate_password_hash(password)}, - }) + mysql.execute(""" + UPDATE users + SET password = %s + WHERE id = %s + LIMIT 1 + """,(generate_password_hash(password),userid,)) + mysql.commit() -def set_user_email(nickname, email): - user = db.users.find_one({"nickname": nickname}) - user_with_email = db.users.find_one({"email": email}) - if user_with_email: +def set_user_email(mysql, nickname, email): + userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id") + useridemail = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id") + if useridemail: raise AccountWithEmailExists() - if not user: + if not userid: raise AccountNotExisting() elif email: - db.users.update_one({"_id": user["_id"]}, { - "$set": {"email": email}, - }) + mysql.execute(""" + UPDATE users + SET email = %s + WHERE id = %s + LIMIT 1 + """,(email,userid,)) + mysql.commit() -def set_user_admin(nickname, admin): - db.users.update({"nickname": nickname}, {"$set": {"admin": admin}}) +def set_user_admin(mysql, nickname, admin): + mysql.execute(""" + UPDATE users + SET admin = %s + WHERE nickname = %s + LIMIT 1 + """,(admin,nickname,)) + mysql.commit() diff --git a/ffmap/web/api.py b/ffmap/web/api.py index 0bd03be..eadff6f 100644 --- a/ffmap/web/api.py +++ b/ffmap/web/api.py @@ -2,11 +2,10 @@ from ffmap.routertools import * from ffmap.maptools import * -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL from ffmap.stattools import record_global_stats from flask import Blueprint, request, make_response, redirect, url_for, jsonify, Response -from pymongo import MongoClient from bson.json_util import dumps as bson2json import json @@ -14,23 +13,39 @@ from operator import itemgetter api = Blueprint("api", __name__) -db = FreifunkDB().handle() - # map ajax @api.route('/get_nearest_router') def get_nearest_router(): - res_router = db.routers.find_one( - {"position": {"$near": {"$geometry": { - "type": "Point", - "coordinates": [float(request.args.get("lng")), float(request.args.get("lat"))] - }}}}, - { - "hostname": 1, - "neighbours": 1, - "position": 1, - "description": 1, - } - ) + lng = float(request.args.get("lng")) + lat = float(request.args.get("lat")) + + mysql = FreifunkMySQL() + res_router = mysql.findone(""" + SELECT id, hostname, lat, lng, description, + ( acos( cos( radians(%s) ) + * cos( radians( lat ) ) + * cos( radians( lng ) - radians(%s) ) + + sin( radians(%s) ) * sin( radians( lat ) ) + ) + ) AS distance + FROM + router + WHERE lat IS NOT NULL AND lng IS NOT NULL + ORDER BY + distance ASC + LIMIT 1 + """,(lat,lng,lat,)) + + res_router["neighbours"] = mysql.fetchall(""" + SELECT nb.mac, nb.quality, nb.net_if, r.hostname, r.id + FROM router_neighbor AS nb + INNER JOIN ( + SELECT router, mac FROM router_netif GROUP BY mac, router + ) AS net ON nb.mac = net.mac + INNER JOIN router as r ON net.router = r.id + WHERE nb.router = %s""",(res_router["id"],)) + mysql.close() + r = make_response(bson2json(res_router)) r.mimetype = 'application/json' return r @@ -38,71 +53,104 @@ def get_nearest_router(): # router by mac (link from router webui) @api.route('/get_router_by_mac/') def get_router_by_mac(mac): - res_routers = db.routers.find({"netifs.mac": mac.lower()}, {"_id": 1}) - if res_routers.count() != 1: + mysql = FreifunkMySQL() + res_routers = mysql.fetchall(""" + SELECT id + FROM router + INNER JOIN router_netif ON router.id = router_netif.router + WHERE mac = %s + GROUP BY mac, id + """,(mac.lower(),)) + mysql.close() + if len(res_routers) != 1: return redirect(url_for("router_list", q="netifs.mac:%s" % mac)) else: - return redirect(url_for("router_info", dbid=next(res_routers)["_id"])) + return redirect(url_for("router_info", dbid=res_routers[0]["id"])) @api.route('/alfred', methods=['GET', 'POST']) def alfred(): - #set_alfred_data = {65: "hallo", 66: "welt"} - set_alfred_data = {} - r = make_response(json.dumps(set_alfred_data)) - #import cProfile, pstats, io - #pr = cProfile.Profile() - #pr.enable() - if request.method == 'POST': - alfred_data = request.get_json() - if alfred_data: - # load router status xml data - for mac, xml in alfred_data.get("64", {}).items(): - import_nodewatcher_xml(mac, xml) - r.headers['X-API-STATUS'] = "ALFRED data imported" - detect_offline_routers() - delete_orphaned_routers() - record_global_stats() - update_mapnik_csv() - #pr.disable() - #s = io.StringIO() - #sortby = 'cumulative' - #ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - #ps.print_stats() - #print(s.getvalue()) - r.mimetype = 'application/json' - return r + try: + mysql = FreifunkMySQL() + #cur = mysql.cursor() + #set_alfred_data = {65: "hallo", 66: "welt"} + set_alfred_data = {} + r = make_response(json.dumps(set_alfred_data)) + #import cProfile, pstats, io + #pr = cProfile.Profile() + #pr.enable() + if request.method == 'POST': + alfred_data = request.get_json() + + if alfred_data: + # load router status xml data + for mac, xml in alfred_data.get("64", {}).items(): + import_nodewatcher_xml(mysql, mac, xml) + mysql.commit() + r.headers['X-API-STATUS'] = "ALFRED data imported" + detect_offline_routers(mysql) + delete_orphaned_routers(mysql) + delete_old_stats(mysql) + record_global_stats(mysql) + update_mapnik_csv(mysql) + mysql.close() + #pr.disable() + #s = io.StringIO() + #sortby = 'cumulative' + #ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + #ps.print_stats() + #print(s.getvalue()) + r.mimetype = 'application/json' + return r + except Exception as e: # most generic exception you can catch + logf = open("/data/fff/fail00.txt", "a") + logf.write("{}\n".format(e)) + logf.close() # https://github.com/ffansbach/de-map/blob/master/schema/nodelist-schema-1.0.0.json @api.route('/nodelist') def nodelist(): - router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'system.clients', 'position.coordinates', 'last_contact']) + mysql = FreifunkMySQL() + router_data = mysql.fetchall(""" + SELECT id, hostname, status, clients, last_contact, lat, lng + FROM router + """,()) + mysql.utcawaretuple(router_data,"last_contact") + mysql.close() nodelist_data = {'version': '1.0.0'} nodelist_data['nodes'] = list() for router in router_data: nodelist_data['nodes'].append( { - 'id': str(router['_id']), + 'id': str(router['id']), 'name': router['hostname'], 'node_type': 'AccessPoint', - 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), + 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']), 'status': { 'online': router['status'] == 'online', - 'clients': router['system']['clients'], + 'clients': router['clients'], 'lastcontact': router['last_contact'].isoformat() } } ) if 'position' in router: nodelist_data['nodes'][-1]['position'] = { - 'lat': router['position']['coordinates'][1], - 'long': router['position']['coordinates'][0] + 'lat': router['lat'], + 'long': router['lng'] } return jsonify(nodelist_data) @api.route('/wifianal/') def wifianal(selecthood): - router_data = db.routers.find({'hood': selecthood}, projection=['hostname','netifs']) + mysql = FreifunkMySQL() + router_data = mysql.fetchall(""" + SELECT hostname, mac, netif + FROM router + INNER JOIN router_netif ON router.id = router_netif.router + WHERE hood = %s + GROUP BY id, netif + """,(selecthood,)) + mysql.close() s = "#----------WifiAnalyzer alias file----------\n" s += "# \n" @@ -118,64 +166,76 @@ def wifianal(selecthood): s += "# \n" for router in router_data: - if not 'netifs' in router: + if not router['mac']: continue - for netif in router['netifs']: - if not 'mac' in netif: - continue - if netif['name'] == 'br-mesh': - s += netif["mac"] + "|Mesh_" + router['hostname'] + "\n" - elif netif['name'] == 'w2ap': - s += netif["mac"] + "|" + router['hostname'] + "\n" - elif netif['name'] == 'w5ap': - s += netif["mac"] + "|W5_" + router['hostname'] + "\n" - elif netif['name'] == 'w5mesh': - s += netif["mac"] + "|W5Mesh_" + router['hostname'] + "\n" + if router["netif"] == 'br-mesh': + s += router["mac"] + "|Mesh_" + router['hostname'] + "\n" + elif router["netif"] == 'w2ap': + s += router["mac"] + "|" + router['hostname'] + "\n" + elif router["netif"] == 'w5ap': + s += router["mac"] + "|W5_" + router['hostname'] + "\n" + elif router["netif"] == 'w5mesh': + s += router["mac"] + "|W5Mesh_" + router['hostname'] + "\n" return Response(s,mimetype='text/plain') @api.route('/routers') def routers(): - router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'hood', 'user.nickname', 'hardware.name', 'software.firmware', 'system.clients', 'position.coordinates', 'last_contact', 'netifs']) + # Suppresses routers without br-mesh + mysql = FreifunkMySQL() + router_data = mysql.fetchall(""" + SELECT router.id, hostname, status, hood, contact, nickname, hardware, firmware, clients, lat, lng, last_contact, mac + FROM router + INNER JOIN router_netif ON router.id = router_netif.router + LEFT JOIN users ON router.contact = users.email + WHERE netif = 'br-mesh' + """) + mysql.utcawaretuple(router_data,"last_contact") + router_net = mysql.fetchall(""" + SELECT id, netif, COUNT(router) AS count + FROM router + INNER JOIN router_netif ON router.id = router_netif.router + GROUP BY id, netif + """) + mysql.close() + net_dict = {} + for rs in router_net: + if not rs["id"] in net_dict: + net_dict[rs["id"]] = [] + net_dict[rs["id"]].append(rs["netif"]) nodelist_data = {'version': '1.0.0'} nodelist_data['nodes'] = list() for router in router_data: - hood = "" - user = "" - firmware = "" - mac = "" fastd = 0 l2tp = 0 - if 'hood' in router: - hood = router['hood'] - if 'user' in router: - user = router['user']['nickname'] - if 'software' in router: - firmware = router['software']['firmware'] + hood = router['hood'] + user = router['nickname'] + firmware = router['firmware'] + mac = router['mac'] - if 'netifs' in router: - for netif in router['netifs']: - if netif['name'] == 'fffVPN': + if router["id"] in net_dict: + for netif in net_dict[router["id"]]: + if netif == 'fffVPN': fastd += 1 - elif netif['name'].startswith('l2tp'): + elif netif.startswith('l2tp'): l2tp += 1 - elif netif['name'] == 'br-mesh' and 'mac' in netif: - mac = netif["mac"] + #elif netif['netif'] == 'br-mesh' and 'mac' in netif: + # mac = netif["mac"] nodelist_data['nodes'].append( { - 'id': str(router['_id']), + 'id': str(router['id']), 'name': router['hostname'], 'mac': mac, 'hood': hood, 'status': router['status'], 'user': user, - 'hardware': router['hardware']['name'], + 'hardware': router['hardware'], 'firmware': firmware, - 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), - 'clients': router['system']['clients'], + 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']), + 'clients': router['clients'], 'lastcontact': router['last_contact'].isoformat(), 'uplink': { 'fastd': fastd, @@ -183,101 +243,118 @@ def routers(): } } ) - if 'position' in router: - nodelist_data['nodes'][-1]['position'] = { - 'lat': router['position']['coordinates'][1], - 'lng': router['position']['coordinates'][0] - } + nodelist_data['nodes'][-1]['position'] = { + 'lat': router['lat'], + 'lng': router['lng'] + } return jsonify(nodelist_data) @api.route('/nopos') def no_position(): - router_data = db.routers.find(filter={'position': { '$exists': False}}, projection=['_id', 'hostname', 'system.contact', 'user.nickname', 'software.firmware']) - #nodelist_data = dict() - nodelist_data = list() - for router in router_data: - nodelist_data.append( - { - 'name': router['hostname'], - 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), - 'firmware': router['software']['firmware'] - } - ) - if 'system' in router and 'contact' in router['system']: - nodelist_data[-1]['contact'] = router['system']['contact'] - if 'user' in router and 'nickname' in router['user']: - nodelist_data[-1]['owner'] = router['user']['nickname'] - else: - nodelist_data[-1]['owner'] = '' + mysql = FreifunkMySQL() + router_data = mysql.fetchall(""" + SELECT router.id, hostname, contact, nickname, firmware + FROM router + LEFT JOIN users ON router.contact = users.email + WHERE lat IS NULL OR lng IS NULL + """) + mysql.close() + #nodelist_data = dict() + nodelist_data = list() + for router in router_data: + nick = router['nickname'] + if not nick: + nick = "" + nodelist_data.append( + { + 'name': router['hostname'], + 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']), + 'firmware': router['firmware'], + 'contact': router['contact'], + 'owner': nick + } + ) + nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False) + nodes = dict() + nodes['nodes'] = list(nodelist_data2) - nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False) - nodes = dict() - nodes['nodes'] = list(nodelist_data2) + return jsonify(nodes) - return jsonify(nodes) - -import pymongo @api.route('/routers_by_nickname/') def get_routers_by_nickname(nickname): - try: - user = db.users.find_one({"nickname": nickname}) - assert user - except AssertionError: - return "User not found" - - nodelist_data = dict() - nodelist_data['nodes'] = list() - routers=db.routers.find({"user._id": user["_id"]}, {"hostname": 1, "netifs": 1, "_id": 1}).sort("hostname", pymongo.ASCENDING) - for router in routers: - #print(router['hostname']) - for netif in router['netifs']: - if netif['name'] == 'br-mesh': - #print(netif['ipv6_fe80_addr']) - nodelist_data['nodes'].append( - { - 'name': router['hostname'], - 'oid': str(router['_id']), - 'ipv6_fe80_addr': netif['ipv6_fe80_addr'] - } - ) - return jsonify(nodelist_data) + mysql = FreifunkMySQL() + users = mysql.fetchall(""" + SELECT id + FROM users + WHERE nickname = %s + LIMIT 1 + """,(nickname,)) + if len(users)==0: + mysql.close() + return "User not found" + nodelist_data = dict() + nodelist_data['nodes'] = list() + routers = mysql.fetchall(""" + SELECT router.id, hostname, contact, nickname, firmware, mac, fe80_addr + FROM router + INNER JOIN users ON router.contact = users.email + INNER JOIN router_netif ON router.id = router_netif.router + WHERE nickname = %s AND netif = 'br-mesh' + ORDER BY hostname ASC + """,(nickname,)) + mysql.close() + for router in routers: + nodelist_data['nodes'].append( + { + 'name': router['hostname'], + 'oid': str(router['id']), + 'mac': router['mac'], + 'ipv6_fe80_addr': router['fe80_addr'] + } + ) + return jsonify(nodelist_data) @api.route('/routers_by_keyxchange_id/') def get_routers_by_keyxchange_id(keyxchange_id): - try: - hood = db.hoods.find_one({"keyxchange_id": int(keyxchange_id)}) - assert hood - except AssertionError: - return "Hood not found" - nodelist_data = dict() - nodelist_data['nodes'] = list() - routers = db.routers.find({"hood": hood["name"]}, {"hostname": 1, "hardware": 1, "netifs": 1, "_id": 1, "software": 1, "position": 1, "system": 1, "position_comment": 1, "description": 1}).sort("hostname", pymongo.ASCENDING) - for router in routers: - for netif in router['netifs']: - if netif['name'] == 'br-mesh': - if 'ipv6_fe80_addr' not in netif: - continue - nodelist_data['nodes'].append( - { - 'name': router['hostname'], - 'ipv6_fe80_addr': netif['ipv6_fe80_addr'], - 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), - 'firmware': router['software']['firmware'], - 'hardware': router['hardware']['name'] - } - ) - if 'position' in router: - nodelist_data['nodes'][-1]['position'] = { - 'lat': router['position']['coordinates'][1], - 'long': router['position']['coordinates'][0] - } - if 'system' in router and 'contact' in router['system']: - nodelist_data['nodes'][-1]['contact'] = router['system']['contact'] - if 'description' in router: - nodelist_data['nodes'][-1]['description'] = router['description'] + mysql = FreifunkMySQL() + hood = mysql.findone(""" + SELECT name + FROM hoods + WHERE id = %s + LIMIT 1 + """,(int(keyxchange_id),)) + if not hood: + mysql.close() + return "Hood not found" - if 'position_comment' in router: - nodelist_data['nodes'][-1]['position']['comment'] = router['position_comment'] - return jsonify(nodelist_data) + nodelist_data = dict() + nodelist_data['nodes'] = list() + routers = mysql.fetchall(""" + SELECT router.id, hostname, hardware, mac, fe80_addr, firmware, lat, lng, contact, position_comment, description + FROM router + INNER JOIN router_netif ON router.id = router_netif.router + WHERE hood = %s AND netif = 'br-mesh' + ORDER BY hostname ASC + """,(hood["name"],)) + mysql.close() + for router in routers: + nodelist_data['nodes'].append( + { + 'name': router['hostname'], + 'ipv6_fe80_addr': router['fe80_addr'], + 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']), + 'firmware': router['firmware'], + 'hardware': router['hardware'], + 'contact': router['contact'], + 'description': router['description'] + } + ) + nodelist_data['nodes'][-1]['position'] = { + 'lat': router['lat'], + 'long': router['lng'] + } + if router['position_comment']: + nodelist_data['nodes'][-1]['position']['comment'] = router['position_comment'] + return jsonify(nodelist_data) diff --git a/ffmap/web/application.py b/ffmap/web/application.py index 1d35687..665561d 100755 --- a/ffmap/web/application.py +++ b/ffmap/web/application.py @@ -6,24 +6,23 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) from ffmap.web.api import api from ffmap.web.filters import filters -from ffmap.dbtools import FreifunkDB +from ffmap.mysqltools import FreifunkMySQL from ffmap import stattools from ffmap.usertools import * +from ffmap.routertools import delete_router from ffmap.web.helpers import * from flask import Flask, render_template, request, Response, redirect, url_for, flash, session import bson -import pymongo from bson.json_util import dumps as bson2json from bson.objectid import ObjectId import base64 +import datetime app = Flask(__name__) app.register_blueprint(api, url_prefix='/api') app.register_blueprint(filters) -db = FreifunkDB().handle() - tileurls = { "links_and_routers": "/tiles/links_and_routers", "hoods": "/tiles/hoods", @@ -45,131 +44,221 @@ def router_map(): @app.route('/routers') def router_list(): query, query_str = parse_router_list_search_query(request.args) - return render_template("router_list.html", query_str=query_str, routers=db.routers.find(query, { - "hostname": 1, - "status": 1, - "hood": 1, - "user.nickname": 1, - "hardware.name": 1, - "created": 1, - "system.uptime": 1, - "system.clients": 1, - }).sort("hostname", pymongo.ASCENDING)) + where, tuple = query2where(query) + mysql = FreifunkMySQL() + + routers = mysql.fetchall(""" + SELECT router.id, hostname, status, hood, contact, nickname, hardware, router.created, sys_uptime, clients + FROM router + LEFT JOIN users ON router.contact = users.email + {} + ORDER BY hostname ASC + """.format(where),tuple) + mysql.close() + mysql.utcawaretuple(routers,"created") + + return render_template("router_list.html", query_str=query_str, routers=routers, numrouters=len(routers)) @app.route('/routers/', methods=['GET', 'POST']) def router_info(dbid): try: - router = db.routers.find_one({"_id": ObjectId(dbid)}) - assert router - if request.method == 'POST': - if request.form.get("act") == "delete": - user = None - # a router may not have a owner, but admin users still can delete it - if ("user" in router) and ("nickname" in router["user"]): - user = router["user"]["nickname"] - if is_authorized(user, session): - db.routers.delete_one({"_id": ObjectId(dbid)}) - flash("Router %s deleted!" % router["hostname"], "success") - return redirect(url_for("index")) - else: - flash("You are not authorized to perform this action!", "danger") - except (bson.errors.InvalidId, AssertionError): - return "Router not found" - if request.args.get('json', None) != None: - del router["stats"] - #FIXME: Only as admin - return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json') - else: - return render_template("router.html", router=router, tileurls=tileurls) + mysql = FreifunkMySQL() + router = mysql.findone("""SELECT * FROM router WHERE id = %s LIMIT 1""",(dbid,)) + + if router: + mysql.utcaware(router,["created","last_contact"]) + + router["user"] = mysql.findone("SELECT nickname FROM users WHERE email = %s",(router["contact"],),"nickname") + router["netifs"] = mysql.fetchall("""SELECT * FROM router_netif WHERE router = %s""",(dbid,)) + for n in router["netifs"]: + n["ipv6_addrs"] = mysql.fetchall("""SELECT ipv6 FROM router_ipv6 WHERE router = %s AND netif = %s""",(dbid,n["netif"],),"ipv6") + + router["neighbours"] = mysql.fetchall(""" + SELECT nb.mac, nb.quality, nb.net_if, r.hostname, r.id + FROM router_neighbor AS nb + INNER JOIN ( + SELECT router, mac FROM router_netif GROUP BY mac, router + ) AS net ON nb.mac = net.mac + INNER JOIN router as r ON net.router = r.id + WHERE nb.router = %s""",(dbid,)) + # FIX SQL: only one from router_netif + + router["events"] = mysql.fetchall("""SELECT * FROM router_events WHERE router = %s""",(dbid,)) + mysql.utcawaretuple(router["events"],"time") + + router["stats"] = mysql.fetchall("""SELECT * FROM router_stats WHERE router = %s""",(dbid,)) + for s in router["stats"]: + s["netifs"] = mysql.fetchdict(""" + SELECT netif, rx, tx FROM router_stats_netif WHERE router = %s AND time = %s + """,(dbid,s["time"],),"netif") + s["neighbours"] = mysql.fetchdict(""" + SELECT quality, mac FROM router_stats_neighbor WHERE router = %s AND time = %s + """,(dbid,s["time"],),"mac","quality") + #s["neighbours"] = mysql.fetchdict(""" + # SELECT nb.mac, nb.quality, r.hostname, r.id + # FROM router_stats_neighbor AS nb + # INNER JOIN ( + # SELECT router, mac FROM router_netif GROUP BY mac, router + # ) AS net ON nb.mac = net.mac + # INNER JOIN router as r ON net.router = r.id + # WHERE nb.router = %s AND time = %s""",(dbid,s["time"],),"mac") + mysql.utcaware(s["time"]) + + if request.method == 'POST': + if request.form.get("act") == "delete": + user = None + # a router may not have a owner, but admin users still can delete it + if ("user" in router): + user = router["user"] + if is_authorized(user, session): + #if True: + delete_router(mysql,dbid) + flash("Router %s deleted!" % router["hostname"], "success") + mysql.close() + return redirect(url_for("index")) + else: + flash("You are not authorized to perform this action!", "danger") + mysql.close() + else: + mysql.close() + return "Router not found" + + if request.args.get('json', None) != None: + del router["stats"] + #FIXME: Only as admin + return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json') + else: + return render_template("router.html", router=router, tileurls=tileurls) + except Exception as e: # most generic exception you can catch + logf = open("/data/fff/fail3.txt", "a") + logf.write("{}\n".format(str(e))) + logf.close() @app.route('/users') def user_list(): + mysql = FreifunkMySQL() + users = mysql.fetchall("SELECT id, nickname, email, created, admin FROM users ORDER BY nickname ASC") + user_routers = stattools.router_user_sum(mysql) + mysql.close() + mysql.utcawaretuple(users,"created") + return render_template("user_list.html", - user_routers = stattools.router_user_sum(), - users = db.users.find({}, {"nickname": 1, "email": 1, "created": 1, "admin": 1}).sort("nickname", pymongo.ASCENDING) + user_routers = user_routers, + users = users, + users_count = len(users) ) @app.route('/users/', methods=['GET', 'POST']) def user_info(nickname): - try: - user = db.users.find_one({"nickname": nickname}) - assert user - except AssertionError: + mysql = FreifunkMySQL() + user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + user["created"] = mysql.utcaware(user["created"]) + if not user: + mysql.close() return "User not found" - if request.method == 'POST': - if is_authorized(user["nickname"], session): - if request.form.get("action") == "changepw": - if request.form["password"] != request.form["password_rep"]: - flash("Passwords did not match!", "danger") - elif request.form["password"] == "": - flash("Password must not be empty!", "danger") - else: - set_user_password(user["nickname"], request.form["password"]) - flash("Password changed!", "success") - elif request.form.get("action") == "changemail": - if request.form["email"] != request.form["email_rep"]: - flash("E-Mail addresses do not match!", "danger") - elif not "@" in request.form["email"]: - flash("Invalid E-Mail addresse!", "danger") - else: - try: - set_user_email(user["nickname"], request.form["email"]) - flash("E-Mail changed!", "success") - if not session.get('admin'): - password = base64.b32encode(os.urandom(10)).decode() - set_user_password(user["nickname"], password) - send_email( - recipient = request.form['email'], - subject = "Password for %s" % user['nickname'], - content = "Hello %s,\n\n" % user["nickname"] + - "You changed your email address on https://monitoring.freifunk-franken.de/\n" + - "To verify your new email address your password was changed to %s\n" % password + - "... and sent to your new address. Please log in and change it.\n\n" + - "Regards,\nFreifunk Franken Monitoring System" - ) - return logout() - else: - # force db data reload - user = db.users.find_one({"nickname": nickname}) - except AccountWithEmailExists: - flash("There is already an account with this E-Mail Address!", "danger") - elif request.form.get("action") == "changeadmin": - if session.get('admin'): - set_user_admin(nickname, request.form.get("admin") == "true") - # force db data reload - user = db.users.find_one({"nickname": nickname}) - elif request.form.get("action") == "deleteaccount": - if session.get('admin'): - db.users.delete_one({"nickname": nickname}) - flash("User %s deleted!" % nickname, "success") - return redirect(url_for("user_list")) - else: - flash("You are not authorized to perform this action!", "danger") - routers=db.routers.find({"user._id": user["_id"]}, { - "hostname": 1, - "status": 1, - "hood": 1, - "software.firmware": 1, - "hardware.name": 1, - "created": 1, - "system.uptime": 1, - "system.clients": 1, - }).sort("hostname", pymongo.ASCENDING) - return render_template("user.html", user=user, routers=routers) + try: + if request.method == 'POST': + if is_authorized(user["nickname"], session): + if request.form.get("action") == "changepw": + if request.form["password"] != request.form["password_rep"]: + flash("Passwords did not match!", "danger") + elif request.form["password"] == "": + flash("Password must not be empty!", "danger") + else: + set_user_password(mysql, user["nickname"], request.form["password"]) + flash("Password changed!", "success") + elif request.form.get("action") == "changemail": + if request.form["email"] != request.form["email_rep"]: + flash("E-Mail addresses do not match!", "danger") + elif not "@" in request.form["email"]: + flash("Invalid E-Mail addresse!", "danger") + else: + try: + set_user_email(mysql, user["nickname"], request.form["email"]) + flash("E-Mail changed!", "success") + if not session.get('admin'): + password = base64.b32encode(os.urandom(10)).decode() + set_user_password(mysql, user["nickname"], password) + send_email( + recipient = request.form['email'], + subject = "Password for %s" % user['nickname'], + content = "Hello %s,\n\n" % user["nickname"] + + "You changed your email address on https://monitoring.freifunk-franken.de/\n" + + "To verify your new email address your password was changed to %s\n" % password + + "... and sent to your new address. Please log in and change it.\n\n" + + "Regards,\nFreifunk Franken Monitoring System" + ) + mysql.close() + return logout() + else: + # force db data reload + mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + except AccountWithEmailExists: + flash("There is already an account with this E-Mail Address!", "danger") + elif request.form.get("action") == "changeadmin": + if session.get('admin'): + set_user_admin(mysql, nickname, request.form.get("admin") == "true") + # force db data reload + mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + elif request.form.get("action") == "deleteaccount": + if session.get('admin'): + cur.execute("DELETE FROM users WHERE nickname = %s LIMIT 1",(nickname,)) + mysql.commit() + flash("User %s deleted!" % nickname, "success") + mysql.close() + return redirect(url_for("user_list")) + else: + flash("You are not authorized to perform this action!", "danger") + routers = mysql.fetchall(""" + SELECT id, hostname, status, hood, firmware, hardware, created, sys_uptime, clients + FROM router + WHERE contact = %s + ORDER BY hostname ASC + """,(user["email"],)) + mysql.close() + mysql.utcawaretuple(routers,"created") + return render_template("user.html", user=user, routers=routers, routers_count=len(routers)) + except Exception as e: + logf = open("/data/fff/fail626.txt", "a") + logf.write("{}\n".format(str(e))) + logf.close() + mysql.close() @app.route('/statistics') def global_statistics(): - hoods = stattools.hoods() + mysql = FreifunkMySQL() + hoods = stattools.hoods(mysql) + + stats = mysql.fetchall("SELECT * FROM stats_global") + mysql.utcawaretuple(stats,"time") + + newest_routers = mysql.fetchall(""" + SELECT id, hostname, hood, created + FROM router + WHERE hardware <> 'Legacy' + ORDER BY created DESC + LIMIT %s + """,(len(hoods)+1,)) + mysql.utcawaretuple(newest_routers,"created") + + clients = stattools.total_clients(mysql) + router_status = stattools.router_status(mysql) + router_models = stattools.router_models(mysql) + router_firmwares = stattools.router_firmwares(mysql) + hoods_sum = stattools.hoods_sum(mysql) + mysql.close() + return render_template("statistics.html", - stats = db.stats.find({}, {"_id": 0}), - clients = stattools.total_clients(), - router_status = stattools.router_status(), - router_models = stattools.router_models(), - router_firmwares = stattools.router_firmwares(), + stats = stats, + clients = clients, + router_status = router_status, + router_models = router_models, + router_firmwares = router_firmwares, hoods = hoods, - hoods_sum = stattools.hoods_sum(), - newest_routers = db.routers.find({"hardware.name": {"$ne": "Legacy"}}, {"hostname": 1, "hood": 1, "created": 1}).sort("created", pymongo.DESCENDING).limit(len(hoods)+1) + hoods_sum = hoods_sum, + newest_routers = newest_routers ) + mysql.close() @app.route('/register', methods=['GET', 'POST']) def register(): @@ -181,10 +270,10 @@ def register(): recipient = request.form['email'], subject = "Password for %s" % request.form['user'], content = "Hello %s,\n\n" % request.form['user'] + - "You created an account on https://monitoring.freifunk-franken.de/\n" + - "To verify your new email address your password was autogenerated to %s\n" % password + - "... and sent to your address. Please log in and change it.\n\n" + - "Regards,\nFreifunk Franken Monitoring System" + "You created an account on https://monitoring.freifunk-franken.de/\n" + + "To verify your new email address your password was autogenerated to %s\n" % password + + "... and sent to your address. Please log in and change it.\n\n" + + "Regards,\nFreifunk Franken Monitoring System" ) flash("Registration successful! - Your password was sent to %s" % request.form['email'], "success") except AccountWithEmailExists: @@ -198,8 +287,9 @@ def resetpw(): try: if request.method == 'POST': token = base64.b32encode(os.urandom(10)).decode() - user = db.users.find_one({"email": request.form['email']}) - reset_user_password(request.form['email'], token) + mysql = FreifunkMySQL() + user = mysql.findone("SELECT nickname FROM users WHERE email = %s",(request.form['email'],)) + reset_user_password(mysql, request.form['email'], token) send_email( recipient = request.form['email'], subject = "Password reset link", @@ -211,10 +301,12 @@ def resetpw(): "Regards,\nFreifunk Franken Monitoring System" ) flash("A password reset link was sent to %s" % request.form['email'], "success") + mysql.close() elif "token" in request.args: password = base64.b32encode(os.urandom(10)).decode() - reset_user_password(request.args['email'], request.args['token'], password) - user = db.users.find_one({"email": request.args['email']}) + mysql = FreifunkMySQL() + reset_user_password(mysql, request.args['email'], request.args['token'], password) + user = mysql.findone("SELECT nickname FROM users WHERE email = %s",(request.args['email'],)) send_email( recipient = request.args['email'], subject = "Your new Password", @@ -225,6 +317,7 @@ def resetpw(): "Regards,\nFreifunk Franken Monitoring System" ) flash("Password reset successful! - Your password was sent to %s" % request.args['email'], "success") + mysql.close() except AccountNotExisting: flash("No Account found with this E-Mail address!", "danger") except InvalidToken: diff --git a/ffmap/web/filters.py b/ffmap/web/filters.py index 8c56e31..051d4de 100644 --- a/ffmap/web/filters.py +++ b/ffmap/web/filters.py @@ -8,7 +8,6 @@ import sys import json import datetime import re -import pymongo import hashlib sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) @@ -99,8 +98,6 @@ def bson_to_json(bsn): @filters.app_template_filter('statbson2json') def statbson_to_json(bsn): - if isinstance(bsn, pymongo.cursor.Cursor): - bsn = list(bsn) for point in bsn: point["time"] = {"$date": int(point["time"].timestamp()*1000)} return json.dumps(bsn) @@ -178,7 +175,7 @@ def gravatar_url(email): @filters.app_template_filter('webui_addr') def webui_addr(router_netifs): try: - for br_mesh in filter(lambda n: n["name"] == "br-mesh", router_netifs): + for br_mesh in filter(lambda n: n["netif"] == "br-mesh", router_netifs): for ipv6 in br_mesh["ipv6_addrs"]: if ipv6.startswith("fd") and len(ipv6) > 25: # This selects the first ULA address, if present diff --git a/ffmap/web/helpers.py b/ffmap/web/helpers.py index 3e4ca9c..db79b28 100644 --- a/ffmap/web/helpers.py +++ b/ffmap/web/helpers.py @@ -19,19 +19,40 @@ allowed_filters = ( 'status', 'hood', 'community', - 'user.nickname', - 'hardware.name', - 'software.firmware', - 'netifs.mac', - 'netifs.name', + 'nickname', + 'hardware', + 'firmware', + 'mac', + 'netif', 'netmon_id', 'hostname', - 'system.contact', + 'contact', ) + +def query2where(query): + s = "" + t = [] + i = 0 + for k, v in query.items(): + if not k in allowed_filters: + # prevent SQL injection + continue + if i==0: + prefix = " WHERE " + else: + prefix = " AND " + i += 1 + s += prefix + k + " = %s" + t.append(v) + return (s,tuple(t)) + def parse_router_list_search_query(args): query_usr = bson.SON() if "q" in args: for word in args["q"].strip().split(" "): + if not word: + # Case of "q=" without arguments + break if not ':' in word: key = "hostname" value = word @@ -41,26 +62,28 @@ def parse_router_list_search_query(args): query_usr[key] = query_usr.get(key, "") + value query = {} for key, value in query_usr.items(): - if value == "EXISTS": - query[key] = {"$exists": True} - elif value == "EXISTS_NOT": - query[key] = {"$exists": False} - elif key == 'netifs.mac': + #if value == "EXISTS": + # query[key] = {"$exists": True} + #elif value == "EXISTS_NOT": + # query[key] = {"$exists": False} + if key == 'mac': query[key] = value.lower() - elif key == 'netifs.name': - query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'} + #elif key == 'netif': + # query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'} elif key == 'hostname': - query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'} - elif key == 'hardware.name': - query[key] = {"$regex": value.replace('.', '\.').replace('_', ' '), "$options": 'i'} - elif key == 'netmon_id': - query[key] = int(value) - elif key == 'system.contact': - if not '\.' in value: - value = re.escape(value) - query[key] = {"$regex": value, "$options": 'i'} - elif value.startswith('!'): - query[key] = {"$ne": value.replace('!', '', 1)} + query[key] = value.replace('\\', '') + elif key == 'hardware': + query[key] = value.replace('\\', '').replace('_', ' ') + #elif key == 'netmon_id': + # query[key] = int(value) + elif key == 'contact': + query[key] = value.replace('\\', '') + #elif key == 'contact': + # if not '\.' in value: + # value = re.escape(value) + # query[key] = {"$regex": value, "$options": 'i'} + #elif value.startswith('!'): + # query[key] = {"$ne": value.replace('!', '', 1)} else: query[key] = value return (query, format_query(query_usr)) diff --git a/ffmap/web/static/js/graph.js b/ffmap/web/static/js/graph.js index 23980df..c05b72d 100644 --- a/ffmap/web/static/js/graph.js +++ b/ffmap/web/static/js/graph.js @@ -146,9 +146,9 @@ function memory_graph() { var len, i; for (len=router_stats.length, i=0; i"; } else { + console.log("Has no neighbours."); popup_html += "
"; } - popup_html += 'Router '+router.hostname+''; + popup_html += 'Router '+router.hostname+''; popup_html += "
" if (has_neighbours) { popup_html += ''; @@ -123,7 +125,7 @@ map.on('click', function(pos) { for (var i = 0; i < router.neighbours.length; i++) { neighbour = router.neighbours[i]; // skip unknown neighbours - if ('_id' in neighbour) { + if ('id' in neighbour) { var tr_color = "#04ff0a"; if (neighbour.quality == -1) { tr_color = "#0684c4"; } else if (neighbour.quality < 105) { tr_color = "#ff1e1e"; } @@ -133,7 +135,7 @@ map.on('click', function(pos) { else if (neighbour.quality < 205) { tr_color = "#ffeb79"; } else if (neighbour.quality < 230) { tr_color = "#79ff7c"; } popup_html += ""; - popup_html += ''; + popup_html += ''; popup_html += ""; popup_html += ""; popup_html += ""; @@ -142,7 +144,7 @@ map.on('click', function(pos) { popup_html += "
'+escapeHTML(neighbour.hostname)+''+escapeHTML(neighbour.hostname)+'"+neighbour.quality+""+escapeHTML(neighbour.net_if)+"
"; } popup = L.popup({offset: new L.Point(1, 1), maxWidth: 500}) - .setLatLng([router.position.coordinates[1], router.position.coordinates[0]]) + .setLatLng([router.lat, router.lng]) .setContent(popup_html) .openOn(map); } diff --git a/ffmap/web/templates/router.html b/ffmap/web/templates/router.html index 290409d..456e124 100644 --- a/ffmap/web/templates/router.html +++ b/ffmap/web/templates/router.html @@ -67,8 +67,8 @@