freifunk-franken
/
monitoring
mirror of https://github.com/FreifunkFranken/fff-monitoring.git
3
0
Fork 0
Code Issues Releases Wiki Activity

MySQL: alpha3 

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>
This commit is contained in:
Adrian Schmutzler 2017-11-05 19:48:29 +01:00
parent 1b4ae0fe51
commit e3fe995407
25 changed files with 1687 additions and 929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
copyusers.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/python3
from ffmap.mysqltools import FreifunkMySQL
import pymongo
from bson.json_util import dumps as bson2json
from bson.objectid import ObjectId
import base64
import datetime
client = MongoClient(tz_aware=True, connect=False)
db = client.freifunk
users = db.users.find({}, {"nickname": 1, "password":1, "email": 1, "token": 1, "created": 1, "admin": 1})
mysql = FreifunkMySQL()
cur = mysql.cursor()
for u in users:
#print(u)
cur.execute("""
INSERT INTO users (nickname, password, token, email, created, admin)
VALUES (%s, %s, %s, %s, %s, %s)
""",(u.get("nickname"),u.get("password"),u.get("token"),u.get("email",""),u.get("created"),u.get("admin",0),))
mysql.commit()
mysql.close()

196
ffmap/db/hooddata.py Executable file
View File

@ -0,0 +1,196 @@
#!/usr/bin/python
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
from ffmap.mysqltools import FreifunkMySQL
import math
mysql = FreifunkMySQL()
hoods = [
{
"keyxchange_id": 1,
"name": "Default",
"net": "10.50.16.0/20"
},
{
"keyxchange_id": 2,
"name": "Fuerth",
"net": "10.50.32.0/21",
"position": {"lng": 10.966, "lat": 49.4814}
},
{
"keyxchange_id": 3,
"name": "Nuernberg",
"net": "10.50.40.0/21",
"position": {"lng": 11.05, "lat": 49.444}
},
{
"keyxchange_id": 4,
"name": "Ansbach",
"net": "10.50.48.0/21",
"position": {"lng": 10.571667, "lat": 49.300833}
},
{
"keyxchange_id": 5,
"name": "Hassberge",
"net": "10.50.56.0/21",
"position": {"lng": 10.568013390003, "lat": 50.093555895082}
},
{
"keyxchange_id": 6,
"name": "Erlangen",
"net": "10.50.64.0/21",
"position": {"lng": 11.0019221, "lat": 49.6005981}
},
{
"keyxchange_id": 7,
"name": "Wuerzburg",
"net": "10.50.72.0/21",
"position": {"lng": 9.93489, "lat": 49.79688}
},
{
"keyxchange_id": 8,
"name": "Bamberg",
"net": "10.50.124.0/22",
"position": {"lng": 10.95, "lat": 49.89}
},
{
"keyxchange_id": 9,
"name": "BGL",
"net": "10.50.80.0/21",
"position": {"lng": 12.8825, "lat": 47.7314}
},
{
"keyxchange_id": 10,
"name": "HassbergeSued",
"net": "10.50.60.0/22",
"position": {"lng": 10.568013390003, "lat": 50.04501}
},
{
"keyxchange_id": 11,
"name": "NbgLand",
"net": "10.50.88.0/21",
"position": {"lng": 11.162796020507812, "lat": 49.39200496388418}
},
{
"keyxchange_id": 12,
"name": "Hof",
"net": "10.50.104.0/21",
"position": {"lng": 11.917545, "lat": 50.312209}
},
{
"keyxchange_id": 13,
"name": "Aschaffenburg",
"net": "10.50.96.0/22",
"position": {"lng": 9.146826, "lat": 49.975661}
},
{
"keyxchange_id": 14,
"name": "Marktredwitz",
"net": "10.50.112.0/22",
"position": {"lng": 12.084797, "lat": 50.002915}
},
{
"keyxchange_id": 15,
"name": "Forchheim",
"net": "10.50.116.0/22",
"position": {"lng": 11.059474, "lat": 49.718820}
},
{
"keyxchange_id": 16,
"name": "Muenchberg",
"net": "10.50.120.0/22",
"position": {"lng": 11.79, "lat": 50.19}
},
{
"keyxchange_id": 17,
"name": "Adelsdorf",
"net": "10.50.144.0/22",
"position": {"lng": 10.894235, "lat": 49.709945}
},
{
"keyxchange_id": 18,
"name": "Schweinfurt",
"net": "10.50.160.0/22",
"position": {"lng": 10.21267, "lat": 50.04683}
},
{
"keyxchange_id": 19,
"name": "ErlangenWest",
"net": "10.50.152.0/22",
"position": {"lng": 10.984488, "lat": 49.6035981}
},
{
"keyxchange_id": 20,
"name": "Ebermannstadt",
"net": "10.50.148.0/22",
"position": {"lng": 11.18538, "lat": 49.78173}
},
{
"keyxchange_id": 21,
"name": "Lauf",
"net": "10.50.156.0/22",
"position": {"lng": 11.278789, "lat": 49.509972}
},
{
"keyxchange_id": 22,
"name": "Bayreuth",
"net": "10.50.168.0/22",
"position": {"lng": 11.580566, "lat": 49.94814}
},
{
"keyxchange_id": 23,
"name": "Fichtelberg",
"net": "10.50.172.0/22",
"position": {"lng": 11.852292, "lat": 49.998920}
},
{
"keyxchange_id": 24,
"name": "Rehau",
"net": "10.50.176.0/22",
"position": {"lng": 12.035305, "lat": 50.247594}
},
{
"keyxchange_id": 25,
"name": "Coburg",
"net": "10.50.180.0/22",
"position": {"lng": 10.964414, "lat": 50.259675}
},
{
"keyxchange_id": 26,
"name": "Ebern",
"net": "10.50.184.0/22",
"position": {"lng": 10.798395, "lat": 50.095572}
},
{
"keyxchange_id": 27,
"name": "Arnstein",
"net": "10.50.188.0/22",
"position": {"lng": 9.970957, "lat": 49.978117}
},
{
"keyxchange_id": 28,
"name": "Erlenbach",
"net": "10.50.192.0/22",
"position": {"lng": 9.157491, "lat": 49.803930}
}]
for h in hoods:
coord = h.get("position",{})
if coord.get("lat"):
cos_lat = math.cos(math.radians(coord.get("lat")))
sin_lat = math.sin(math.radians(coord.get("lat")))
else:
cos_lat = None
sin_lat = None
mysql.execute("""
INSERT INTO hoods (id, name, net, lat, lng, cos_lat, sin_lat)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",(h["keyxchange_id"],h["name"],h["net"],coord.get("lat"),coord.get("lng"),cos_lat,sin_lat,))
mysql.commit()
mysql.close()

205
ffmap/db/hoods.py Normal file → Executable file
View File

@ -1,182 +1,33 @@
#!/usr/bin/python #!/usr/bin/python3
from pymongo import MongoClient import os
client = MongoClient() import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
db = client.freifunk from ffmap.mysqltools import FreifunkMySQL
# create db indexes mysql = FreifunkMySQL()
db.hoods.delete_many({})
db.hoods.create_index([("position", "2dsphere")])
hoods = [ mysql.execute("""
{ CREATE TABLE hoods (
"keyxchange_id": 1, `id` int(11) NOT NULL,
"name": "Default", `name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,
"net": "10.50.16.0/20" `net` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
}, `lat` double DEFAULT NULL,
{ `lng` double DEFAULT NULL,
"keyxchange_id": 2, `cos_lat` double DEFAULT NULL,
"name": "Fuerth", `sin_lat` double DEFAULT NULL
"net": "10.50.32.0/21", ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
"position": {"type": "Point", "coordinates": [10.966, 49.4814]} """)
},
{
"keyxchange_id": 3,
"name": "Nuernberg",
"net": "10.50.40.0/21",
"position": {"type": "Point", "coordinates": [11.05, 49.444]}
},
{
"keyxchange_id": 4,
"name": "Ansbach",
"net": "10.50.48.0/21",
"position": {"type": "Point", "coordinates": [10.571667, 49.300833]}
},
{
"keyxchange_id": 5,
"name": "Hassberge",
"net": "10.50.56.0/21",
"position": {"type": "Point", "coordinates": [10.568013390003, 50.093555895082]}
},
{
"keyxchange_id": 6,
"name": "Erlangen",
"net": "10.50.64.0/21",
"position": {"type": "Point", "coordinates": [11.0019221, 49.6005981]}
},
{
"keyxchange_id": 7,
"name": "Wuerzburg",
"net": "10.50.72.0/21",
"position": {"type": "Point", "coordinates": [9.93489, 49.79688]}
},
{
"keyxchange_id": 8,
"name": "Bamberg",
"net": "10.50.124.0/22",
"position": {"type": "Point", "coordinates": [10.95, 49.89]}
},
{
"keyxchange_id": 9,
"name": "BGL",
"net": "10.50.80.0/21",
"position": {"type": "Point", "coordinates": [12.8825, 47.7314]}
},
{
"keyxchange_id": 10,
"name": "HassbergeSued",
"net": "10.50.60.0/22",
"position": {"type": "Point", "coordinates": [10.568013390003, 50.04501]}
},
{
"keyxchange_id": 11,
"name": "NbgLand",
"net": "10.50.88.0/21",
"position": {"type": "Point", "coordinates": [11.162796020507812, 49.39200496388418]}
},
{
"keyxchange_id": 12,
"name": "Hof",
"net": "10.50.104.0/21",
"position": {"type": "Point", "coordinates": [11.917545, 50.312209]}
},
{
"keyxchange_id": 13,
"name": "Aschaffenburg",
"net": "10.50.96.0/22",
"position": {"type": "Point", "coordinates": [9.146826, 49.975661]}
},
{
"keyxchange_id": 14,
"name": "Marktredwitz",
"net": "10.50.112.0/22",
"position": {"type": "Point", "coordinates": [12.084797, 50.002915]}
},
{
"keyxchange_id": 15,
"name": "Forchheim",
"net": "10.50.116.0/22",
"position": {"type": "Point", "coordinates": [11.059474, 49.718820]}
},
{
"keyxchange_id": 16,
"name": "Muenchberg",
"net": "10.50.120.0/22",
"position": {"type": "Point", "coordinates": [11.79, 50.19]}
},
{
"keyxchange_id": 17,
"name": "Adelsdorf",
"net": "10.50.144.0/22",
"position": {"type": "Point", "coordinates": [10.894235, 49.709945]}
},
{
"keyxchange_id": 18,
"name": "Schweinfurt",
"net": "10.50.160.0/22",
"position": {"type": "Point", "coordinates": [10.21267, 50.04683]}
},
{
"keyxchange_id": 19,
"name": "ErlangenWest",
"net": "10.50.152.0/22",
"position": {"type": "Point", "coordinates": [10.984488, 49.6035981]}
},
{
"keyxchange_id": 20,
"name": "Ebermannstadt",
"net": "10.50.148.0/22",
"position": {"type": "Point", "coordinates": [11.18538, 49.78173]}
},
{
"keyxchange_id": 21,
"name": "Lauf",
"net": "10.50.156.0/22",
"position": {"type": "Point", "coordinates": [11.278789, 49.509972]}
},
{
"keyxchange_id": 22,
"name": "Bayreuth",
"net": "10.50.168.0/22",
"position": {"type": "Point", "coordinates": [11.580566, 49.94814]}
},
{
"keyxchange_id": 23,
"name": "Fichtelberg",
"net": "10.50.172.0/22",
"position": {"type": "Point", "coordinates": [11.852292, 49.998920]}
},
{
"keyxchange_id": 24,
"name": "Rehau",
"net": "10.50.176.0/22",
"position": {"type": "Point", "coordinates": [12.035305, 50.247594]}
},
{
"keyxchange_id": 25,
"name": "Coburg",
"net": "10.50.180.0/22",
"position": {"type": "Point", "coordinates": [10.964414, 50.259675]}
},
{
"keyxchange_id": 26,
"name": "Ebern",
"net": "10.50.184.0/22",
"position": {"type": "Point", "coordinates": [10.798395, 50.095572]}
},
{
"keyxchange_id": 27,
"name": "Arnstein",
"net": "10.50.188.0/22",
"position": {"type": "Point", "coordinates": [9.970957, 49.978117]}
},
{
"keyxchange_id": 28,
"name": "Erlenbach",
"net": "10.50.192.0/22",
"position": {"type": "Point", "coordinates": [9.157491, 49.803930]}
}]
for hood in hoods: mysql.execute("""
db.hoods.insert_one(hood) ALTER TABLE hoods
ADD PRIMARY KEY (`id`),
ADD KEY `name` (`name`),
ADD KEY `lat` (`lat`),
ADD KEY `lng` (`lng`),
ADD KEY `cos_lat` (`cos_lat`),
ADD KEY `sin_lat` (`sin_lat`)
""")
mysql.close()

2
ffmap/db/init_db.py
View File

@ -3,3 +3,5 @@
import routers import routers
import hoods import hoods
import stats import stats
import users
import hooddata

184
ffmap/db/routers.py Normal file → Executable file
View File

@ -1,14 +1,178 @@
#!/usr/bin/python3 #!/usr/bin/python3
from pymongo import MongoClient import os
client = MongoClient() import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
db = client.freifunk from ffmap.mysqltools import FreifunkMySQL
# create db indexes mysql = FreifunkMySQL()
db.routers.create_index("hostname")
db.routers.create_index("status") mysql.execute("""
db.routers.create_index("created") CREATE TABLE router (
db.routers.create_index("last_contact") `id` int(11) NOT NULL,
db.routers.create_index("netifs.mac") `status` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
db.routers.create_index([("position", "2dsphere")]) `hostname` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`created` datetime NOT NULL,
`last_contact` datetime NOT NULL,
`sys_time` datetime NOT NULL,
`sys_uptime` int(11) NOT NULL,
`sys_memfree` int(11) NOT NULL,
`sys_membuff` int(11) NOT NULL,
`sys_memcache` int(11) NOT NULL,
`sys_loadavg` double NOT NULL,
`sys_procrun` smallint(6) NOT NULL,
`sys_proctot` smallint(6) NOT NULL,
`clients` smallint(6) NOT NULL,
`wan_uplink` tinyint(1) NOT NULL,
`cpu` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`chipset` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`hardware` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`os` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`batman` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`kernel` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`nodewatcher` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`firmware` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`firmware_rev` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`description` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`position_comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`community` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`hood` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
`status_text` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`contact` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`lng` double DEFAULT NULL,
`lat` double DEFAULT NULL,
`neighbors` smallint(6) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_events (
`router` int(11) NOT NULL,
`time` datetime NOT NULL,
`type` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_ipv6 (
`router` int(11) NOT NULL,
`netif` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`ipv6` varchar(60) COLLATE utf8_unicode_ci NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_neighbor (
`router` int(11) NOT NULL,
`mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
`quality` smallint(6) NOT NULL,
`net_if` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
`type` varchar(10) COLLATE utf8_unicode_ci DEFAULT 'l2'
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_netif (
`router` int(11) NOT NULL,
`netif` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`mtu` smallint(6) NOT NULL,
`rx_bytes` bigint(20) NOT NULL,
`tx_bytes` bigint(20) NOT NULL,
`rx` bigint(20) NOT NULL,
`tx` bigint(20) NOT NULL,
`fe80_addr` varchar(60) COLLATE utf8_unicode_ci NOT NULL,
`ipv4_addr` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
`mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_stats (
`router` int(11) NOT NULL,
`time` datetime NOT NULL,
`sys_proctot` smallint(6) NOT NULL,
`sys_procrun` smallint(6) NOT NULL,
`sys_memcache` int(11) NOT NULL,
`sys_membuff` int(11) NOT NULL,
`sys_memfree` int(11) NOT NULL,
`loadavg` double NOT NULL,
`clients` smallint(6) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_stats_neighbor (
`router` int(11) NOT NULL,
`mac` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
`time` datetime NOT NULL,
`quality` smallint(6) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
CREATE TABLE router_stats_netif (
`router` int(11) NOT NULL,
`netif` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
`rx` bigint(20) NOT NULL,
`tx` bigint(20) NOT NULL,
`time` datetime NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
ALTER TABLE router
ADD PRIMARY KEY (`id`),
ADD KEY `created` (`created`),
ADD KEY `hostname` (`hostname`),
ADD KEY `status` (`status`),
ADD KEY `last_contact` (`last_contact`),
ADD KEY `lat` (`lat`),
ADD KEY `lng` (`lng`),
ADD KEY `contact` (`contact`),
ADD KEY `hood` (`hood`)
""")
mysql.execute("""
ALTER TABLE router_events
ADD PRIMARY KEY (`router`,`time`,`type`)
""")
mysql.execute("""
ALTER TABLE router_ipv6
ADD PRIMARY KEY (`router`,`netif`,`ipv6`)
""")
mysql.execute("""
ALTER TABLE router_neighbor
ADD PRIMARY KEY (`router`,`mac`,`net_if`)
""")
mysql.execute("""
ALTER TABLE router_netif
ADD PRIMARY KEY (`router`,`netif`),
ADD KEY `mac` (`mac`)
""")
mysql.execute("""
ALTER TABLE router_stats
ADD PRIMARY KEY (`router`,`time`)
""")
mysql.execute("""
ALTER TABLE router_stats_neighbor
ADD PRIMARY KEY (`router`,`mac`,`time`)
""")
mysql.execute("""
ALTER TABLE router_stats_netif
ADD PRIMARY KEY (`router`,`netif`,`time`)
""")
mysql.execute("""
ALTER TABLE router
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT
""")
mysql.close()

27
ffmap/db/stats.py Normal file → Executable file
View File

@ -1,9 +1,26 @@
#!/usr/bin/python3 #!/usr/bin/python3
from pymongo import MongoClient import os
client = MongoClient() import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
db = client.freifunk from ffmap.mysqltools import FreifunkMySQL
# create capped collection mysql = FreifunkMySQL()
db.create_collection("stats", capped=True, size=10*1024*1024, max=4320)
mysql.execute("""
CREATE TABLE `stats_global` (
`time` datetime NOT NULL,
`clients` int(11) NOT NULL,
`online` int(11) NOT NULL,
`offline` int(11) NOT NULL,
`unknown` int(11) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
ALTER TABLE `stats_global`
ADD PRIMARY KEY (`time`)
""")
mysql.close()

38
ffmap/db/users.py Normal file → Executable file
View File

@ -1,10 +1,36 @@
#!/usr/bin/python3 #!/usr/bin/python3
from pymongo import MongoClient import os
client = MongoClient() import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
db = client.freifunk from ffmap.mysqltools import FreifunkMySQL
# create db indexes mysql = FreifunkMySQL()
db.users.create_index("email")
db.users.create_index("nickname") mysql.execute("""
CREATE TABLE `users` (
`id` int(11) NOT NULL,
`nickname` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`password` varchar(250) COLLATE utf8_unicode_ci DEFAULT NULL,
`token` varchar(250) COLLATE utf8_unicode_ci DEFAULT NULL,
`email` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
`created` datetime NOT NULL,
`admin` tinyint(4) NOT NULL DEFAULT '0'
)
ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
mysql.execute("""
ALTER TABLE `users`
ADD PRIMARY KEY (`id`),
ADD KEY `nickname` (`nickname`),
ADD KEY `email` (`email`)
""")
mysql.execute("""
ALTER TABLE `users`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT
""")
mysql.close()

15
ffmap/dbtools.py
View File

@ -1,15 +0,0 @@
#!/usr/bin/python3
from pymongo import MongoClient
class FreifunkDB(object):
client = None
db = None
@classmethod
def handle(cls):
if not cls.client:
cls.client = MongoClient(tz_aware=True, connect=False)
if not cls.db:
cls.db = cls.client.freifunk
return cls.db

107
ffmap/maptools.py
View File

@ -4,7 +4,7 @@ import os
import sys import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
import math import math
import numpy as np import numpy as np
@ -12,8 +12,6 @@ from scipy.spatial import Voronoi
import urllib.request, json import urllib.request, json
db = FreifunkDB().handle()
CONFIG = { CONFIG = {
"csv_dir": "/var/lib/ffmap/csv" "csv_dir": "/var/lib/ffmap/csv"
} }
@ -73,83 +71,88 @@ def draw_voronoi_lines(csv, hoods):
csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2)) csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2))
def update_mapnik_csv(): def update_mapnik_csv(mysql):
with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv: with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv:
csv.write("lng,lat,status\n") csv.write("lng,lat,status\n")
for router in db.routers.find({"position.coordinates": {"$exists": True}}, {"status": 1, "position": 1}): routers = mysql.fetchall("""
SELECT status, lat, lng FROM router
WHERE lat IS NOT NULL AND lng IS NOT NULL
""")
for router in routers:
csv.write("%f,%f,%s\n" % ( csv.write("%f,%f,%s\n" % (
router["position"]["coordinates"][0], router["lng"],
router["position"]["coordinates"][1], router["lat"],
router["status"] router["status"]
)) ))
dblinks = mysql.fetchall("""
SELECT r1.lat AS rlat, r1.lng AS rlng, r2.lat AS nlat, r2.lng AS nlng, n.type AS type, quality
FROM router AS r1
INNER JOIN router_neighbor AS n ON r1.id = n.router
INNER JOIN (
SELECT router, mac FROM router_netif GROUP BY mac, router
) AS net ON n.mac = net.mac
INNER JOIN router AS r2 ON net.router = r2.id
WHERE r1.lat IS NOT NULL AND r1.lng IS NOT NULL AND r2.lat IS NOT NULL AND r2.lng IS NOT NULL
AND r1.status = 'online'
""")
links = []
linksl3 = []
for row in dblinks:
if row.get("type")=="l3":
linksl3.append((
row["rlng"],
row["rlat"],
row["nlng"],
row["nlat"]
))
else:
links.append((
row["rlng"],
row["rlat"],
row["nlng"],
row["nlat"],
row["quality"]
))
with open(os.path.join(CONFIG["csv_dir"], "links.csv"), "w") as csv: with open(os.path.join(CONFIG["csv_dir"], "links.csv"), "w") as csv:
csv.write("WKT,quality\n") csv.write("WKT,quality\n")
links = []
for router in db.routers.find(
{
"position.coordinates": {"$exists": True},
"neighbours": {"$exists": True},
"status": "online"
},
{"position": 1, "neighbours": 1}
):
for neighbour in router["neighbours"]:
if "position" in neighbour and not neighbour.get("type"):
links.append((
router["position"]["coordinates"][0],
router["position"]["coordinates"][1],
neighbour["position"]["coordinates"][0],
neighbour["position"]["coordinates"][1],
neighbour["quality"]
))
for link in sorted(links, key=lambda l: l[4]): for link in sorted(links, key=lambda l: l[4]):
csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link) csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link)
with open(os.path.join(CONFIG["csv_dir"], "l3_links.csv"), "w") as csv: with open(os.path.join(CONFIG["csv_dir"], "l3_links.csv"), "w") as csv:
csv.write("WKT\n") csv.write("WKT\n")
for router in db.routers.find( for link in linksl3:
{ csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % link)
"position.coordinates": {"$exists": True},
"neighbours": {"$exists": True},
"status": "online"
},
{"position": 1, "neighbours": 1}
):
for neighbour in router["neighbours"]:
if "position" in neighbour and neighbour.get("type") and neighbour["type"] == "l3":
csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (
router["position"]["coordinates"][0],
router["position"]["coordinates"][1],
neighbour["position"]["coordinates"][0],
neighbour["position"]["coordinates"][1]
))
dbhoods = mysql.fetchall("""
SELECT name, lat, lng FROM hoods
WHERE lat IS NOT NULL AND lng IS NOT NULL
""")
with open(os.path.join(CONFIG["csv_dir"], "hood-points.csv"), "w", encoding="UTF-8") as csv: with open(os.path.join(CONFIG["csv_dir"], "hood-points.csv"), "w", encoding="UTF-8") as csv:
csv.write("lng,lat,name\n") csv.write("lng,lat,name\n")
for hood in db.hoods.find({"position": {"$exists": True}}): for hood in dbhoods:
csv.write("%f,%f,\"%s\"\n" % ( csv.write("%f,%f,\"%s\"\n" % (
hood["position"]["coordinates"][0], hood["lng"],
hood["position"]["coordinates"][1], hood["lat"],
hood["name"] hood["name"]
)) ))
with open(os.path.join(CONFIG["csv_dir"], "hoods.csv"), "w") as csv: with open(os.path.join(CONFIG["csv_dir"], "hoods.csv"), "w") as csv:
csv.write("WKT\n") csv.write("WKT\n")
hoods = [] hoods = []
for hood in db.hoods.find({"position": {"$exists": True}}): for hood in dbhoods:
# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
x, y = merc_sphere(hood["position"]["coordinates"][0], hood["position"]["coordinates"][1]) x, y = merc_sphere(hood["lng"], hood["lat"])
hoods.append([x, y]) hoods.append([x, y])
draw_voronoi_lines(csv, hoods) draw_voronoi_lines(csv, hoods)
with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
dbhoodsv2 = json.loads(url.read().decode())
with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv: with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv:
csv.write("lng,lat,name\n") csv.write("lng,lat,name\n")
with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url: for hood in dbhoodsv2:
data = json.loads(url.read().decode())
for hood in data:
if not ( 'lon' in hood and 'lat' in hood ): if not ( 'lon' in hood and 'lat' in hood ):
continue continue
csv.write("%f,%f,\"%s\"\n" % ( csv.write("%f,%f,\"%s\"\n" % (
@ -161,10 +164,8 @@ def update_mapnik_csv():
with open(os.path.join(CONFIG["csv_dir"], "hoodsv2.csv"), "w") as csv: with open(os.path.join(CONFIG["csv_dir"], "hoodsv2.csv"), "w") as csv:
csv.write("WKT\n") csv.write("WKT\n")
hoods = [] hoods = []
with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
data = json.loads(url.read().decode())
for hood in data: for hood in dbhoodsv2:
if not ( 'lon' in hood and 'lat' in hood ): if not ( 'lon' in hood and 'lat' in hood ):
continue continue
# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat

8
ffmap/mysqlconfig.example.py Normal file
View File

@ -0,0 +1,8 @@
#!/usr/bin/python3
mysq = {
"host":"localhost",
"user":"root",
"passwd":"password",
"db":"dbname"
}

98
ffmap/mysqltools.py Normal file
View File

@ -0,0 +1,98 @@
#!/usr/bin/python3
import MySQLdb
from ffmap.mysqlconfig import mysq
from ffmap.misc import *
#import pytz
class FreifunkMySQL:
db = None
cur = None
def __init__(self):
#global mysq
self.db = MySQLdb.connect(host=mysq["host"], user=mysq["user"], passwd=mysq["passwd"], db=mysq["db"])
self.cur = self.db.cursor(MySQLdb.cursors.DictCursor)
def close(self):
self.db.close()
def cursor(self):
return self.cur
def commit(self):
self.db.commit()
def fetchall(self,str,tup=(),key=None):
self.cur.execute(str,tup)
result = self.cur.fetchall()
if len(result) > 0:
if key:
rnew = []
for r in result:
rnew.append(r[key])
return rnew
else:
return result
else:
return ()
def fetchdict(self,str,tup,key,value=None):
self.cur.execute(str,tup)
dict = {}
for d in self.cur.fetchall():
if value:
dict[d[key]] = d[value]
else:
dict[d[key]] = d
return dict
def findone(self,str,tup,sel=None):
self.cur.execute(str,tup)
result = self.cur.fetchall()
if len(result) > 0:
if sel:
return result[0][sel]
else:
return result[0]
else:
return False
def execute(self,a,b=None):
if b:
return self.cur.execute(a,b)
else:
return self.cur.execute(a)
def utcnow(self):
return utcnow().strftime('%Y-%m-%d %H:%M:%S')
def formatdt(self,dt):
return dt.strftime('%Y-%m-%d %H:%M:%S')
def utcaware(self,data,keys=None):
if keys:
for k in keys:
#self.utcaware(data[k])
#data[k] = pytz.utc.localize(data[k])
data[k] = data[k].replace(tzinfo=datetime.timezone.utc)
else:
#data = pytz.utc.localize(data)
data = data.replace(tzinfo=datetime.timezone.utc)
return data
def utcawaretuple(self,data,index=None):
if index:
for r in data:
#self.utcaware(r[index])
#r[index] = pytz.utc.localize(r[index])
r[index] = r[index].replace(tzinfo=datetime.timezone.utc)
else:
for r in data:
#self.utcaware(r)
#r = pytz.utc.localize(r)
r = r.replace(tzinfo=datetime.timezone.utc)
return data

471
ffmap/routertools.py
View File

@ -4,7 +4,7 @@ import os
import sys import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
from ffmap.misc import * from ffmap.misc import *
import lxml.etree import lxml.etree
@ -13,8 +13,6 @@ import requests
from bson import SON from bson import SON
from contextlib import suppress from contextlib import suppress
db = FreifunkDB().handle()
CONFIG = { CONFIG = {
"vpn_netif": "fffVPN", "vpn_netif": "fffVPN",
"vpn_netif_l2tp": "l2tp", "vpn_netif_l2tp": "l2tp",
@ -26,9 +24,23 @@ CONFIG = {
router_rate_limit_list = {} router_rate_limit_list = {}
def import_nodewatcher_xml(mac, xml): def delete_router(mysql,dbid):
cur = mysql.cursor()
cur.execute("DELETE FROM router WHERE id = %s",(dbid,))
cur.execute("DELETE FROM router_netif WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_ipv6 WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_neighbor WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_events WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_stats WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_stats_neighbor WHERE router = %s",(dbid,))
cur.execute("DELETE FROM router_stats_netif WHERE router = %s",(dbid,))
mysql.commit()
def import_nodewatcher_xml(mysql, mac, xml):
global router_rate_limit_list global router_rate_limit_list
cur = mysql.cursor()
t = utcnow() t = utcnow()
if mac in router_rate_limit_list: if mac in router_rate_limit_list:
if (t - router_rate_limit_list[mac]) < datetime.timedelta(minutes=5): if (t - router_rate_limit_list[mac]) < datetime.timedelta(minutes=5):
@ -36,188 +48,335 @@ def import_nodewatcher_xml(mac, xml):
router_rate_limit_list[mac] = t router_rate_limit_list[mac] = t
router_id = None router_id = None
olddata = []
uptime = 0
events = [] events = []
status_comment = ""
try: try:
router = db.routers.find_one({"netifs.mac": mac.lower()}, {"stats": 0, "events": 0}) cur.execute("SELECT router FROM router_netif WHERE mac = %s LIMIT 1",(mac.lower(),))
if router: result = cur.fetchall()
router_id = router["_id"] if len(result)>0:
router_id = result[0]["router"]
cur.execute("SELECT sys_uptime AS uptime, firmware, hostname, hood, status, lat, lng FROM router WHERE id = %s LIMIT 1",(router_id,))
result = cur.fetchall()
if len(result)>0:
olddata = result[0]
uptime = olddata["uptime"]
router_update = parse_nodewatcher_xml(xml) router_update = parse_nodewatcher_xml(xml)
# keep hood up to date # keep hood up to date
if not "hood" in router_update: if not router_update["hood"]:
# router didn't send his hood in XML # router didn't send his hood in XML
if "position" in router_update: lat = router_update.get("lat")
# router has new position info from netmon lng = router_update.get("lng")
router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router_update["position"]}}})["name"] if olddata and not lat and not lng:
elif router and "position" in router:
# hoods might change as well # hoods might change as well
router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router["position"]}}})["name"] lat = olddata.get("lat")
lng = olddata.get("lng")
if router: if lat and lng:
router_update["hood"] = mysql.findone("""
SELECT name,
( acos( cos( radians(%s) )
* cos_lat
* cos( radians( lng ) - radians(%s) )
+ sin( radians(%s) ) * sin_lat
)
) AS distance
FROM
hoods
WHERE lat IS NOT NULL AND lng IS NOT NULL
ORDER BY
distance ASC
LIMIT 1
""",(lat,lng,lat,),"name")
else:
router_update["hood"] = None
if router_id:
# statistics # statistics
calculate_network_io(router, router_update) calculate_network_io(cur, router_id, uptime, router_update)
db.routers.update_one({"_id": router_id}, { ru = router_update
"$set": router_update, rus = router_update["system"]
"$push": {"stats": SON([ ruh = router_update["hardware"]
("$each", new_router_stats(router, router_update)), ruso = router_update["software"]
("$slice", int(CONFIG["router_stat_days"] * -1 * 24 * (3600 / 300))) cur.execute("""
]) UPDATE router
}}) SET status = %s, hostname = %s, last_contact = %s, sys_time = %s, sys_uptime = %s, sys_memfree = %s, sys_membuff = %s, sys_memcache = %s,
sys_loadavg = %s, sys_procrun = %s, sys_proctot = %s, clients = %s, wan_uplink = %s, cpu = %s, chipset = %s, hardware = %s, os = %s,
batman = %s, kernel = %s, nodewatcher = %s, firmware = %s, firmware_rev = %s, description = %s, position_comment = %s, community = %s, hood = %s,
status_text = %s, contact = %s, lng = %s, lat = %s, neighbors = %s
WHERE id = %s
""",(
ru["status"],ru["hostname"],ru["last_contact"],rus["time"],rus["uptime"],rus["memory"]["free"],rus["memory"]["buffering"],rus["memory"]["caching"],
rus["loadavg"],rus["processes"]["runnable"],rus["processes"]["total"],rus["clients"],rus["has_wan_uplink"],ruh["cpu"],ruh["chipset"],ruh["name"],ruso["os"],
ruso["batman_adv"],ruso["kernel"],ruso["nodewatcher"],ruso["firmware"],ruso["firmware_rev"],ru["description"],ru["position_comment"],ru["community"],ru["hood"],
ru["system"]["status_text"],ru["system"]["contact"],ru["lng"],ru["lat"],rus["visible_neighbours"],router_id,))
cur.execute("DELETE FROM router_netif WHERE router = %s",(router_id,))
cur.execute("DELETE FROM router_ipv6 WHERE router = %s",(router_id,))
cur.execute("DELETE FROM router_neighbor WHERE router = %s",(router_id,))
uptime = 0
new_router_stats(mysql, router_id, uptime, router_update)
else: else:
# insert new router # insert new router
router_update["created"] = utcnow() created = mysql.utcnow()
router_update["stats"] = [] #events = [] # don't fire sub-events of created events
events = [] # don't fire sub-events of created events #router_update["events"] = [{
router_update["events"] = [{ # "time": utcnow(),
"time": utcnow(), # "type": "created",
"type": "created", #}]
}] ru = router_update
router_id = db.routers.insert_one(router_update).inserted_id rus = router_update["system"]
ruh = router_update["hardware"]
ruso = router_update["software"]
cur.execute("""
INSERT INTO router (status, hostname, created, last_contact, sys_time, sys_uptime, sys_memfree, sys_membuff, sys_memcache,
sys_loadavg, sys_procrun, sys_proctot, clients, wan_uplink, cpu, chipset, hardware, os,
batman, kernel, nodewatcher, firmware, firmware_rev, description, position_comment, community, hood,
status_text, contact, lng, lat, neighbors)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""",(
ru["status"],ru["hostname"],created,ru["last_contact"],rus["time"],rus["uptime"],rus["memory"]["free"],rus["memory"]["buffering"],rus["memory"]["caching"],
rus["loadavg"],rus["processes"]["runnable"],rus["processes"]["total"],rus["clients"],rus["has_wan_uplink"],ruh["cpu"],ruh["chipset"],ruh["name"],ruso["os"],
ruso["batman_adv"],ruso["kernel"],ruso["nodewatcher"],ruso["firmware"],ruso["firmware_rev"],ru["description"],ru["position_comment"],ru["community"],ru["hood"],
ru["system"]["status_text"],ru["system"]["contact"],ru["lng"],ru["lat"],rus["visible_neighbours"],))
router_id = cur.lastrowid
events_append(mysql,router_id,"created","")
for n in router_update["netifs"]:
cur.execute("""
INSERT INTO router_netif (router, netif, mtu, rx_bytes, tx_bytes, rx, tx, fe80_addr, ipv4_addr, mac)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""",(
router_id,n["name"],n["mtu"],n["traffic"]["rx_bytes"],n["traffic"]["tx_bytes"],n["traffic"]["rx"],n["traffic"]["tx"],n["ipv6_fe80_addr"],n["ipv4_addr"],n["mac"],))
for a in n["ipv6_addrs"]:
cur.execute("INSERT INTO router_ipv6 (router, netif, ipv6) VALUES (%s, %s, %s)",(router_id,n["name"],a,))
for n in router_update["neighbours"]:
cur.execute("INSERT INTO router_neighbor (router, mac, quality, net_if, type) VALUES (%s, %s, %s, %s, %s)",(router_id,n["mac"],n["quality"],n["net_if"],n["type"],))
status = router_update["status"] status = router_update["status"]
except ValueError as e: except ValueError as e:
import traceback import traceback
print("Warning: Unable to parse xml from %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) print("Warning: Unable to parse xml from %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__")))
if router: if router_id:
db.routers.update_one({"_id": router_id}, {"$set": { set_status(mysql,router_id,"unknown")
"status": "unknown",
"last_contact": utcnow()
}})
status = "unknown" status = "unknown"
status_comment = "Invalid XML" status_comment = "Invalid XML"
except OverflowError as e: except OverflowError as e:
import traceback import traceback
print("Warning: Overflow Error when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) print("Warning: Overflow Error when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__")))
if router: if router_id:
db.routers.update_one({"_id": router_id}, {"$set": { set_status(mysql,router_id,"unknown")
"status": "unknown",
"last_contact": utcnow()
}})
status = "unknown" status = "unknown"
status_comment = "Integer Overflow" status_comment = "Integer Overflow"
except Exception as e: except Exception as e:
import traceback import traceback
print("Warning: Exception occurred when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__"))) print("Warning: Exception occurred when saving %s: %s\n__%s" % (mac, e, traceback.format_exc().replace("\n", "\n__")))
if router: if router_id:
db.routers.update_one({"_id": router_id}, {"$set": { set_status(mysql,router_id,"unknown")
"status": "unknown",
"last_contact": utcnow()
}})
status = "unknown" status = "unknown"
status_comment = "Exception occurred" status_comment = "Exception occurred"
if router_id: if olddata:
# fire events # fire events
with suppress(KeyError, TypeError, UnboundLocalError): with suppress(KeyError, TypeError, UnboundLocalError):
if router["system"]["uptime"] > router_update["system"]["uptime"]: if olddata["uptime"] > router_update["system"]["uptime"]:
events.append({ events_append(mysql,router_id,"reboot","")
"time": utcnow(),
"type": "reboot",
})
with suppress(KeyError, TypeError, UnboundLocalError): with suppress(KeyError, TypeError, UnboundLocalError):
if router["software"]["firmware"] != router_update["software"]["firmware"]: if olddata["firmware"] != router_update["software"]["firmware"]:
events.append({ events_append(mysql,router_id,"update",
"time": utcnow(), "%s -> %s" % (olddata["firmware"], router_update["software"]["firmware"]))
"type": "update", #events.append({
"comment": "%s -> %s" % (router["software"]["firmware"], router_update["software"]["firmware"]), # "time": utcnow(),
}) # "type": "update",
# "comment": "%s -> %s" % (olddata["firmware"], router_update["software"]["firmware"]),
#})
with suppress(KeyError, TypeError, UnboundLocalError): with suppress(KeyError, TypeError, UnboundLocalError):
if router["hostname"] != router_update["hostname"]: if olddata["hostname"] != router_update["hostname"]:
events.append({ events_append(mysql,router_id,"hostname",
"time": utcnow(), "%s -> %s" % (olddata["hostname"], router_update["hostname"]))
"type": "hostname", #events.append({
"comment": "%s -> %s" % (router["hostname"], router_update["hostname"]), # "time": utcnow(),
}) # "type": "hostname",
# "comment": "%s -> %s" % (olddata["hostname"], router_update["hostname"]),
#})
with suppress(KeyError, TypeError, UnboundLocalError): with suppress(KeyError, TypeError, UnboundLocalError):
if router["hood"] != router_update["hood"]: if olddata["hood"] != router_update["hood"]:
events.append({ events_append(mysql,router_id,"hood",
"time": utcnow(), "%s -> %s" % (olddata["hood"], router_update["hood"]))
"type": "hood", #events.append({
"comment": "%s -> %s" % (router["hood"], router_update["hood"]), # "time": utcnow(),
}) # "type": "hood",
# "comment": "%s -> %s" % (olddata["hood"], router_update["hood"]),
#})
with suppress(KeyError, TypeError): with suppress(KeyError, TypeError):
if router["status"] != status: if olddata["status"] != status:
event = { events_append(mysql,router_id,status,status_comment)
"time": utcnow(), #event = {
"type": status, # "time": utcnow(),
} # "type": status,
with suppress(NameError): #}
event["comment"] = status_comment #with suppress(NameError):
events.append(event) # event["comment"] = status_comment
#events.append(event)
if len(events) > 0: def detect_offline_routers(mysql):
db.routers.update_one({"_id": router_id}, {"$push": {"events": SON([ cur = mysql.cursor()
("$each", events),
("$slice", -10), threshold=mysql.formatdt(utcnow() - datetime.timedelta(minutes=CONFIG["offline_threshold_minutes"]))
])}}) now=mysql.utcnow()
cur.execute("""
SELECT id
FROM router
WHERE last_contact < %s AND status <> 'offline'
""",(threshold,))
result = cur.fetchall()
for r in result:
cur.execute("""
INSERT INTO router_events ( router, time, type, comment )
VALUES (%s, %s, 'offline', '')
""",(r["id"],now,))
cur.execute("""
UPDATE router
SET status = 'offline', clients = 0
WHERE last_contact < %s AND status <> 'offline'
""",(threshold,))
mysql.commit()
def detect_offline_routers(): def delete_orphaned_routers(mysql):
db.routers.update_many({ cur = mysql.cursor()
"last_contact": {"$lt": utcnow() - datetime.timedelta(minutes=CONFIG["offline_threshold_minutes"])},
"status": {"$ne": "offline"} threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["orphan_threshold_days"]))
}, {
"$set": {"status": "offline", "system.clients": 0}, cur.execute("""
"$push": {"events": { DELETE r, e, i, nb, net FROM router AS r
"time": utcnow(), INNER JOIN router_events AS e ON r.id = e.router
"type": "offline" INNER JOIN router_ipv6 AS i ON r.id = i.router
} INNER JOIN router_neighbor AS nb ON r.id = nb.router
}}) INNER JOIN router_netif AS net ON r.id = net.router
WHERE r.last_contact < %s AND r.status <> 'offline'
""",(threshold,))
mysql.commit()
def delete_orphaned_routers(): def delete_old_stats(mysql):
db.routers.delete_many({ threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["router_stat_days"]))
"last_contact": {"$lt": utcnow() - datetime.timedelta(days=CONFIG["orphan_threshold_days"])},
"status": "offline" mysql.execute("""
}) DELETE FROM router_stats
WHERE time < %s
""",(threshold,))
def new_router_stats(router, router_update): mysql.execute("""
if router["system"]["uptime"] < router_update["system"]["uptime"]: DELETE FROM router_stats_neighbor
netifs = {} WHERE time < %s
neighbours = {} """,(threshold,))
mysql.execute("""
DELETE FROM router_stats_netif
WHERE time < %s
""",(threshold,))
mysql.commit()
def events_append(mysql,router_id,event,comment):
mysql.execute("""
INSERT INTO router_events (router, time, type, comment)
VALUES (%s, %s, %s, %s)
""",(
router_id,
mysql.utcnow(),
event,
comment,))
def set_status(mysql,router_id,status):
mysql.execute("""
UPDATE router
SET status = %s, last_contact = %s
WHERE id = %s
""",(
status,
mysql.utcnow(),
router_id,))
def new_router_stats(mysql, router_id, uptime, router_update):
if uptime < router_update["system"]["uptime"]:
time = mysql.utcnow()
mysql.execute("""
INSERT INTO router_stats (router, time, sys_memfree, sys_membuff, sys_memcache, loadavg, sys_procrun, sys_proctot, clients)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""",(
router_id,
time,
router_update["system"]["memory"]['free'],
router_update["system"]["memory"]['buffering'],
router_update["system"]["memory"]['caching'],
router_update["system"]["loadavg"],
router_update["system"]["processes"]['runnable'],
router_update["system"]["processes"]['total'],
router_update["system"]["clients"],))
for netif in router_update["netifs"]: for netif in router_update["netifs"]:
# sanitize name # sanitize name
name = netif["name"].replace(".", "").replace("$", "") name = netif["name"].replace(".", "").replace("$", "")
with suppress(KeyError): with suppress(KeyError):
netifs[name] = {"rx": netif["traffic"]["rx"], "tx": netif["traffic"]["tx"]} mysql.execute("""
INSERT INTO router_stats_netif (router, netif, time, rx, tx)
VALUES (%s, %s, %s, %s, %s)
""",(
router_id,
name,
time,
netif["traffic"]["rx"],
netif["traffic"]["tx"],))
for neighbour in router_update["neighbours"]: for neighbour in router_update["neighbours"]:
with suppress(KeyError): with suppress(KeyError):
neighbours[neighbour["mac"]] = neighbour["quality"] mysql.execute("""
return [{ INSERT INTO router_stats_neighbor (router, mac, time, quality)
"time": utcnow(), VALUES (%s, %s, %s, %s)
"netifs": netifs, """,(
"neighbours": neighbours, router_id,
"memory": router_update["system"]["memory"], neighbour["mac"],
"loadavg": router_update["system"]["loadavg"], time,
"processes": router_update["system"]["processes"], neighbour["quality"],))
"clients": router_update["system"]["clients"],
}]
else:
# don't push old data
return []
def calculate_network_io(router, router_update): def calculate_network_io(cur, router_id, uptime, router_update):
""" """
router: old router dict router: old router dict
router_update: new router dict (which will be updated with new data) router_update: new router dict (which will be updated with new data)
""" """
cur.execute("SELECT netif, rx_bytes, tx_bytes, rx, tx FROM router_netif WHERE router = %s",(router_id,));
results = cur.fetchall()
with suppress(KeyError, StopIteration): with suppress(KeyError, StopIteration):
if router["system"]["uptime"] < router_update["system"]["uptime"]: if uptime < router_update["system"]["uptime"]:
timediff = router_update["system"]["uptime"] - router["system"]["uptime"] timediff = router_update["system"]["uptime"] - uptime
for netif in router["netifs"]: for row in results:
netif_update = next(filter(lambda n: n["name"] == netif["name"], router_update["netifs"])) netif_update = next(filter(lambda n: n["name"] == row["netif"], router_update["netifs"]))
rx_diff = netif_update["traffic"]["rx_bytes"] - netif["traffic"]["rx_bytes"] rx_diff = netif_update["traffic"]["rx_bytes"] - int(row["rx_bytes"])
tx_diff = netif_update["traffic"]["tx_bytes"] - netif["traffic"]["tx_bytes"] tx_diff = netif_update["traffic"]["tx_bytes"] - int(row["tx_bytes"])
if rx_diff >= 0 and tx_diff >= 0: if rx_diff >= 0 and tx_diff >= 0:
netif_update["traffic"]["rx"] = int(rx_diff / timediff) netif_update["traffic"]["rx"] = int(rx_diff / timediff)
netif_update["traffic"]["tx"] = int(tx_diff / timediff) netif_update["traffic"]["tx"] = int(tx_diff / timediff)
else: else:
for netif in router["netifs"]: for row in results:
netif_update = next(filter(lambda n: n["name"] == netif["name"], router_update["netifs"])) netif_update = next(filter(lambda n: n["name"] == row["netif"], router_update["netifs"]))
netif_update["traffic"]["rx"] = netif["traffic"]["rx"] netif_update["traffic"]["rx"] = int(row["rx"])
netif_update["traffic"]["tx"] = netif["traffic"]["tx"] netif_update["traffic"]["tx"] = int(row["tx"])
return uptime
def parse_nodewatcher_xml(xml): def parse_nodewatcher_xml(xml):
try: try:
@ -227,11 +386,11 @@ def parse_nodewatcher_xml(xml):
router_update = { router_update = {
"status": tree.xpath("/data/system_data/status/text()")[0], "status": tree.xpath("/data/system_data/status/text()")[0],
"hostname": tree.xpath("/data/system_data/hostname/text()")[0], "hostname": tree.xpath("/data/system_data/hostname/text()")[0],
"last_contact": utcnow(), "last_contact": utcnow().strftime('%Y-%m-%d %H:%M:%S'),
"neighbours": [], "neighbours": [],
"netifs": [], "netifs": [],
"system": { "system": {
"time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])), "time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])).strftime('%Y-%m-%d %H:%M:%S'),
"uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])), "uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])),
"memory": { "memory": {
"free": int(tree.xpath("/data/system_data/memory_free/text()")[0]), "free": int(tree.xpath("/data/system_data/memory_free/text()")[0]),
@ -276,38 +435,41 @@ def parse_nodewatcher_xml(xml):
router_update["hardware"]["name"] = "Legacy" router_update["hardware"]["name"] = "Legacy"
router_update["hardware"]["name"] = tree.xpath("/data/system_data/model/text()")[0] router_update["hardware"]["name"] = tree.xpath("/data/system_data/model/text()")[0]
# data.system_data.chipset
with suppress(IndexError):
router_update["hardware"]["chipset"] = "Unknown"
router_update["hardware"]["chipset"] = tree.xpath("/data/system_data/chipset/text()")[0]
# data.system_data.description # data.system_data.description
with suppress(IndexError): with suppress(IndexError):
router_update["description"] = ""
router_update["description"] = tree.xpath("/data/system_data/description/text()")[0] router_update["description"] = tree.xpath("/data/system_data/description/text()")[0]
# data.system_data.position_comment # data.system_data.position_comment
with suppress(IndexError): with suppress(IndexError):
router_update["position_comment"] = ""
router_update["position_comment"] = tree.xpath("/data/system_data/position_comment/text()")[0] router_update["position_comment"] = tree.xpath("/data/system_data/position_comment/text()")[0]
# data.system_data.firmware_community # data.system_data.firmware_community
with suppress(IndexError): with suppress(IndexError):
router_update["community"] = ""
router_update["community"] = tree.xpath("/data/system_data/firmware_community/text()")[0] router_update["community"] = tree.xpath("/data/system_data/firmware_community/text()")[0]
# data.system_data.hood # data.system_data.hood
with suppress(IndexError): with suppress(IndexError):
router_update["hood"] = ""
router_update["hood"] = tree.xpath("/data/system_data/hood/text()")[0].lower() router_update["hood"] = tree.xpath("/data/system_data/hood/text()")[0].lower()
# data.system_data.status_text # data.system_data.status_text
with suppress(IndexError): with suppress(IndexError):
router_update["system"]["status_text"] = ""
router_update["system"]["status_text"] = tree.xpath("/data/system_data/status_text/text()")[0] router_update["system"]["status_text"] = tree.xpath("/data/system_data/status_text/text()")[0]
# data.system_data.contact # data.system_data.contact
with suppress(IndexError): with suppress(IndexError):
router_update["system"]["contact"] = ""
#router_update["user"] = ""
router_update["system"]["contact"] = tree.xpath("/data/system_data/contact/text()")[0] router_update["system"]["contact"] = tree.xpath("/data/system_data/contact/text()")[0]
user = db.users.find_one({"email": router_update["system"]["contact"]}) #user = db.users.find_one({"email": router_update["system"]["contact"]})
if user: #if user:
# post-netmon router gets its user assigned # # post-netmon router gets its user assigned
router_update["user"] = {"nickname": user["nickname"], "_id": user["_id"]} # #router_update["user"] = {"nickname": user["nickname"], "_id": user["_id"]}
# router_update["user"] = user["nickname"]
# data.system_data.geo # data.system_data.geo
with suppress(AssertionError, IndexError): with suppress(AssertionError, IndexError):
@ -316,10 +478,8 @@ def parse_nodewatcher_xml(xml):
assert lng != 0 assert lng != 0
assert lat != 0 assert lat != 0
router_update["position"] = { router_update["lng"] = lng
"type": "Point", router_update["lat"] = lat
"coordinates": [lng, lat]
}
#FIXME: tmp workaround to get similar hardware names #FIXME: tmp workaround to get similar hardware names
router_update["hardware"]["name"] = router_update["hardware"]["name"].replace("nanostation-m", "Ubiquiti Nanostation M") router_update["hardware"]["name"] = router_update["hardware"]["name"].replace("nanostation-m", "Ubiquiti Nanostation M")
@ -341,15 +501,19 @@ def parse_nodewatcher_xml(xml):
"traffic": { "traffic": {
"rx_bytes": int(netif.xpath("traffic_rx/text()")[0]), "rx_bytes": int(netif.xpath("traffic_rx/text()")[0]),
"tx_bytes": int(netif.xpath("traffic_tx/text()")[0]), "tx_bytes": int(netif.xpath("traffic_tx/text()")[0]),
"rx": 0,
"tx": 0,
}, },
} }
with suppress(IndexError): with suppress(IndexError):
interface["ipv6_fe80_addr"] = ""
interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0] interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0]
interface["ipv6_addrs"] = []
if len(netif.xpath("ipv6_addr/text()")) > 0: if len(netif.xpath("ipv6_addr/text()")) > 0:
interface["ipv6_addrs"] = []
for ipv6_addr in netif.xpath("ipv6_addr/text()"): for ipv6_addr in netif.xpath("ipv6_addr/text()"):
interface["ipv6_addrs"].append(ipv6_addr.lower().split("/")[0]) interface["ipv6_addrs"].append(ipv6_addr.lower().split("/")[0])
with suppress(IndexError): with suppress(IndexError):
interface["ipv4_addr"] = ""
interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0] interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0]
with suppress(IndexError): with suppress(IndexError):
@ -379,8 +543,8 @@ def parse_nodewatcher_xml(xml):
"mac": o_mac.lower(), "mac": o_mac.lower(),
"quality": int(o_link_quality), "quality": int(o_link_quality),
"net_if": o_out_if, "net_if": o_out_if,
"type": "l2"
} }
set_hostname_and_pos_for_neighbour(neighbour)
router_update["neighbours"].append(neighbour) router_update["neighbours"].append(neighbour)
l3_neighbours = get_l3_neighbours(tree) l3_neighbours = get_l3_neighbours(tree)
@ -392,20 +556,6 @@ def parse_nodewatcher_xml(xml):
except (AssertionError, lxml.etree.XMLSyntaxError, IndexError) as e: except (AssertionError, lxml.etree.XMLSyntaxError, IndexError) as e:
raise ValueError("%s: %s" % (e.__class__.__name__, str(e))) raise ValueError("%s: %s" % (e.__class__.__name__, str(e)))
def set_hostname_and_pos_for_neighbour(neighbour):
with suppress(AssertionError, TypeError):
neighbour_router = db.routers.find_one(
{"netifs.mac": neighbour["mac"]}, {"hostname": 1, "position": 1})
neighbour["_id"] = neighbour_router["_id"]
neighbour["hostname"] = neighbour_router["hostname"]
assert "position" in neighbour_router
assert "coordinates" in neighbour_router["position"]
assert neighbour_router["position"]["coordinates"][0] != 0
assert neighbour_router["position"]["coordinates"][1] != 0
neighbour["position"] = neighbour_router["position"]
def get_l3_neighbours(tree): def get_l3_neighbours(tree):
l3_neighbours = list() l3_neighbours = list()
for neighbour in tree.xpath("/data/babel_neighbours/*"): for neighbour in tree.xpath("/data/babel_neighbours/*"):
@ -417,7 +567,6 @@ def get_l3_neighbours(tree):
"net_if": out_if, "net_if": out_if,
"type": "l3" "type": "l3"
} }
set_hostname_and_pos_for_neighbour(neighbour)
l3_neighbours.append(neighbour) l3_neighbours.append(neighbour)
return l3_neighbours return l3_neighbours

163
ffmap/stattools.py
View File

@ -4,91 +4,102 @@ import os
import sys import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
from ffmap.misc import * from ffmap.misc import *
db = FreifunkDB().handle() CONFIG = {
"global_stat_days": 30,
}
def total_clients(): def total_clients(mysql):
r = db.routers.aggregate([{"$group": { return mysql.findone("""
"_id": None, SELECT SUM(clients) AS clients
"clients": {"$sum": "$system.clients"} FROM router
}}]) """,(),"clients")
return next(r)["clients"]
def router_status(): def router_status(mysql):
r = db.routers.aggregate([{"$group": { return mysql.fetchdict("""
"_id": "$status", SELECT status, COUNT(id) AS count
"count": {"$sum": 1} FROM router
}}]) GROUP BY status
""",(),"status","count")
def router_models(mysql):
return mysql.fetchdict("""
SELECT hardware, COUNT(id) AS count
FROM router
GROUP BY hardware
""",(),"hardware","count")
def router_firmwares(mysql):
return mysql.fetchdict("""
SELECT firmware, COUNT(id) AS count
FROM router
GROUP BY firmware
""",(),"firmware","count")
def hoods(mysql):
data = mysql.fetchall("""
SELECT hood, status, COUNT(id) AS count
FROM router
GROUP BY hood, status
""")
result = {} result = {}
for rs in r: for rs in data:
result[rs["_id"]] = rs["count"] if not rs["hood"]:
rs["hood"] = "default"
if not rs["hood"] in result:
result[rs["hood"]] = {}
result[rs["hood"]][rs["status"]] = rs["count"]
return result return result
def router_models(): def hoods_sum(mysql):
r = db.routers.aggregate([{"$group": { data = mysql.fetchall("""
"_id": "$hardware.name", SELECT hood, COUNT(id) AS count, SUM(clients) AS clients
"count": {"$sum": 1} FROM router
}}]) GROUP BY hood
""")
result = {} result = {}
for rs in r: for rs in data:
result[rs["_id"]] = rs["count"] if not rs["hood"]:
rs["hood"] = "default"
result[rs["hood"]] = {"routers": rs["count"], "clients": rs["clients"]}
return result return result
def router_firmwares(): def record_global_stats(mysql):
r = db.routers.aggregate([{"$group": { threshold=mysql.formatdt(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"]))
"_id": "$software.firmware", time = mysql.utcnow()
"count": {"$sum": 1} status = router_status(mysql)
}}])
old = mysql.findone("SELECT time FROM stats_global WHERE time = %s LIMIT 1",(time,))
if old:
mysql.execute("""
UPDATE stats_global
SET clients = %s, online = %s, offline = %s, unknown = %s
WHERE time = %s
""",(total_clients(mysql),status.get("online",0),status.get("offline",0),status.get("unknown",0),time,))
else:
mysql.execute("""
INSERT INTO stats_global (time, clients, online, offline, unknown)
VALUES (%s, %s, %s, %s, %s)
""",(time,total_clients(mysql),status.get("online",0),status.get("offline",0),status.get("unknown",0),))
mysql.execute("""
DELETE FROM stats_global
WHERE time < %s
""",(threshold,))
mysql.commit()
def router_user_sum(mysql):
data = mysql.fetchall("""
SELECT contact, COUNT(id) AS count, SUM(clients) AS clients
FROM router
GROUP BY contact
""")
result = {} result = {}
for rs in r: for rs in data:
result[rs["_id"]] = rs["count"] if rs["contact"]:
return result result[rs["contact"]] = {"routers": rs["count"], "clients": rs["clients"]}
def hoods():
r = db.routers.aggregate([{"$group": {
"_id": {"hood": "$hood", "status": "$status"},
"count": {"$sum": 1},
}}])
result = {}
for rs in r:
if not "hood" in rs["_id"]:
rs["_id"]["hood"] = "default"
if not rs["_id"]["hood"] in result:
result[rs["_id"]["hood"]] = {}
result[rs["_id"]["hood"]][rs["_id"]["status"]] = rs["count"]
return result
def hoods_sum():
r = db.routers.aggregate([{"$group": {
"_id": "$hood",
"count": {"$sum": 1},
"clients": {"$sum": "$system.clients"}
}}])
result = {}
for rs in r:
if not rs["_id"]:
rs["_id"] = "default"
result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]}
return result
def record_global_stats():
db.stats.insert_one({
"time": utcnow(),
"router_status": router_status(),
"total_clients": total_clients()
})
def router_user_sum():
r = db.routers.aggregate([{"$group": {
"_id": "$user.nickname",
"count": {"$sum": 1},
"clients": {"$sum": "$system.clients"}
}}])
result = {}
for rs in r:
if rs["_id"]:
result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]}
return result return result

116
ffmap/usertools.py
View File

@ -4,13 +4,11 @@ import os
import sys import sys
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
from ffmap.misc import * from ffmap.misc import *
from werkzeug.security import generate_password_hash, check_password_hash from werkzeug.security import generate_password_hash, check_password_hash
db = FreifunkDB().handle()
class AccountWithEmailExists(Exception): class AccountWithEmailExists(Exception):
pass pass
@ -24,67 +22,107 @@ class InvalidToken(Exception):
pass pass
def register_user(nickname, email, password): def register_user(nickname, email, password):
user_with_nick = db.users.find_one({"nickname": nickname}) mysql = FreifunkMySQL()
user_with_email = db.users.find_one({"email": email})
user_with_nick = mysql.findone("SELECT id, email FROM users WHERE nickname = %s LIMIT 1",(nickname,))
user_with_email = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id")
pw = generate_password_hash(password)
if user_with_email: if user_with_email:
mysql.close()
raise AccountWithEmailExists() raise AccountWithEmailExists()
elif user_with_nick and "email" in user_with_nick: elif user_with_nick and user_with_nick["email"]:
mysql.close()
raise AccountWithNicknameExists() raise AccountWithNicknameExists()
else: else:
user_update = { time = mysql.utcnow()
"nickname": nickname,
"password": generate_password_hash(password),
"email": email,
"created": utcnow()
}
if user_with_nick: if user_with_nick:
db.users.update_one({"_id": user_with_nick["_id"]}, {"$set": user_update}) mysql.execute("""
return user_with_nick["_id"] UPDATE users
SET password = %s, email = %s, created = %s, token = NULL
WHERE id = %s
LIMIT 1
""",(pw,email,time,user_with_nick["id"],))
mysql.commit()
mysql.close()
return user_with_nick["id"]
else: else:
return db.users.insert_one(user_update).inserted_id mysql.execute("""
INSERT INTO users (nickname, password, email, created, token)
VALUES (%s, %s, %s, %s, NULL)
""",(nickname,pw,email,time,))
userid = mysql.cursor().lastrowid
mysql.commit()
mysql.close()
return userid
def check_login_details(nickname, password): def check_login_details(nickname, password):
user = db.users.find_one({"nickname": nickname}) mysql = FreifunkMySQL()
user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
mysql.close()
if user and check_password_hash(user.get('password', ''), password): if user and check_password_hash(user.get('password', ''), password):
return user return user
else: else:
return False return False
def reset_user_password(email, token=None, password=None): def reset_user_password(mysql, email, token=None, password=None):
user = db.users.find_one({"email": email}) userid = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id")
if not user: if not user:
raise AccountNotExisting() raise AccountNotExisting()
elif password: elif password:
if user.get("token") == token: if user.get("token") == token:
db.users.update_one({"_id": user["_id"]}, { mysql.execute("""
"$set": {"password": generate_password_hash(password)}, UPDATE users
"$unset": {"token": 1}, SET password = %s, token = NULL
}) WHERE id = %s
LIMIT 1
""",(generate_password_hash(password),userid,))
mysql.commit()
else: else:
raise InvalidToken() raise InvalidToken()
elif token: elif token:
db.users.update_one({"_id": user["_id"]}, {"$set": {"token": token}}) mysql.execute("""
UPDATE users
SET token = %s
WHERE id = %s
LIMIT 1
""",(token,userid,))
mysql.commit()
def set_user_password(nickname, password): def set_user_password(mysql, nickname, password):
user = db.users.find_one({"nickname": nickname}) userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id")
if not user: if not userid:
raise AccountNotExisting() raise AccountNotExisting()
elif password: elif password:
db.users.update_one({"_id": user["_id"]}, { mysql.execute("""
"$set": {"password": generate_password_hash(password)}, UPDATE users
}) SET password = %s
WHERE id = %s
LIMIT 1
""",(generate_password_hash(password),userid,))
mysql.commit()
def set_user_email(nickname, email): def set_user_email(mysql, nickname, email):
user = db.users.find_one({"nickname": nickname}) userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id")
user_with_email = db.users.find_one({"email": email}) useridemail = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id")
if user_with_email: if useridemail:
raise AccountWithEmailExists() raise AccountWithEmailExists()
if not user: if not userid:
raise AccountNotExisting() raise AccountNotExisting()
elif email: elif email:
db.users.update_one({"_id": user["_id"]}, { mysql.execute("""
"$set": {"email": email}, UPDATE users
}) SET email = %s
WHERE id = %s
LIMIT 1
""",(email,userid,))
mysql.commit()
def set_user_admin(nickname, admin): def set_user_admin(mysql, nickname, admin):
db.users.update({"nickname": nickname}, {"$set": {"admin": admin}}) mysql.execute("""
UPDATE users
SET admin = %s
WHERE nickname = %s
LIMIT 1
""",(admin,nickname,))
mysql.commit()

413
ffmap/web/api.py
View File

@ -2,11 +2,10 @@
from ffmap.routertools import * from ffmap.routertools import *
from ffmap.maptools import * from ffmap.maptools import *
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
from ffmap.stattools import record_global_stats from ffmap.stattools import record_global_stats
from flask import Blueprint, request, make_response, redirect, url_for, jsonify, Response from flask import Blueprint, request, make_response, redirect, url_for, jsonify, Response
from pymongo import MongoClient
from bson.json_util import dumps as bson2json from bson.json_util import dumps as bson2json
import json import json
@ -14,23 +13,39 @@ from operator import itemgetter
api = Blueprint("api", __name__) api = Blueprint("api", __name__)
db = FreifunkDB().handle()
# map ajax # map ajax
@api.route('/get_nearest_router') @api.route('/get_nearest_router')
def get_nearest_router(): def get_nearest_router():
res_router = db.routers.find_one( lng = float(request.args.get("lng"))
{"position": {"$near": {"$geometry": { lat = float(request.args.get("lat"))
"type": "Point",
"coordinates": [float(request.args.get("lng")), float(request.args.get("lat"))] mysql = FreifunkMySQL()
}}}}, res_router = mysql.findone("""
{ SELECT id, hostname, lat, lng, description,
"hostname": 1, ( acos( cos( radians(%s) )
"neighbours": 1, * cos( radians( lat ) )
"position": 1, * cos( radians( lng ) - radians(%s) )
"description": 1, + sin( radians(%s) ) * sin( radians( lat ) )
} )
) ) AS distance
FROM
router
WHERE lat IS NOT NULL AND lng IS NOT NULL
ORDER BY
distance ASC
LIMIT 1
""",(lat,lng,lat,))
res_router["neighbours"] = mysql.fetchall("""
SELECT nb.mac, nb.quality, nb.net_if, r.hostname, r.id
FROM router_neighbor AS nb
INNER JOIN (
SELECT router, mac FROM router_netif GROUP BY mac, router
) AS net ON nb.mac = net.mac
INNER JOIN router as r ON net.router = r.id
WHERE nb.router = %s""",(res_router["id"],))
mysql.close()
r = make_response(bson2json(res_router)) r = make_response(bson2json(res_router))
r.mimetype = 'application/json' r.mimetype = 'application/json'
return r return r
@ -38,71 +53,104 @@ def get_nearest_router():
# router by mac (link from router webui) # router by mac (link from router webui)
@api.route('/get_router_by_mac/<mac>') @api.route('/get_router_by_mac/<mac>')
def get_router_by_mac(mac): def get_router_by_mac(mac):
res_routers = db.routers.find({"netifs.mac": mac.lower()}, {"_id": 1}) mysql = FreifunkMySQL()
if res_routers.count() != 1: res_routers = mysql.fetchall("""
SELECT id
FROM router
INNER JOIN router_netif ON router.id = router_netif.router
WHERE mac = %s
GROUP BY mac, id
""",(mac.lower(),))
mysql.close()
if len(res_routers) != 1:
return redirect(url_for("router_list", q="netifs.mac:%s" % mac)) return redirect(url_for("router_list", q="netifs.mac:%s" % mac))
else: else:
return redirect(url_for("router_info", dbid=next(res_routers)["_id"])) return redirect(url_for("router_info", dbid=res_routers[0]["id"]))
@api.route('/alfred', methods=['GET', 'POST']) @api.route('/alfred', methods=['GET', 'POST'])
def alfred(): def alfred():
#set_alfred_data = {65: "hallo", 66: "welt"} try:
set_alfred_data = {} mysql = FreifunkMySQL()
r = make_response(json.dumps(set_alfred_data)) #cur = mysql.cursor()
#import cProfile, pstats, io #set_alfred_data = {65: "hallo", 66: "welt"}
#pr = cProfile.Profile() set_alfred_data = {}
#pr.enable() r = make_response(json.dumps(set_alfred_data))
if request.method == 'POST': #import cProfile, pstats, io
alfred_data = request.get_json() #pr = cProfile.Profile()
if alfred_data: #pr.enable()
# load router status xml data if request.method == 'POST':
for mac, xml in alfred_data.get("64", {}).items(): alfred_data = request.get_json()
import_nodewatcher_xml(mac, xml)
r.headers['X-API-STATUS'] = "ALFRED data imported" if alfred_data:
detect_offline_routers() # load router status xml data
delete_orphaned_routers() for mac, xml in alfred_data.get("64", {}).items():
record_global_stats() import_nodewatcher_xml(mysql, mac, xml)
update_mapnik_csv() mysql.commit()
#pr.disable() r.headers['X-API-STATUS'] = "ALFRED data imported"
#s = io.StringIO() detect_offline_routers(mysql)
#sortby = 'cumulative' delete_orphaned_routers(mysql)
#ps = pstats.Stats(pr, stream=s).sort_stats(sortby) delete_old_stats(mysql)
#ps.print_stats() record_global_stats(mysql)
#print(s.getvalue()) update_mapnik_csv(mysql)
r.mimetype = 'application/json' mysql.close()
return r #pr.disable()
#s = io.StringIO()
#sortby = 'cumulative'
#ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
#ps.print_stats()
#print(s.getvalue())
r.mimetype = 'application/json'
return r
except Exception as e: # most generic exception you can catch
logf = open("/data/fff/fail00.txt", "a")
logf.write("{}\n".format(e))
logf.close()
# https://github.com/ffansbach/de-map/blob/master/schema/nodelist-schema-1.0.0.json # https://github.com/ffansbach/de-map/blob/master/schema/nodelist-schema-1.0.0.json
@api.route('/nodelist') @api.route('/nodelist')
def nodelist(): def nodelist():
router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'system.clients', 'position.coordinates', 'last_contact']) mysql = FreifunkMySQL()
router_data = mysql.fetchall("""
SELECT id, hostname, status, clients, last_contact, lat, lng
FROM router
""",())
mysql.utcawaretuple(router_data,"last_contact")
mysql.close()
nodelist_data = {'version': '1.0.0'} nodelist_data = {'version': '1.0.0'}
nodelist_data['nodes'] = list() nodelist_data['nodes'] = list()
for router in router_data: for router in router_data:
nodelist_data['nodes'].append( nodelist_data['nodes'].append(
{ {
'id': str(router['_id']), 'id': str(router['id']),
'name': router['hostname'], 'name': router['hostname'],
'node_type': 'AccessPoint', 'node_type': 'AccessPoint',
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
'status': { 'status': {
'online': router['status'] == 'online', 'online': router['status'] == 'online',
'clients': router['system']['clients'], 'clients': router['clients'],
'lastcontact': router['last_contact'].isoformat() 'lastcontact': router['last_contact'].isoformat()
} }
} }
) )
if 'position' in router: if 'position' in router:
nodelist_data['nodes'][-1]['position'] = { nodelist_data['nodes'][-1]['position'] = {
'lat': router['position']['coordinates'][1], 'lat': router['lat'],
'long': router['position']['coordinates'][0] 'long': router['lng']
} }
return jsonify(nodelist_data) return jsonify(nodelist_data)
@api.route('/wifianal/<selecthood>') @api.route('/wifianal/<selecthood>')
def wifianal(selecthood): def wifianal(selecthood):
router_data = db.routers.find({'hood': selecthood}, projection=['hostname','netifs']) mysql = FreifunkMySQL()
router_data = mysql.fetchall("""
SELECT hostname, mac, netif
FROM router
INNER JOIN router_netif ON router.id = router_netif.router
WHERE hood = %s
GROUP BY id, netif
""",(selecthood,))
mysql.close()
s = "#----------WifiAnalyzer alias file----------\n" s = "#----------WifiAnalyzer alias file----------\n"
s += "# \n" s += "# \n"
@ -118,64 +166,76 @@ def wifianal(selecthood):
s += "# \n" s += "# \n"
for router in router_data: for router in router_data:
if not 'netifs' in router: if not router['mac']:
continue continue
for netif in router['netifs']: if router["netif"] == 'br-mesh':
if not 'mac' in netif: s += router["mac"] + "|Mesh_" + router['hostname'] + "\n"
continue elif router["netif"] == 'w2ap':
if netif['name'] == 'br-mesh': s += router["mac"] + "|" + router['hostname'] + "\n"
s += netif["mac"] + "|Mesh_" + router['hostname'] + "\n" elif router["netif"] == 'w5ap':
elif netif['name'] == 'w2ap': s += router["mac"] + "|W5_" + router['hostname'] + "\n"
s += netif["mac"] + "|" + router['hostname'] + "\n" elif router["netif"] == 'w5mesh':
elif netif['name'] == 'w5ap': s += router["mac"] + "|W5Mesh_" + router['hostname'] + "\n"
s += netif["mac"] + "|W5_" + router['hostname'] + "\n"
elif netif['name'] == 'w5mesh':
s += netif["mac"] + "|W5Mesh_" + router['hostname'] + "\n"
return Response(s,mimetype='text/plain') return Response(s,mimetype='text/plain')
@api.route('/routers') @api.route('/routers')
def routers(): def routers():
router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'hood', 'user.nickname', 'hardware.name', 'software.firmware', 'system.clients', 'position.coordinates', 'last_contact', 'netifs']) # Suppresses routers without br-mesh
mysql = FreifunkMySQL()
router_data = mysql.fetchall("""
SELECT router.id, hostname, status, hood, contact, nickname, hardware, firmware, clients, lat, lng, last_contact, mac
FROM router
INNER JOIN router_netif ON router.id = router_netif.router
LEFT JOIN users ON router.contact = users.email
WHERE netif = 'br-mesh'
""")
mysql.utcawaretuple(router_data,"last_contact")
router_net = mysql.fetchall("""
SELECT id, netif, COUNT(router) AS count
FROM router
INNER JOIN router_netif ON router.id = router_netif.router
GROUP BY id, netif
""")
mysql.close()
net_dict = {}
for rs in router_net:
if not rs["id"] in net_dict:
net_dict[rs["id"]] = []
net_dict[rs["id"]].append(rs["netif"])
nodelist_data = {'version': '1.0.0'} nodelist_data = {'version': '1.0.0'}
nodelist_data['nodes'] = list() nodelist_data['nodes'] = list()
for router in router_data: for router in router_data:
hood = ""
user = ""
firmware = ""
mac = ""
fastd = 0 fastd = 0
l2tp = 0 l2tp = 0
if 'hood' in router: hood = router['hood']
hood = router['hood'] user = router['nickname']
if 'user' in router: firmware = router['firmware']
user = router['user']['nickname'] mac = router['mac']
if 'software' in router:
firmware = router['software']['firmware']
if 'netifs' in router: if router["id"] in net_dict:
for netif in router['netifs']: for netif in net_dict[router["id"]]:
if netif['name'] == 'fffVPN': if netif == 'fffVPN':
fastd += 1 fastd += 1
elif netif['name'].startswith('l2tp'): elif netif.startswith('l2tp'):
l2tp += 1 l2tp += 1
elif netif['name'] == 'br-mesh' and 'mac' in netif: #elif netif['netif'] == 'br-mesh' and 'mac' in netif:
mac = netif["mac"] # mac = netif["mac"]
nodelist_data['nodes'].append( nodelist_data['nodes'].append(
{ {
'id': str(router['_id']), 'id': str(router['id']),
'name': router['hostname'], 'name': router['hostname'],
'mac': mac, 'mac': mac,
'hood': hood, 'hood': hood,
'status': router['status'], 'status': router['status'],
'user': user, 'user': user,
'hardware': router['hardware']['name'], 'hardware': router['hardware'],
'firmware': firmware, 'firmware': firmware,
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), 'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
'clients': router['system']['clients'], 'clients': router['clients'],
'lastcontact': router['last_contact'].isoformat(), 'lastcontact': router['last_contact'].isoformat(),
'uplink': { 'uplink': {
'fastd': fastd, 'fastd': fastd,
@ -183,101 +243,118 @@ def routers():
} }
} }
) )
if 'position' in router: nodelist_data['nodes'][-1]['position'] = {
nodelist_data['nodes'][-1]['position'] = { 'lat': router['lat'],
'lat': router['position']['coordinates'][1], 'lng': router['lng']
'lng': router['position']['coordinates'][0] }
}
return jsonify(nodelist_data) return jsonify(nodelist_data)
@api.route('/nopos') @api.route('/nopos')
def no_position(): def no_position():
router_data = db.routers.find(filter={'position': { '$exists': False}}, projection=['_id', 'hostname', 'system.contact', 'user.nickname', 'software.firmware']) mysql = FreifunkMySQL()
#nodelist_data = dict() router_data = mysql.fetchall("""
nodelist_data = list() SELECT router.id, hostname, contact, nickname, firmware
for router in router_data: FROM router
nodelist_data.append( LEFT JOIN users ON router.contact = users.email
{ WHERE lat IS NULL OR lng IS NULL
'name': router['hostname'], """)
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']), mysql.close()
'firmware': router['software']['firmware'] #nodelist_data = dict()
} nodelist_data = list()
) for router in router_data:
if 'system' in router and 'contact' in router['system']: nick = router['nickname']
nodelist_data[-1]['contact'] = router['system']['contact'] if not nick:
if 'user' in router and 'nickname' in router['user']: nick = ""
nodelist_data[-1]['owner'] = router['user']['nickname'] nodelist_data.append(
else: {
nodelist_data[-1]['owner'] = '' 'name': router['hostname'],
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
'firmware': router['firmware'],
'contact': router['contact'],
'owner': nick
}
)
nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False)
nodes = dict()
nodes['nodes'] = list(nodelist_data2)
nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False) return jsonify(nodes)
nodes = dict()
nodes['nodes'] = list(nodelist_data2)
return jsonify(nodes)
import pymongo
@api.route('/routers_by_nickname/<nickname>') @api.route('/routers_by_nickname/<nickname>')
def get_routers_by_nickname(nickname): def get_routers_by_nickname(nickname):
try: mysql = FreifunkMySQL()
user = db.users.find_one({"nickname": nickname}) users = mysql.fetchall("""
assert user SELECT id
except AssertionError: FROM users
return "User not found" WHERE nickname = %s
LIMIT 1
nodelist_data = dict() """,(nickname,))
nodelist_data['nodes'] = list() if len(users)==0:
routers=db.routers.find({"user._id": user["_id"]}, {"hostname": 1, "netifs": 1, "_id": 1}).sort("hostname", pymongo.ASCENDING) mysql.close()
for router in routers: return "User not found"
#print(router['hostname'])
for netif in router['netifs']:
if netif['name'] == 'br-mesh':
#print(netif['ipv6_fe80_addr'])
nodelist_data['nodes'].append(
{
'name': router['hostname'],
'oid': str(router['_id']),
'ipv6_fe80_addr': netif['ipv6_fe80_addr']
}
)
return jsonify(nodelist_data)
nodelist_data = dict()
nodelist_data['nodes'] = list()
routers = mysql.fetchall("""
SELECT router.id, hostname, contact, nickname, firmware, mac, fe80_addr
FROM router
INNER JOIN users ON router.contact = users.email
INNER JOIN router_netif ON router.id = router_netif.router
WHERE nickname = %s AND netif = 'br-mesh'
ORDER BY hostname ASC
""",(nickname,))
mysql.close()
for router in routers:
nodelist_data['nodes'].append(
{
'name': router['hostname'],
'oid': str(router['id']),
'mac': router['mac'],
'ipv6_fe80_addr': router['fe80_addr']
}
)
return jsonify(nodelist_data)
@api.route('/routers_by_keyxchange_id/<keyxchange_id>') @api.route('/routers_by_keyxchange_id/<keyxchange_id>')
def get_routers_by_keyxchange_id(keyxchange_id): def get_routers_by_keyxchange_id(keyxchange_id):
try: mysql = FreifunkMySQL()
hood = db.hoods.find_one({"keyxchange_id": int(keyxchange_id)}) hood = mysql.findone("""
assert hood SELECT name
except AssertionError: FROM hoods
return "Hood not found" WHERE id = %s
nodelist_data = dict() LIMIT 1
nodelist_data['nodes'] = list() """,(int(keyxchange_id),))
routers = db.routers.find({"hood": hood["name"]}, {"hostname": 1, "hardware": 1, "netifs": 1, "_id": 1, "software": 1, "position": 1, "system": 1, "position_comment": 1, "description": 1}).sort("hostname", pymongo.ASCENDING) if not hood:
for router in routers: mysql.close()
for netif in router['netifs']: return "Hood not found"
if netif['name'] == 'br-mesh':
if 'ipv6_fe80_addr' not in netif:
continue
nodelist_data['nodes'].append(
{
'name': router['hostname'],
'ipv6_fe80_addr': netif['ipv6_fe80_addr'],
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']),
'firmware': router['software']['firmware'],
'hardware': router['hardware']['name']
}
)
if 'position' in router:
nodelist_data['nodes'][-1]['position'] = {
'lat': router['position']['coordinates'][1],
'long': router['position']['coordinates'][0]
}
if 'system' in router and 'contact' in router['system']:
nodelist_data['nodes'][-1]['contact'] = router['system']['contact']
if 'description' in router:
nodelist_data['nodes'][-1]['description'] = router['description']
if 'position_comment' in router: nodelist_data = dict()
nodelist_data['nodes'][-1]['position']['comment'] = router['position_comment'] nodelist_data['nodes'] = list()
return jsonify(nodelist_data) routers = mysql.fetchall("""
SELECT router.id, hostname, hardware, mac, fe80_addr, firmware, lat, lng, contact, position_comment, description
FROM router
INNER JOIN router_netif ON router.id = router_netif.router
WHERE hood = %s AND netif = 'br-mesh'
ORDER BY hostname ASC
""",(hood["name"],))
mysql.close()
for router in routers:
nodelist_data['nodes'].append(
{
'name': router['hostname'],
'ipv6_fe80_addr': router['fe80_addr'],
'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
'firmware': router['firmware'],
'hardware': router['hardware'],
'contact': router['contact'],
'description': router['description']
}
)
nodelist_data['nodes'][-1]['position'] = {
'lat': router['lat'],
'long': router['lng']
}
if router['position_comment']:
nodelist_data['nodes'][-1]['position']['comment'] = router['position_comment']
return jsonify(nodelist_data)

329
ffmap/web/application.py
View File

@ -6,24 +6,23 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
from ffmap.web.api import api from ffmap.web.api import api
from ffmap.web.filters import filters from ffmap.web.filters import filters
from ffmap.dbtools import FreifunkDB from ffmap.mysqltools import FreifunkMySQL
from ffmap import stattools from ffmap import stattools
from ffmap.usertools import * from ffmap.usertools import *
from ffmap.routertools import delete_router
from ffmap.web.helpers import * from ffmap.web.helpers import *
from flask import Flask, render_template, request, Response, redirect, url_for, flash, session from flask import Flask, render_template, request, Response, redirect, url_for, flash, session
import bson import bson
import pymongo
from bson.json_util import dumps as bson2json from bson.json_util import dumps as bson2json
from bson.objectid import ObjectId from bson.objectid import ObjectId
import base64 import base64
import datetime
app = Flask(__name__) app = Flask(__name__)
app.register_blueprint(api, url_prefix='/api') app.register_blueprint(api, url_prefix='/api')
app.register_blueprint(filters) app.register_blueprint(filters)
db = FreifunkDB().handle()
tileurls = { tileurls = {
"links_and_routers": "/tiles/links_and_routers", "links_and_routers": "/tiles/links_and_routers",
"hoods": "/tiles/hoods", "hoods": "/tiles/hoods",
@ -45,131 +44,221 @@ def router_map():
@app.route('/routers') @app.route('/routers')
def router_list(): def router_list():
query, query_str = parse_router_list_search_query(request.args) query, query_str = parse_router_list_search_query(request.args)
return render_template("router_list.html", query_str=query_str, routers=db.routers.find(query, { where, tuple = query2where(query)
"hostname": 1, mysql = FreifunkMySQL()
"status": 1,
"hood": 1, routers = mysql.fetchall("""
"user.nickname": 1, SELECT router.id, hostname, status, hood, contact, nickname, hardware, router.created, sys_uptime, clients
"hardware.name": 1, FROM router
"created": 1, LEFT JOIN users ON router.contact = users.email
"system.uptime": 1, {}
"system.clients": 1, ORDER BY hostname ASC
}).sort("hostname", pymongo.ASCENDING)) """.format(where),tuple)
mysql.close()
mysql.utcawaretuple(routers,"created")
return render_template("router_list.html", query_str=query_str, routers=routers, numrouters=len(routers))
@app.route('/routers/<dbid>', methods=['GET', 'POST']) @app.route('/routers/<dbid>', methods=['GET', 'POST'])
def router_info(dbid): def router_info(dbid):
try: try:
router = db.routers.find_one({"_id": ObjectId(dbid)}) mysql = FreifunkMySQL()
assert router router = mysql.findone("""SELECT * FROM router WHERE id = %s LIMIT 1""",(dbid,))
if request.method == 'POST':
if request.form.get("act") == "delete": if router:
user = None mysql.utcaware(router,["created","last_contact"])
# a router may not have a owner, but admin users still can delete it
if ("user" in router) and ("nickname" in router["user"]): router["user"] = mysql.findone("SELECT nickname FROM users WHERE email = %s",(router["contact"],),"nickname")
user = router["user"]["nickname"] router["netifs"] = mysql.fetchall("""SELECT * FROM router_netif WHERE router = %s""",(dbid,))
if is_authorized(user, session): for n in router["netifs"]:
db.routers.delete_one({"_id": ObjectId(dbid)}) n["ipv6_addrs"] = mysql.fetchall("""SELECT ipv6 FROM router_ipv6 WHERE router = %s AND netif = %s""",(dbid,n["netif"],),"ipv6")
flash("<b>Router <i>%s</i> deleted!</b>" % router["hostname"], "success")
return redirect(url_for("index")) router["neighbours"] = mysql.fetchall("""
else: SELECT nb.mac, nb.quality, nb.net_if, r.hostname, r.id
flash("<b>You are not authorized to perform this action!</b>", "danger") FROM router_neighbor AS nb
except (bson.errors.InvalidId, AssertionError): INNER JOIN (
return "Router not found" SELECT router, mac FROM router_netif GROUP BY mac, router
if request.args.get('json', None) != None: ) AS net ON nb.mac = net.mac
del router["stats"] INNER JOIN router as r ON net.router = r.id
#FIXME: Only as admin WHERE nb.router = %s""",(dbid,))
return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json') # FIX SQL: only one from router_netif
else:
return render_template("router.html", router=router, tileurls=tileurls) router["events"] = mysql.fetchall("""SELECT * FROM router_events WHERE router = %s""",(dbid,))
mysql.utcawaretuple(router["events"],"time")
router["stats"] = mysql.fetchall("""SELECT * FROM router_stats WHERE router = %s""",(dbid,))
for s in router["stats"]:
s["netifs"] = mysql.fetchdict("""
SELECT netif, rx, tx FROM router_stats_netif WHERE router = %s AND time = %s
""",(dbid,s["time"],),"netif")
s["neighbours"] = mysql.fetchdict("""
SELECT quality, mac FROM router_stats_neighbor WHERE router = %s AND time = %s
""",(dbid,s["time"],),"mac","quality")
#s["neighbours"] = mysql.fetchdict("""
# SELECT nb.mac, nb.quality, r.hostname, r.id
# FROM router_stats_neighbor AS nb
# INNER JOIN (
# SELECT router, mac FROM router_netif GROUP BY mac, router
# ) AS net ON nb.mac = net.mac
# INNER JOIN router as r ON net.router = r.id
# WHERE nb.router = %s AND time = %s""",(dbid,s["time"],),"mac")
mysql.utcaware(s["time"])
if request.method == 'POST':
if request.form.get("act") == "delete":
user = None
# a router may not have a owner, but admin users still can delete it
if ("user" in router):
user = router["user"]
if is_authorized(user, session):
#if True:
delete_router(mysql,dbid)
flash("<b>Router <i>%s</i> deleted!</b>" % router["hostname"], "success")
mysql.close()
return redirect(url_for("index"))
else:
flash("<b>You are not authorized to perform this action!</b>", "danger")
mysql.close()
else:
mysql.close()
return "Router not found"
if request.args.get('json', None) != None:
del router["stats"]
#FIXME: Only as admin
return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json')
else:
return render_template("router.html", router=router, tileurls=tileurls)
except Exception as e: # most generic exception you can catch
logf = open("/data/fff/fail3.txt", "a")
logf.write("{}\n".format(str(e)))
logf.close()
@app.route('/users') @app.route('/users')
def user_list(): def user_list():
mysql = FreifunkMySQL()
users = mysql.fetchall("SELECT id, nickname, email, created, admin FROM users ORDER BY nickname ASC")
user_routers = stattools.router_user_sum(mysql)
mysql.close()
mysql.utcawaretuple(users,"created")
return render_template("user_list.html", return render_template("user_list.html",
user_routers = stattools.router_user_sum(), user_routers = user_routers,
users = db.users.find({}, {"nickname": 1, "email": 1, "created": 1, "admin": 1}).sort("nickname", pymongo.ASCENDING) users = users,
users_count = len(users)
) )
@app.route('/users/<nickname>', methods=['GET', 'POST']) @app.route('/users/<nickname>', methods=['GET', 'POST'])
def user_info(nickname): def user_info(nickname):
try: mysql = FreifunkMySQL()
user = db.users.find_one({"nickname": nickname}) user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
assert user user["created"] = mysql.utcaware(user["created"])
except AssertionError: if not user:
mysql.close()
return "User not found" return "User not found"
if request.method == 'POST': try:
if is_authorized(user["nickname"], session): if request.method == 'POST':
if request.form.get("action") == "changepw": if is_authorized(user["nickname"], session):
if request.form["password"] != request.form["password_rep"]: if request.form.get("action") == "changepw":
flash("<b>Passwords did not match!</b>", "danger") if request.form["password"] != request.form["password_rep"]:
elif request.form["password"] == "": flash("<b>Passwords did not match!</b>", "danger")
flash("<b>Password must not be empty!</b>", "danger") elif request.form["password"] == "":
else: flash("<b>Password must not be empty!</b>", "danger")
set_user_password(user["nickname"], request.form["password"]) else:
flash("<b>Password changed!</b>", "success") set_user_password(mysql, user["nickname"], request.form["password"])
elif request.form.get("action") == "changemail": flash("<b>Password changed!</b>", "success")
if request.form["email"] != request.form["email_rep"]: elif request.form.get("action") == "changemail":
flash("<b>E-Mail addresses do not match!</b>", "danger") if request.form["email"] != request.form["email_rep"]:
elif not "@" in request.form["email"]: flash("<b>E-Mail addresses do not match!</b>", "danger")
flash("<b>Invalid E-Mail addresse!</b>", "danger") elif not "@" in request.form["email"]:
else: flash("<b>Invalid E-Mail addresse!</b>", "danger")
try: else:
set_user_email(user["nickname"], request.form["email"]) try:
flash("<b>E-Mail changed!</b>", "success") set_user_email(mysql, user["nickname"], request.form["email"])
if not session.get('admin'): flash("<b>E-Mail changed!</b>", "success")
password = base64.b32encode(os.urandom(10)).decode() if not session.get('admin'):
set_user_password(user["nickname"], password) password = base64.b32encode(os.urandom(10)).decode()
send_email( set_user_password(mysql, user["nickname"], password)
recipient = request.form['email'], send_email(
subject = "Password for %s" % user['nickname'], recipient = request.form['email'],
content = "Hello %s,\n\n" % user["nickname"] + subject = "Password for %s" % user['nickname'],
"You changed your email address on https://monitoring.freifunk-franken.de/\n" + content = "Hello %s,\n\n" % user["nickname"] +
"To verify your new email address your password was changed to %s\n" % password + "You changed your email address on https://monitoring.freifunk-franken.de/\n" +
"... and sent to your new address. Please log in and change it.\n\n" + "To verify your new email address your password was changed to %s\n" % password +
"Regards,\nFreifunk Franken Monitoring System" "... and sent to your new address. Please log in and change it.\n\n" +
) "Regards,\nFreifunk Franken Monitoring System"
return logout() )
else: mysql.close()
# force db data reload return logout()
user = db.users.find_one({"nickname": nickname}) else:
except AccountWithEmailExists: # force db data reload
flash("<b>There is already an account with this E-Mail Address!</b>", "danger") mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
elif request.form.get("action") == "changeadmin": except AccountWithEmailExists:
if session.get('admin'): flash("<b>There is already an account with this E-Mail Address!</b>", "danger")
set_user_admin(nickname, request.form.get("admin") == "true") elif request.form.get("action") == "changeadmin":
# force db data reload if session.get('admin'):
user = db.users.find_one({"nickname": nickname}) set_user_admin(mysql, nickname, request.form.get("admin") == "true")
elif request.form.get("action") == "deleteaccount": # force db data reload
if session.get('admin'): mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
db.users.delete_one({"nickname": nickname}) elif request.form.get("action") == "deleteaccount":
flash("<b>User <i>%s</i> deleted!</b>" % nickname, "success") if session.get('admin'):
return redirect(url_for("user_list")) cur.execute("DELETE FROM users WHERE nickname = %s LIMIT 1",(nickname,))
else: mysql.commit()
flash("<b>You are not authorized to perform this action!</b>", "danger") flash("<b>User <i>%s</i> deleted!</b>" % nickname, "success")
routers=db.routers.find({"user._id": user["_id"]}, { mysql.close()
"hostname": 1, return redirect(url_for("user_list"))
"status": 1, else:
"hood": 1, flash("<b>You are not authorized to perform this action!</b>", "danger")
"software.firmware": 1, routers = mysql.fetchall("""
"hardware.name": 1, SELECT id, hostname, status, hood, firmware, hardware, created, sys_uptime, clients
"created": 1, FROM router
"system.uptime": 1, WHERE contact = %s
"system.clients": 1, ORDER BY hostname ASC
}).sort("hostname", pymongo.ASCENDING) """,(user["email"],))
return render_template("user.html", user=user, routers=routers) mysql.close()
mysql.utcawaretuple(routers,"created")
return render_template("user.html", user=user, routers=routers, routers_count=len(routers))
except Exception as e:
logf = open("/data/fff/fail626.txt", "a")
logf.write("{}\n".format(str(e)))
logf.close()
mysql.close()
@app.route('/statistics') @app.route('/statistics')
def global_statistics(): def global_statistics():
hoods = stattools.hoods() mysql = FreifunkMySQL()
hoods = stattools.hoods(mysql)
stats = mysql.fetchall("SELECT * FROM stats_global")
mysql.utcawaretuple(stats,"time")
newest_routers = mysql.fetchall("""
SELECT id, hostname, hood, created
FROM router
WHERE hardware <> 'Legacy'
ORDER BY created DESC
LIMIT %s
""",(len(hoods)+1,))
mysql.utcawaretuple(newest_routers,"created")
clients = stattools.total_clients(mysql)
router_status = stattools.router_status(mysql)
router_models = stattools.router_models(mysql)
router_firmwares = stattools.router_firmwares(mysql)
hoods_sum = stattools.hoods_sum(mysql)
mysql.close()
return render_template("statistics.html", return render_template("statistics.html",
stats = db.stats.find({}, {"_id": 0}), stats = stats,
clients = stattools.total_clients(), clients = clients,
router_status = stattools.router_status(), router_status = router_status,
router_models = stattools.router_models(), router_models = router_models,
router_firmwares = stattools.router_firmwares(), router_firmwares = router_firmwares,
hoods = hoods, hoods = hoods,
hoods_sum = stattools.hoods_sum(), hoods_sum = hoods_sum,
newest_routers = db.routers.find({"hardware.name": {"$ne": "Legacy"}}, {"hostname": 1, "hood": 1, "created": 1}).sort("created", pymongo.DESCENDING).limit(len(hoods)+1) newest_routers = newest_routers
) )
mysql.close()
@app.route('/register', methods=['GET', 'POST']) @app.route('/register', methods=['GET', 'POST'])
def register(): def register():
@ -181,10 +270,10 @@ def register():
recipient = request.form['email'], recipient = request.form['email'],
subject = "Password for %s" % request.form['user'], subject = "Password for %s" % request.form['user'],
content = "Hello %s,\n\n" % request.form['user'] + content = "Hello %s,\n\n" % request.form['user'] +
"You created an account on https://monitoring.freifunk-franken.de/\n" + "You created an account on https://monitoring.freifunk-franken.de/\n" +
"To verify your new email address your password was autogenerated to %s\n" % password + "To verify your new email address your password was autogenerated to %s\n" % password +
"... and sent to your address. Please log in and change it.\n\n" + "... and sent to your address. Please log in and change it.\n\n" +
"Regards,\nFreifunk Franken Monitoring System" "Regards,\nFreifunk Franken Monitoring System"
) )
flash("<b>Registration successful!</b> - Your password was sent to %s" % request.form['email'], "success") flash("<b>Registration successful!</b> - Your password was sent to %s" % request.form['email'], "success")
except AccountWithEmailExists: except AccountWithEmailExists:
@ -198,8 +287,9 @@ def resetpw():
try: try:
if request.method == 'POST': if request.method == 'POST':
token = base64.b32encode(os.urandom(10)).decode() token = base64.b32encode(os.urandom(10)).decode()
user = db.users.find_one({"email": request.form['email']}) mysql = FreifunkMySQL()
reset_user_password(request.form['email'], token) user = mysql.findone("SELECT nickname FROM users WHERE email = %s",(request.form['email'],))
reset_user_password(mysql, request.form['email'], token)
send_email( send_email(
recipient = request.form['email'], recipient = request.form['email'],
subject = "Password reset link", subject = "Password reset link",
@ -211,10 +301,12 @@ def resetpw():
"Regards,\nFreifunk Franken Monitoring System" "Regards,\nFreifunk Franken Monitoring System"
) )
flash("<b>A password reset link was sent to %s</b>" % request.form['email'], "success") flash("<b>A password reset link was sent to %s</b>" % request.form['email'], "success")
mysql.close()
elif "token" in request.args: elif "token" in request.args:
password = base64.b32encode(os.urandom(10)).decode() password = base64.b32encode(os.urandom(10)).decode()
reset_user_password(request.args['email'], request.args['token'], password) mysql = FreifunkMySQL()
user = db.users.find_one({"email": request.args['email']}) reset_user_password(mysql, request.args['email'], request.args['token'], password)
user = mysql.findone("SELECT nickname FROM users WHERE email = %s",(request.args['email'],))
send_email( send_email(
recipient = request.args['email'], recipient = request.args['email'],
subject = "Your new Password", subject = "Your new Password",
@ -225,6 +317,7 @@ def resetpw():
"Regards,\nFreifunk Franken Monitoring System" "Regards,\nFreifunk Franken Monitoring System"
) )
flash("<b>Password reset successful!</b> - Your password was sent to %s" % request.args['email'], "success") flash("<b>Password reset successful!</b> - Your password was sent to %s" % request.args['email'], "success")
mysql.close()
except AccountNotExisting: except AccountNotExisting:
flash("<b>No Account found with this E-Mail address!</b>", "danger") flash("<b>No Account found with this E-Mail address!</b>", "danger")
except InvalidToken: except InvalidToken:

5
ffmap/web/filters.py
View File

@ -8,7 +8,6 @@ import sys
import json import json
import datetime import datetime
import re import re
import pymongo
import hashlib import hashlib
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
@ -99,8 +98,6 @@ def bson_to_json(bsn):
@filters.app_template_filter('statbson2json') @filters.app_template_filter('statbson2json')
def statbson_to_json(bsn): def statbson_to_json(bsn):
if isinstance(bsn, pymongo.cursor.Cursor):
bsn = list(bsn)
for point in bsn: for point in bsn:
point["time"] = {"$date": int(point["time"].timestamp()*1000)} point["time"] = {"$date": int(point["time"].timestamp()*1000)}
return json.dumps(bsn) return json.dumps(bsn)
@ -178,7 +175,7 @@ def gravatar_url(email):
@filters.app_template_filter('webui_addr') @filters.app_template_filter('webui_addr')
def webui_addr(router_netifs): def webui_addr(router_netifs):
try: try:
for br_mesh in filter(lambda n: n["name"] == "br-mesh", router_netifs): for br_mesh in filter(lambda n: n["netif"] == "br-mesh", router_netifs):
for ipv6 in br_mesh["ipv6_addrs"]: for ipv6 in br_mesh["ipv6_addrs"]:
if ipv6.startswith("fd") and len(ipv6) > 25: if ipv6.startswith("fd") and len(ipv6) > 25:
# This selects the first ULA address, if present # This selects the first ULA address, if present

71
ffmap/web/helpers.py
View File

@ -19,19 +19,40 @@ allowed_filters = (
'status', 'status',
'hood', 'hood',
'community', 'community',
'user.nickname', 'nickname',
'hardware.name', 'hardware',
'software.firmware', 'firmware',
'netifs.mac', 'mac',
'netifs.name', 'netif',
'netmon_id', 'netmon_id',
'hostname', 'hostname',
'system.contact', 'contact',
) )
def query2where(query):
s = ""
t = []
i = 0
for k, v in query.items():
if not k in allowed_filters:
# prevent SQL injection
continue
if i==0:
prefix = " WHERE "
else:
prefix = " AND "
i += 1
s += prefix + k + " = %s"
t.append(v)
return (s,tuple(t))
def parse_router_list_search_query(args): def parse_router_list_search_query(args):
query_usr = bson.SON() query_usr = bson.SON()
if "q" in args: if "q" in args:
for word in args["q"].strip().split(" "): for word in args["q"].strip().split(" "):
if not word:
# Case of "q=" without arguments
break
if not ':' in word: if not ':' in word:
key = "hostname" key = "hostname"
value = word value = word
@ -41,26 +62,28 @@ def parse_router_list_search_query(args):
query_usr[key] = query_usr.get(key, "") + value query_usr[key] = query_usr.get(key, "") + value
query = {} query = {}
for key, value in query_usr.items(): for key, value in query_usr.items():
if value == "EXISTS": #if value == "EXISTS":
query[key] = {"$exists": True} # query[key] = {"$exists": True}
elif value == "EXISTS_NOT": #elif value == "EXISTS_NOT":
query[key] = {"$exists": False} # query[key] = {"$exists": False}
elif key == 'netifs.mac': if key == 'mac':
query[key] = value.lower() query[key] = value.lower()
elif key == 'netifs.name': #elif key == 'netif':
query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'} # query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'}
elif key == 'hostname': elif key == 'hostname':
query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'} query[key] = value.replace('\\', '')
elif key == 'hardware.name': elif key == 'hardware':
query[key] = {"$regex": value.replace('.', '\.').replace('_', ' '), "$options": 'i'} query[key] = value.replace('\\', '').replace('_', ' ')
elif key == 'netmon_id': #elif key == 'netmon_id':
query[key] = int(value) # query[key] = int(value)
elif key == 'system.contact': elif key == 'contact':
if not '\.' in value: query[key] = value.replace('\\', '')
value = re.escape(value) #elif key == 'contact':
query[key] = {"$regex": value, "$options": 'i'} # if not '\.' in value:
elif value.startswith('!'): # value = re.escape(value)
query[key] = {"$ne": value.replace('!', '', 1)} # query[key] = {"$regex": value, "$options": 'i'}
#elif value.startswith('!'):
# query[key] = {"$ne": value.replace('!', '', 1)}
else: else:
query[key] = value query[key] = value
return (query, format_query(query_usr)) return (query, format_query(query_usr))

22
ffmap/web/static/js/graph.js
View File

@ -146,9 +146,9 @@ function memory_graph() {
var len, i; var len, i;
for (len=router_stats.length, i=0; i<len; i++) { for (len=router_stats.length, i=0; i<len; i++) {
try { try {
var free_value = router_stats[i].memory.free*1024; var free_value = router_stats[i].sys_memfree*1024;
var caching_value = router_stats[i].memory.caching*1024; var caching_value = router_stats[i].sys_memcache*1024;
var buffering_value = router_stats[i].memory.buffering*1024; var buffering_value = router_stats[i].sys_membuff*1024;
var date_value = router_stats[i].time.$date; var date_value = router_stats[i].time.$date;
if(free_value != null && caching_value != null && buffering_value != null) { if(free_value != null && caching_value != null && buffering_value != null) {
free.push([date_value, free_value]); free.push([date_value, free_value]);
@ -181,8 +181,8 @@ function process_graph() {
var len, i; var len, i;
for (len=router_stats.length, i=0; i<len; i++) { for (len=router_stats.length, i=0; i<len; i++) {
try { try {
var runnable_value = router_stats[i].processes.runnable; var runnable_value = router_stats[i].sys_procrun;
var total_value = router_stats[i].processes.total; var total_value = router_stats[i].sys_proctot;
var date_value = router_stats[i].time.$date; var date_value = router_stats[i].time.$date;
if(runnable_value != null && total_value != null) { if(runnable_value != null && total_value != null) {
runnable.push([date_value, runnable_value]); runnable.push([date_value, runnable_value]);
@ -274,7 +274,7 @@ function global_client_graph() {
var len, i; var len, i;
for (len=global_stats.length, i=0; i<len; i++) { for (len=global_stats.length, i=0; i<len; i++) {
try { try {
var client_value = global_stats[i].total_clients; var client_value = global_stats[i].clients;
var date_value = global_stats[i].time.$date; var date_value = global_stats[i].time.$date;
if(client_value != null) { if(client_value != null) {
clients.push([date_value, client_value]); clients.push([date_value, client_value]);
@ -304,9 +304,9 @@ function global_router_graph() {
var len, i; var len, i;
for (len=global_stats.length, i=0; i<len; i++) { for (len=global_stats.length, i=0; i<len; i++) {
try { try {
var offline_value = global_stats[i].router_status.offline; var offline_value = global_stats[i].offline;
var online_value = global_stats[i].router_status.online; var online_value = global_stats[i].online;
var unknown_value = global_stats[i].router_status.unknown; var unknown_value = global_stats[i].unknown;
var date_value = global_stats[i].time.$date; var date_value = global_stats[i].time.$date;
if (offline_value == null) offline_value = 0; if (offline_value == null) offline_value = 0;
if (online_value == null) online_value = 0; if (online_value == null) online_value = 0;
@ -354,7 +354,7 @@ function global_router_firmwares_graph() {
}); });
placeholder.bind("plotclick", function(event, pos, obj) { placeholder.bind("plotclick", function(event, pos, obj) {
if (obj && obj.series.label != "Other") { if (obj && obj.series.label != "Other") {
window.location.href = routers_page_url + "?q=software.firmware:" + obj.series.label; window.location.href = routers_page_url + "?q=firmware:" + obj.series.label;
} }
}); });
} }
@ -379,7 +379,7 @@ function global_router_models_graph() {
}); });
placeholder.bind("plotclick", function(event, pos, obj) { placeholder.bind("plotclick", function(event, pos, obj) {
if (obj && obj.series.label != "Other") { if (obj && obj.series.label != "Other") {
window.location.href = routers_page_url + "?q=hardware.name:" + obj.series.label.replace(/ /g, '_'); window.location.href = routers_page_url + "?q=hardware:" + obj.series.label.replace(/ /g, '_');
} }
}); });
} }

16
ffmap/web/static/js/map.js
View File

@ -75,8 +75,8 @@ map.on('click', function(pos) {
ajax_get_request(url_get_nearest_router + "?lng=" + lng + "&lat=" + lat, function(router) { ajax_get_request(url_get_nearest_router + "?lng=" + lng + "&lat=" + lat, function(router) {
// decide if router is close enough // decide if router is close enough
var lng_delta = Math.abs(lng - router.position.coordinates[0]) var lng_delta = Math.abs(lng - router.lng)
var lat_delta = Math.abs(lat - router.position.coordinates[1]) var lat_delta = Math.abs(lat - router.lat)
// convert degree distances into px distances on the map // convert degree distances into px distances on the map
var x_delta_px = lng_delta * px_per_deg_lng; var x_delta_px = lng_delta * px_per_deg_lng;
@ -99,19 +99,21 @@ map.on('click', function(pos) {
has_neighbours = false; has_neighbours = false;
for (var i = 0; i < router.neighbours.length; i++) { for (var i = 0; i < router.neighbours.length; i++) {
neighbour = router.neighbours[i]; neighbour = router.neighbours[i];
if ('_id' in neighbour) { if ('id' in neighbour) {
has_neighbours = true; has_neighbours = true;
} }
} }
} }
if (has_neighbours) { if (has_neighbours) {
console.log("Has "+router.neighbours.length+" neighbours.");
popup_html += "<div class=\"popup-headline with-neighbours\">"; popup_html += "<div class=\"popup-headline with-neighbours\">";
} }
else { else {
console.log("Has no neighbours.");
popup_html += "<div class=\"popup-headline\">"; popup_html += "<div class=\"popup-headline\">";
} }
popup_html += '<b>Router <a href="' + url_router_info + router._id.$oid +'">'+router.hostname+'</a></b>'; popup_html += '<b>Router <a href="' + url_router_info + router.id +'">'+router.hostname+'</a></b>';
popup_html += "</div>" popup_html += "</div>"
if (has_neighbours) { if (has_neighbours) {
popup_html += '<table class="neighbours" style="width: 100%;">'; popup_html += '<table class="neighbours" style="width: 100%;">';
@ -123,7 +125,7 @@ map.on('click', function(pos) {
for (var i = 0; i < router.neighbours.length; i++) { for (var i = 0; i < router.neighbours.length; i++) {
neighbour = router.neighbours[i]; neighbour = router.neighbours[i];
// skip unknown neighbours // skip unknown neighbours
if ('_id' in neighbour) { if ('id' in neighbour) {
var tr_color = "#04ff0a"; var tr_color = "#04ff0a";
if (neighbour.quality == -1) { tr_color = "#0684c4"; } if (neighbour.quality == -1) { tr_color = "#0684c4"; }
else if (neighbour.quality < 105) { tr_color = "#ff1e1e"; } else if (neighbour.quality < 105) { tr_color = "#ff1e1e"; }
@ -133,7 +135,7 @@ map.on('click', function(pos) {
else if (neighbour.quality < 205) { tr_color = "#ffeb79"; } else if (neighbour.quality < 205) { tr_color = "#ffeb79"; }
else if (neighbour.quality < 230) { tr_color = "#79ff7c"; } else if (neighbour.quality < 230) { tr_color = "#79ff7c"; }
popup_html += "<tr style=\"background-color: "+tr_color+";\">"; popup_html += "<tr style=\"background-color: "+tr_color+";\">";
popup_html += '<td><a href="'+url_router_info+neighbour._id.$oid+'" title="'+escapeHTML(neighbour.mac)+'">'+escapeHTML(neighbour.hostname)+'</a></td>'; popup_html += '<td><a href="'+url_router_info+neighbour.id+'" title="'+escapeHTML(neighbour.mac)+'">'+escapeHTML(neighbour.hostname)+'</a></td>';
popup_html += "<td>"+neighbour.quality+"</td>"; popup_html += "<td>"+neighbour.quality+"</td>";
popup_html += "<td>"+escapeHTML(neighbour.net_if)+"</td>"; popup_html += "<td>"+escapeHTML(neighbour.net_if)+"</td>";
popup_html += "</tr>"; popup_html += "</tr>";
@ -142,7 +144,7 @@ map.on('click', function(pos) {
popup_html += "</table>"; popup_html += "</table>";
} }
popup = L.popup({offset: new L.Point(1, 1), maxWidth: 500}) popup = L.popup({offset: new L.Point(1, 1), maxWidth: 500})
.setLatLng([router.position.coordinates[1], router.position.coordinates[0]]) .setLatLng([router.lat, router.lng])
.setContent(popup_html) .setContent(popup_html)
.openOn(map); .openOn(map);
} }

68
ffmap/web/templates/router.html
View File

@ -67,8 +67,8 @@
</script> </script>
<script src="{{ url_for('static', filename='js/map.js') }}"></script> <script src="{{ url_for('static', filename='js/map.js') }}"></script>
<script type="text/javascript"> <script type="text/javascript">
{%- if router.position %} {%- if router.lng!="" and router.lat!="" %}
var router_pos = [{{ router.position.coordinates[1] }}, {{ router.position.coordinates[0] }}]; var router_pos = [{{ router.lat }}, {{ router.lng }}];
map.setView(router_pos, 18); map.setView(router_pos, 18);
var marker = L.marker(router_pos, { var marker = L.marker(router_pos, {
icon: L.icon({ icon: L.icon({
@ -97,7 +97,7 @@
</td></tr> </td></tr>
<tr><th>Status</th><td><span class="{{ router.status|status2css }}">{{ router.status }}</span> <tr><th>Status</th><td><span class="{{ router.status|status2css }}">{{ router.status }}</span>
{%- if router.status == "online" %} {%- if router.status == "online" %}
({{ router.system.uptime|format_ts_diff }} up) ({{ router.sys_uptime|format_ts_diff }} up)
{%- endif -%} {%- endif -%}
</td></tr> </td></tr>
<tr><th>Created</th><td> <tr><th>Created</th><td>
@ -105,10 +105,10 @@
</td></tr> </td></tr>
<tr><th class="text-nowrap">Last contact</th><td> <tr><th class="text-nowrap">Last contact</th><td>
{{ router.last_contact|utc2local|format_dt }} {{ router.last_contact|utc2local|format_dt }}
({{ router.last_contact|format_dt_ago }}){{- "" -}} ({{ router.last_contact|utc2local|format_dt_ago }}){{- "" -}}
</td></tr> </td></tr>
{%- if router.system.status_text %} {%- if router.status_text %}
<tr><th>Status Text</th><td>{{ router.system.status_text }}</td></tr> <tr><th>Status Text</th><td>{{ router.status_text }}</td></tr>
{%- endif %} {%- endif %}
{%- if router.description %} {%- if router.description %}
<tr><th>Description</th><td>{{ router.description }}</td></tr> <tr><th>Description</th><td>{{ router.description }}</td></tr>
@ -123,47 +123,43 @@
{%- endif -%} {%- endif -%}
</td></tr> </td></tr>
{%- endif %} {%- endif %}
{%- if router.user or router.system.contact %} {%- if router.user or router.contact %}
<tr><th>User</th><td> <tr><th>User</th><td>
{%- if router.user -%} {%- if router.user -%}
<a href="{{ url_for('user_info', nickname=router.user.nickname) }}">{{ router.user.nickname }}</a> <a href="{{ url_for('user_info', nickname=router.user) }}">{{ router.user }}</a>
{%- if router.system.contact %} {%- if router.contact %}
(<a href="{{ url_for('router_list', q='system.contact:%s' % router.system.contact|anon_email_regex) }}">{{ router.system.contact|anon_email }}</a>) (<a href="{{ url_for('router_list', q='contact:%s' % router.contact|anon_email_regex) }}">{{ router.contact|anon_email }}</a>)
{%- endif -%} {%- endif -%}
{%- else -%} {%- else -%}
<a href="{{ url_for('router_list', q='system.contact:%s' % router.system.contact|anon_email_regex) }}">{{ router.system.contact|anon_email }}</a> <a href="{{ url_for('router_list', q='contact:%s' % router.contact|anon_email_regex) }}">{{ router.contact|anon_email }}</a>
{%- endif -%} {%- endif -%}
</td></tr> </td></tr>
{%- endif %} {%- endif %}
<tr><th>Hardware</th><td><span title="{{ router.hardware.chipset }}">{{ router.hardware.name }}</span></td></tr> <tr><th>Hardware</th><td><span title="{{ router.chipset }}">{{ router.hardware }}</span></td></tr>
<tr><th>WAN Uplink</th><td><span class="{{ "glyphicon glyphicon-ok" if router.system.has_wan_uplink else "glyphicon glyphicon-remove" }}"></span></td></tr> <tr><th>WAN Uplink</th><td><span class="{{ "glyphicon glyphicon-ok" if router.wan_uplink else "glyphicon glyphicon-remove" }}"></span></td></tr>
<tr><th>Clients</th><td>{{ router.system.clients }}</td></tr> <tr><th>Clients</th><td>{{ router.clients }}</td></tr>
</table> </table>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="row" style="display: flex;"> <div class="row" style="display: flex;">
{%- if router.software is defined %}
<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;"> <div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading">Software</div> <div class="panel-heading">Software</div>
<div class="panel-body"> <div class="panel-body">
<table class="table table-condensed"> <table class="table table-condensed">
<tr><th>Firmware</th><td><span title="{{ router.software.firmware_rev }}">{{ router.software.firmware }}</span></td></tr> <tr><th>Firmware</th><td><span title="{{ router.firmware_rev }}">{{ router.firmware }}</span></td></tr>
<tr><th>Operating&nbsp;System</th><td>{{ router.software.os }}</td></tr> <tr><th>Operating&nbsp;System</th><td>{{ router.os }}</td></tr>
<tr><th>Kernel</th><td>{{ router.software.kernel }}</td></tr> <tr><th>Kernel</th><td>{{ router.kernel }}</td></tr>
<tr><th>B.A.T.M.A.N. adv</th><td>{{ router.software.batman_adv }}</td></tr> <tr><th>B.A.T.M.A.N. adv</th><td>{{ router.batman }}</td></tr>
<tr><th>Nodewatcher</th><td>{{ router.software.nodewatcher }}</td></tr> <tr><th>Nodewatcher</th><td>{{ router.nodewatcher }}</td></tr>
</table> </table>
</div> </div>
</div> </div>
{%- if not router.neighbours|length > 0 %} {%- if not router.neighbours|length > 0 %}
</div> </div>
<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;"> <div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
{%- endif %}
{%- else %}
<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
{%- endif %} {%- endif %}
<div class="panel panel-default" style="flex: 1 1 auto;"> <div class="panel panel-default" style="flex: 1 1 auto;">
<div class="panel-heading">Events</div> <div class="panel-heading">Events</div>
@ -195,7 +191,7 @@
</tr> </tr>
{%- for neighbour in router.neighbours %} {%- for neighbour in router.neighbours %}
<tr style="background-color: {{ neighbour.quality|neighbour_color }};"> <tr style="background-color: {{ neighbour.quality|neighbour_color }};">
<td><a href="{{ url_for('router_info', dbid=neighbour._id) }}">{{ neighbour.hostname }}</a></td> <td><a href="{{ url_for('router_info', dbid=neighbour.id) }}">{{ neighbour.hostname }}</a></td>
<td>{{ neighbour.mac }}</td> <td>{{ neighbour.mac }}</td>
<td>{{ neighbour.quality }}</td> <td>{{ neighbour.quality }}</td>
<td>{{ neighbour.net_if }}</td> <td>{{ neighbour.net_if }}</td>
@ -224,10 +220,10 @@
</div> </div>
<ul class="list-group" id="netif-list"> <ul class="list-group" id="netif-list">
{# make sure that br-mesh is on top of the list #} {# make sure that br-mesh is on top of the list #}
{%- for netif in router.netifs if netif.name == 'br-mesh' %} {%- for netif in router.netifs if netif.netif == 'br-mesh' %}
<li class="list-group-item active" data-name="{{ netif.name|replace('.', '')|replace('$', '') }}"> <li class="list-group-item active" data-name="{{ netif.netif|replace('.', '')|replace('$', '') }}">
<h4 class="list-group-item-heading"><div class="row"> <h4 class="list-group-item-heading"><div class="row">
<div class="col-xs-6 col-sm-6">{{ netif.name }}</div> <div class="col-xs-6 col-sm-6">{{ netif.netif }}</div>
<div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div> <div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div>
</div></h4> </div></h4>
<p class="list-group-item-text"><div class="row"> <p class="list-group-item-text"><div class="row">
@ -244,19 +240,19 @@
<br />{{ ipv6_addr }} <br />{{ ipv6_addr }}
{%- endfor -%} {%- endfor -%}
</div> </div>
{%- if netif.traffic.rx is defined %} {%- if netif.rx is defined %}
<div class="col-xs-7 col-sm-7 text-right"> <div class="col-xs-7 col-sm-7 text-right">
<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.traffic.rx|humanize_bytes }}/s <span class="glyphicon glyphicon-arrow-down"></span>{{ netif.rx|humanize_bytes }}/s
<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.traffic.tx|humanize_bytes }}/s <span class="glyphicon glyphicon-arrow-up"></span>{{ netif.tx|humanize_bytes }}/s
</div> </div>
{%- endif %} {%- endif %}
</div></p> </div></p>
</li> </li>
{%- endfor %} {%- endfor %}
{%- for netif in router.netifs if netif.name != 'br-mesh' %} {%- for netif in router.netifs if netif.netif != 'br-mesh' %}
<li class="list-group-item" data-name="{{ netif.name|replace('.', '')|replace('$', '') }}"> <li class="list-group-item" data-name="{{ netif.netif|replace('.', '')|replace('$', '') }}">
<h4 class="list-group-item-heading"><div class="row"> <h4 class="list-group-item-heading"><div class="row">
<div class="col-xs-6 col-sm-6">{{ netif.name }}</div> <div class="col-xs-6 col-sm-6">{{ netif.netif }}</div>
<div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div> <div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div>
</div></h4> </div></h4>
<p class="list-group-item-text"><div class="row"> <p class="list-group-item-text"><div class="row">
@ -273,10 +269,10 @@
<br />{{ ipv6_addr }} <br />{{ ipv6_addr }}
{%- endfor -%} {%- endfor -%}
</div> </div>
{%- if netif.traffic.rx is defined %} {%- if netif.rx is defined %}
<div class="col-xs-7 col-sm-7 text-right"> <div class="col-xs-7 col-sm-7 text-right">
<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.traffic.rx|humanize_bytes }}/s <span class="glyphicon glyphicon-arrow-down"></span>{{ netif.rx|humanize_bytes }}/s
<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.traffic.tx|humanize_bytes }}/s <span class="glyphicon glyphicon-arrow-up"></span>{{ netif.tx|humanize_bytes }}/s
</div> </div>
{%- endif %} {%- endif %}
</div></p> </div></p>

12
ffmap/web/templates/router_list.html
View File

@ -42,21 +42,21 @@
<tbody> <tbody>
{%- for router in routers %} {%- for router in routers %}
<tr> <tr>
<td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router._id) }}">{{ router.hostname }}</a></td> <td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router.id) }}">{{ router.hostname }}</a></td>
<td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td> <td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
<td>{{ router.hood }}</td> <td>{{ router.hood }}</td>
<td>{{ router.user.nickname if router.user }}</td> <td>{{ router.nickname if router.nickname else "" }}</td>
<td class="text-nowrap">{{ router.get("hardware", {}).get("name", "") }}</td> <td class="text-nowrap">{{ router.hardware }}</td>
<td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td> <td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td>
<td class="text-nowrap">{{ router.system.uptime|format_ts_diff }}</td> <td class="text-nowrap">{{ router.sys_uptime|format_ts_diff }}</td>
<td>{{ router.system.clients }}</td> <td>{{ router.clients }}</td>
</tr> </tr>
{%- endfor %} {%- endfor %}
</tbody> </tbody>
</table> </table>
</div> </div>
<div style="margin-bottom: 20px;"> <div style="margin-bottom: 20px;">
{{ routers.count() }} Router{{ "s" if (routers.count() == 1) else "" }} found. {{ numrouters }} Router{{ "s" if (numrouters == 1) else "" }} found.
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function() { $(document).ready(function() {

2
ffmap/web/templates/statistics.html
View File

@ -86,7 +86,7 @@
</tr> </tr>
{%- for router in newest_routers|reverse %} {%- for router in newest_routers|reverse %}
<tr> <tr>
<td><a href="{{ url_for('router_info', dbid=router._id) }}">{{ router.hostname }}</a></td> <td><a href="{{ url_for('router_info', dbid=router.id) }}">{{ router.hostname }}</a></td>
<td>{{ router.hood }}</td> <td>{{ router.hood }}</td>
<td class="text-nowrap">{{ router.created|utc2local|format_dt }}</td> <td class="text-nowrap">{{ router.created|utc2local|format_dt }}</td>
</tr> </tr>

17
ffmap/web/templates/user.html
View File

@ -57,12 +57,11 @@
<table class="table table-condensed"> <table class="table table-condensed">
<tr><th>Nickname</th><td> <tr><th>Nickname</th><td>
{{ user.nickname }} {{ user.nickname }}
(<a href="{{ url_for('router_list', q='user.nickname:%s' % user.nickname) }}">Filter</a>) (<a href="{{ url_for('router_list', q='nickname:%s' % user.nickname) }}">Filter</a>)
</td></tr> </td></tr>
{%- if user.email %} {%- if user.email %}
<tr><th>E-Mail</th><td> <tr><th>E-Mail</th><td>
{{ user.email|anon_email }} {{ user.email|anon_email }}
(<a href="{{ url_for('router_list', q='system.contact:%s' % user.email|anon_email_regex) }}">Filter</a>)
</td></tr> </td></tr>
{%- endif %} {%- endif %}
<tr><th>Created</th><td> <tr><th>Created</th><td>
@ -99,22 +98,22 @@
{%- set total_clients = 0 %} {%- set total_clients = 0 %}
{%- for router in routers %} {%- for router in routers %}
<tr> <tr>
<td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router._id) }}">{{ router.hostname }}</a></td> <td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router.id) }}">{{ router.hostname }}</a></td>
<td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td> <td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
<td>{{ router.hood }}</td> <td>{{ router.hood }}</td>
<td>{{ router.software.firmware }}</td> <td>{{ router.firmware }}</td>
<td class="text-nowrap">{{ router.get("hardware", {}).get("name", "") }}</td> <td class="text-nowrap">{{ router.get("hardware", "") }}</td>
<td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td> <td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td>
<td class="text-nowrap">{{ router.system.uptime|format_ts_diff }}</td> <td class="text-nowrap">{{ router.sys_uptime|format_ts_diff }}</td>
<td>{{ router.system.clients }}</td> <td>{{ router.clients }}</td>
{%- set total_clients = total_clients + router.system.clients %} {%- set total_clients = total_clients + router.clients %}
</tr> </tr>
{%- endfor %} {%- endfor %}
</tbody> </tbody>
</table> </table>
</div> </div>
<div style="margin-bottom: 20px;"> <div style="margin-bottom: 20px;">
{{ routers.count() }} Router{{ "s" if (routers.count() == 1) else "" }} found. {{ routers_count }} Router{{ "s" if (routers_count == 1) else "" }} found.
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function() { $(document).ready(function() {

6
ffmap/web/templates/user_list.html
View File

@ -41,15 +41,15 @@
<span class="glyphicon glyphicon-remove" title="Automatically imported stub from netmon"></span> <span class="glyphicon glyphicon-remove" title="Automatically imported stub from netmon"></span>
{%- endif -%} {%- endif -%}
</td> </td>
<td>{{ user_routers.get(user.nickname, {}).get('routers', 0) }}</td> <td>{{ user_routers.get(user.email, {}).get('routers', 0) }}</td>
<td>{{ user_routers.get(user.nickname, {}).get('clients', 0) }}</td> <td>{{ user_routers.get(user.email, {}).get('clients', 0) }}</td>
</tr> </tr>
{%- endfor %} {%- endfor %}
</tbody> </tbody>
</table> </table>
</div> </div>
<div style="margin-bottom: 20px;"> <div style="margin-bottom: 20px;">
{{ users.count() }} User{{ "s" if (users.count() > 1) else "" }} found. {{ users_count }} User{{ "s" if (users_count > 1) else "" }} found.
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function() { $(document).ready(function() {