Changelog: Update

Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

.gitignore

@@ -1,3 +1,4 @@
 build
 __pycache__
 .*.swp
+ffmap/mysqlconfig.py

README.md

@@ -1,22 +1,50 @@
+## Git Repository Logic
+* Frequent updates are made to the **testing** branch, which is considered "dirty". Commits appearing here may be quickly written, untested, incomplete, etc. This is where the development happens.
+* In unspecified intervals, the piled-up changes in the testing branch are reviewed, ordered and squashed to a smaller set of tidy commits. Those are then pushed to the **master** branch.
+* The tidy-up is marked by an empty commit "Realign with master" in the testing branch. This is roughly equivalent to a *merge*, although for an actual merge the commits would remain unaltered.
+* Development happens in the testing branch. Thus, *testing* is more up-to-date, but *master* is better to understand.
+* The Monitoring web server uses the testing branch.
+
+
+## Debian Dependencies
+```bash
+apt-get install mysql-server python3-mysqldb python python3 python3-requests python3-lxml python3-pip python3-flask python3-dateutil python3-numpy python3-scipy python3-mapnik python3-pip uwsgi-plugin-python3 nginx
+pip3 install wheel pymongo pillow modestmaps simplejson werkzeug
+```
+
+## When updating
+```bash
+apt-get install mysql-server python3-mysqldb python3-mapnik
+apt-get uninstall mongodb python-mapnik uwsgi-plugin-python tilestache
+pip3 install wheel pillow modestmaps simplejson werkzeug
+pip3 uninstall uuid
+```
+
+## Prerequisites
+* Datenbank in MySQL anlegen
+* Git vorbereiten:
+```bash
+git clone https://github.com/asdil12/fff-monitoring
+git clone https://github.com/TileStache/TileStache
+cd fff-monitoring
+cp ffmap/mysqlconfig.example.py ffmap/mysqlconfig.py
+```
+* MySQL Zugangsdaten in mysqlconfig.py eintragen
+
+
 ## Installation
 ```bash
 ./install.sh
 systemctl daemon-reload
-systemctl enable mongodb
 systemctl enable uwsgi-ffmap
 systemctl enable uwsgi-tiles
-systemctl start mongodb
 systemctl start uwsgi-ffmap
 systemctl start uwsgi-tiles
 cd ffmap/db/
 ./init_db.py
 # Then apply NGINX Config
-```
-
-## Debian Dependencies
-```bash
-apt-get install python python3 mongodb python3-requests python3-lxml python3-pip python3-flask python3-dateutil python3-numpy python3-scipy python-mapnik python3-pip uwsgi-plugin-python uwsgi-plugin-python3 nginx tilestache
-pip3 install pymongo
+cd ../.. # go back to fff-monitoring root directory
+./scripts/setupcron.sh
 ```
 
 ## NGINX Config
@@ -51,10 +79,4 @@ server {
 
 ## Admin anlegen
 * User über WebUI anlegen
-* Dann als root:
-```
-# mongo
-> use freifunk;
-> db.users.update({"nickname": "asdil12"}, {"$set": {"admin": true}});
-> exit
-```
+* Dann über z.B. phpmyadmin in der Tabelle users 'admin' auf 1 setzen

contrib/crawl.py

@@ -117,8 +117,8 @@ def crawl(router):
 					continue
 				neighbour = {
 					"mac": o_mac.lower(),
+					"netif": o_out_if,
 					"quality": int(o_link_quality),
-					"net_if": o_out_if,
 				}
 				try:
 					neighbour_router = db.routers.find_one({"netifs.mac": neighbour["mac"]})

ffmap/config.py

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+CONFIG = {
+	"vpn_netif": "fffVPN",				# Name of VPN interface
+	"vpn_netif_l2tp": "l2tp",			# Beginning of names of L2TP interfaces
+	"vpn_netif_aux": "fffauxVPN",		# Name of AUX interface
+	"offline_threshold_minutes": 15,	# Router switches to offline after X minutes
+	"orphan_threshold_days": 10,			# Router switches to orphaned state after X days
+	"delete_threshold_days": 180,		# Router is deleted after X days
+	"gw_netif_threshold_hours": 48,		# Hours which outdated netif from gwinfo is preserved for statistics
+	"router_stat_days": 30,				# Router stats are collected for X days
+	"router_stat_netif": 10,			# Router stats for netifs are collected for X days
+	"router_stat_gw": 1,				# Router stats for gw are collected for X days
+	"router_stat_mindiff_secs": 10,		# Time difference (uptime) in seconds required for a new entry in router stats
+	"router_stat_mindiff_default": 270,	# Time difference (router stats tables) in seconds required for a new entry in router stats
+	"router_stat_mindiff_netif": 270,	# Time difference (router netif stats) in seconds required for a new entry in router stats
+	"event_num_entries": 300,			# Number of events stored per router
+	"global_stat_days": 365,			# Global/hood stats are collected for X days
+	"csv_dir": "/var/lib/ffmap/csv",	# Directory where the .csv files for TileStache/mapnik are stored
+	"debug_dir": "/data/fff/fffmonlog",	# Output directory for debug .txt files
+}

ffmap/db/gws.py

@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+
+mysql = FreifunkMySQL()
+
+mysql.execute("""
+	CREATE TABLE `gw` (
+		`id` smallint(5) UNSIGNED NOT NULL,
+		`name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,
+		`stats_page` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`version` varchar(20) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`last_contact` datetime NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `gw`
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `gw`
+		MODIFY `id` smallint(5) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.execute("""
+	CREATE TABLE `gw_admin` (
+		`gw` smallint(5) UNSIGNED NOT NULL,
+		`name` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
+		`prio` tinyint(3) UNSIGNED NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `gw_admin`
+		ADD PRIMARY KEY (`gw`,`name`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `gw_netif` (
+		`gw` smallint(5) UNSIGNED NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+		`vpnmac` bigint(20) UNSIGNED DEFAULT NULL,
+		`ipv4` char(18) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`ipv6` varchar(60) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`dhcpstart` char(15) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`dhcpend` char(15) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`last_contact` datetime NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `gw_netif`
+		ADD PRIMARY KEY (`mac`),
+		ADD KEY `gw` (`gw`)
+""")
+
+mysql.commit()
+
+mysql.close()

ffmap/db/hoods.py

@@ -1,182 +1,95 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
-from pymongo import MongoClient
-client = MongoClient()
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 
-db = client.freifunk
+from ffmap.mysqltools import FreifunkMySQL
 
-# create db indexes
-db.hoods.delete_many({})
-db.hoods.create_index([("position", "2dsphere")])
+mysql = FreifunkMySQL()
 
-hoods = [
-{
-	"keyxchange_id": 1,
-	"name": "Default",
-	"net": "10.50.16.0/20"
-},
-{
-	"keyxchange_id": 2,
-	"name": "Fuerth",
-	"net": "10.50.32.0/21",
-	"position": {"type": "Point", "coordinates": [10.966, 49.4814]}
-},
-{
-	"keyxchange_id": 3,
-	"name": "Nuernberg",
-	"net": "10.50.40.0/21",
-	"position": {"type": "Point", "coordinates": [11.05, 49.444]}
-},
-{
-	"keyxchange_id": 4,
-	"name": "Ansbach",
-	"net": "10.50.48.0/21",
-	"position": {"type": "Point", "coordinates": [10.571667, 49.300833]}
-},
-{
-	"keyxchange_id": 5,
-	"name": "Hassberge",
-	"net": "10.50.56.0/21",
-	"position": {"type": "Point", "coordinates": [10.568013390003, 50.093555895082]}
-},
-{
-	"keyxchange_id": 6,
-	"name": "Erlangen",
-	"net": "10.50.64.0/21",
-	"position": {"type": "Point", "coordinates": [11.0019221, 49.6005981]}
-},
-{
-	"keyxchange_id": 7,
-	"name": "Wuerzburg",
-	"net": "10.50.72.0/21",
-	"position": {"type": "Point", "coordinates": [9.93489, 49.79688]}
-},
-{
-	"keyxchange_id": 8,
-	"name": "Bamberg",
-	"net": "10.50.124.0/22",
-	"position": {"type": "Point", "coordinates": [10.95, 49.89]}
-},
-{
-	"keyxchange_id": 9,
-	"name": "BGL",
-	"net": "10.50.80.0/21",
-	"position": {"type": "Point", "coordinates": [12.8825, 47.7314]}
-},
-{
-	"keyxchange_id": 10,
-	"name": "HassbergeSued",
-	"net": "10.50.60.0/22",
-	"position": {"type": "Point", "coordinates": [10.568013390003, 50.04501]}
-},
-{
-	"keyxchange_id": 11,
-	"name": "NbgLand",
-	"net": "10.50.88.0/21",
-	"position": {"type": "Point", "coordinates": [11.162796020507812, 49.39200496388418]}
-},
-{
-	"keyxchange_id": 12,
-	"name": "Hof",
-	"net": "10.50.104.0/21",
-	"position": {"type": "Point", "coordinates": [11.917545, 50.312209]}
-},
-{
-	"keyxchange_id": 13,
-	"name": "Aschaffenburg",
-	"net": "10.50.96.0/22",
-	"position": {"type": "Point", "coordinates": [9.146826, 49.975661]}
-},
-{
-	"keyxchange_id": 14,
-	"name": "Marktredwitz",
-	"net": "10.50.112.0/22",
-	"position": {"type": "Point", "coordinates": [12.084797, 50.002915]}
-},
-{
-	"keyxchange_id": 15,
-	"name": "Forchheim",
-	"net": "10.50.116.0/22",
-	"position": {"type": "Point", "coordinates": [11.059474, 49.718820]}
-},
-{
-	"keyxchange_id": 16,
-	"name": "Muenchberg",
-	"net": "10.50.120.0/22",
-	"position": {"type": "Point", "coordinates": [11.79, 50.19]}
-},
-{
-	"keyxchange_id": 17,
-	"name": "Adelsdorf",
-	"net": "10.50.144.0/22",
-	"position": {"type": "Point", "coordinates": [10.894235, 49.709945]}
-},
-{
-	"keyxchange_id": 18,
-	"name": "Schweinfurt",
-	"net": "10.50.160.0/22",
-	"position": {"type": "Point", "coordinates": [10.21267, 50.04683]}
-},
-{
-	"keyxchange_id": 19,
-	"name": "ErlangenWest",
-	"net": "10.50.152.0/22",
-	"position": {"type": "Point", "coordinates": [10.984488, 49.6035981]}
-},
-{
-	"keyxchange_id": 20,
-	"name": "Ebermannstadt",
-	"net": "10.50.148.0/22",
-	"position": {"type": "Point", "coordinates": [11.18538, 49.78173]}
-},
-{
-	"keyxchange_id": 21,
-	"name": "Lauf",
-	"net": "10.50.156.0/22",
-	"position": {"type": "Point", "coordinates": [11.278789, 49.509972]}
-},
-{
-	"keyxchange_id": 22,
-	"name": "Bayreuth",
-	"net": "10.50.168.0/22",
-	"position": {"type": "Point", "coordinates": [11.580566, 49.94814]}
-},
-{
-	"keyxchange_id": 23,
-	"name": "Fichtelberg",
-	"net": "10.50.172.0/22",
-	"position": {"type": "Point", "coordinates": [11.852292, 49.998920]}
-},
-{
-	"keyxchange_id": 24,
-	"name": "Rehau",
-	"net": "10.50.176.0/22",
-	"position": {"type": "Point", "coordinates": [12.035305, 50.247594]}
-},
-	{
-	"keyxchange_id": 25,
-	"name": "Coburg",
-	"net": "10.50.180.0/22",
-	"position": {"type": "Point", "coordinates": [10.964414, 50.259675]}
-},
-{
-	"keyxchange_id": 26,
-	"name": "Ebern",
-	"net": "10.50.184.0/22",
-	"position": {"type": "Point", "coordinates": [10.798395, 50.095572]}
-},
-{
-	"keyxchange_id": 27,
-	"name": "Arnstein",
-	"net": "10.50.188.0/22",
-	"position": {"type": "Point", "coordinates": [9.970957, 49.978117]}
-},
-{
-	"keyxchange_id": 28,
-	"name": "Erlenbach",
-	"net": "10.50.192.0/22",
-	"position": {"type": "Point", "coordinates": [9.157491, 49.803930]}
-}]
+mysql.execute("""
+	CREATE TABLE `hoods` (
+		`id` smallint(6) UNSIGNED NOT NULL,
+		`name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
 
-for hood in hoods:
-	db.hoods.insert_one(hood)
+mysql.execute("""
+	ALTER TABLE `hoods`
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `hoods`
+		MODIFY `id` smallint(6) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.execute("""
+	ALTER TABLE hoods AUTO_INCREMENT = 30001
+""")
+
+mysql.execute("""
+	INSERT INTO hoods (id, name)
+	VALUES (%s, %s)
+""",(10100,Legacy,))
+
+mysql.execute("""
+	CREATE TABLE `hoodsv2` (
+		`id` int(10) UNSIGNED NOT NULL,
+		`name` varchar(30) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+		`net` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+		`lat` double DEFAULT NULL,
+		`lng` double DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `hoodsv2`
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`),
+		ADD KEY `lat` (`lat`),
+		ADD KEY `lng` (`lng`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `polygons` (
+		`id` int(10) UNSIGNED NOT NULL,
+		`polyid` int(10) UNSIGNED NOT NULL,
+		`lat` double NOT NULL,
+		`lon` double NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `polygons`
+		ADD PRIMARY KEY (`id`),
+		ADD KEY `polyid` (`polyid`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `polygons`
+		MODIFY `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.execute("""
+	CREATE TABLE `polyhoods` (
+		`polyid` int(10) UNSIGNED NOT NULL,
+		`hoodid` int(10) UNSIGNED NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `polyhoods`
+		ADD PRIMARY KEY (`polyid`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `polyhoods`
+		MODIFY `polyid` int(10) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.commit()
+
+mysql.close()

ffmap/db/init_db.py

@@ -3,3 +3,5 @@
 import routers
 import hoods
 import stats
+import gws
+import users

ffmap/db/routers.py

@@ -1,14 +1,286 @@
 #!/usr/bin/python3
 
-from pymongo import MongoClient
-client = MongoClient()
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 
-db = client.freifunk
+from ffmap.mysqltools import FreifunkMySQL
 
-# create db indexes
-db.routers.create_index("hostname")
-db.routers.create_index("status")
-db.routers.create_index("created")
-db.routers.create_index("last_contact")
-db.routers.create_index("netifs.mac")
-db.routers.create_index([("position", "2dsphere")])
+mysql = FreifunkMySQL()
+
+mysql.execute("""
+	CREATE TABLE `banned` (
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`added` datetime NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `banned`
+		ADD PRIMARY KEY (`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `blocked` (
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`added` datetime NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `blocked`
+		ADD PRIMARY KEY (`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `netifs` (
+		`id` smallint(6) UNSIGNED NOT NULL,
+		`name` varchar(15) COLLATE utf8_unicode_ci NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `netifs`
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `name` (`name`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `netifs`
+		MODIFY `id` smallint(6) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.execute("""
+	CREATE TABLE `router` (
+		`id` mediumint(8) UNSIGNED NOT NULL,
+		`status` varchar(20) COLLATE utf8_unicode_ci NOT NULL,
+		`hostname` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`created` datetime NOT NULL,
+		`last_contact` datetime NOT NULL,
+		`sys_time` datetime NOT NULL,
+		`sys_uptime` int(11) NOT NULL,
+		`sys_memfree` int(11) NOT NULL,
+		`sys_membuff` int(11) NOT NULL,
+		`sys_memcache` int(11) NOT NULL,
+		`sys_loadavg` float NOT NULL,
+		`sys_procrun` smallint(6) NOT NULL,
+		`sys_proctot` smallint(6) NOT NULL,
+		`clients` smallint(6) NOT NULL,
+		`clients_eth` smallint(6) DEFAULT NULL,
+		`clients_w2` smallint(6) DEFAULT NULL,
+		`clients_w5` smallint(6) DEFAULT NULL,
+		`w2_busy` bigint(20) UNSIGNED DEFAULT NULL,
+		`w2_active` bigint(20) UNSIGNED DEFAULT NULL,
+		`w5_busy` bigint(20) UNSIGNED DEFAULT NULL,
+		`w5_active` bigint(20) UNSIGNED DEFAULT NULL,
+		`w2_airtime` float DEFAULT NULL,
+		`w5_airtime` float DEFAULT NULL,
+		`wan_uplink` tinyint(1) NOT NULL,
+		`tc_enabled` tinyint(1) DEFAULT NULL,
+		`tc_in` float DEFAULT NULL,
+		`tc_out` float DEFAULT NULL,
+		`cpu` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`chipset` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`hardware` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`os` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`batman` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`routing_protocol` varchar(40) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`kernel` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`nodewatcher` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`firmware` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`firmware_rev` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`description` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`position_comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`community` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`hood` smallint(5) UNSIGNED DEFAULT NULL,
+		`v2` tinyint(1) NOT NULL,
+		`local` tinyint(1) NOT NULL,
+		`gateway` tinyint(1) NOT NULL,
+		`status_text` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`contact` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`lng` double DEFAULT NULL,
+		`lat` double DEFAULT NULL,
+		`reset` tinyint(1) NOT NULL DEFAULT '0',
+		`neighbors` smallint(6) NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router`
+		ADD PRIMARY KEY (`id`),
+		ADD KEY `created` (`created`),
+		ADD KEY `hostname` (`hostname`),
+		ADD KEY `status` (`status`),
+		ADD KEY `last_contact` (`last_contact`),
+		ADD KEY `lat` (`lat`),
+		ADD KEY `lng` (`lng`),
+		ADD KEY `contact` (`contact`),
+		ADD KEY `hood` (`hood`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `router`
+		MODIFY `id` mediumint(8) UNSIGNED NOT NULL AUTO_INCREMENT
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_events` (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`time` datetime NOT NULL,
+		`type` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
+		`comment` varchar(200) COLLATE utf8_unicode_ci NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_events`
+		ADD PRIMARY KEY (`router`,`time`,`type`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_gw` (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`quality` float NOT NULL,
+		`nexthop` bigint(20) UNSIGNED DEFAULT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`gw_class` varchar(30) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`selected` tinyint(1) NOT NULL DEFAULT '0'
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_gw`
+		ADD PRIMARY KEY (`router`,`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_ipv6` (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+		`ipv6` binary(16) NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_ipv6`
+		ADD PRIMARY KEY (`router`,`netif`,`ipv6`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_neighbor` (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+		`quality` float NOT NULL,
+		`type` varchar(10) COLLATE utf8_unicode_ci DEFAULT 'l2'
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_neighbor`
+		ADD PRIMARY KEY (`router`,`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_netif` (
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`netif` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+		`mtu` smallint(6) NOT NULL,
+		`rx_bytes` bigint(20) UNSIGNED NOT NULL,
+		`tx_bytes` bigint(20) UNSIGNED NOT NULL,
+		`rx` int(10) UNSIGNED NOT NULL,
+		`tx` int(10) UNSIGNED NOT NULL,
+		`fe80_addr` binary(16) DEFAULT NULL,
+		`ipv4_addr` int(10) UNSIGNED DEFAULT NULL,
+		`mac` bigint(20) UNSIGNED DEFAULT NULL,
+		`wlan_channel` tinyint(3) UNSIGNED DEFAULT NULL,
+		`wlan_type` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`wlan_width` tinyint(3) UNSIGNED DEFAULT NULL,
+		`wlan_ssid` varchar(32) COLLATE utf8_unicode_ci DEFAULT NULL,
+		`wlan_txpower` varchar(8) COLLATE utf8_unicode_ci DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_netif`
+		ADD PRIMARY KEY (`router`,`netif`),
+		ADD KEY `mac` (`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_stats` (
+		`time` int(11) NOT NULL,
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`sys_proctot` smallint(6) NOT NULL,
+		`sys_procrun` smallint(6) NOT NULL,
+		`sys_memcache` int(11) NOT NULL,
+		`sys_membuff` int(11) NOT NULL,
+		`sys_memfree` int(11) NOT NULL,
+		`loadavg` float NOT NULL,
+		`clients` smallint(6) NOT NULL,
+		`clients_eth` smallint(6) DEFAULT NULL,
+		`clients_w2` smallint(6) DEFAULT NULL,
+		`clients_w5` smallint(6) DEFAULT NULL,
+		`airtime_w2` float DEFAULT NULL,
+		`airtime_w5` float DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_stats`
+		ADD PRIMARY KEY (`time`,`router`),
+		ADD KEY `router` (`router`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_stats_gw` (
+		`time` int(11) NOT NULL,
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`quality` float NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_stats_gw`
+		ADD PRIMARY KEY (`time`,`router`,`mac`),
+		ADD KEY `router` (`router`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_stats_neighbor` (
+		`time` int(11) NOT NULL,
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`quality` float NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_stats_neighbor`
+		ADD PRIMARY KEY (`time`,`router`,`mac`),
+		ADD KEY `router` (`router`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `router_stats_netif` (
+		`time` int(11) NOT NULL,
+		`router` mediumint(8) UNSIGNED NOT NULL,
+		`netif` smallint(6) UNSIGNED NOT NULL,
+		`rx` int(10) UNSIGNED NOT NULL,
+		`tx` int(10) UNSIGNED NOT NULL,
+		`deletebit` tinyint(1) NOT NULL DEFAULT '0'
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `router_stats_netif`
+		ADD PRIMARY KEY (`time`,`router`,`netif`),
+		ADD KEY `router` (`router`),
+		ADD KEY `deletebit` (`deletebit`)
+""")
+
+mysql.commit()
+
+mysql.close()

ffmap/db/stats.py

@@ -1,9 +1,69 @@
 #!/usr/bin/python3
 
-from pymongo import MongoClient
-client = MongoClient()
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 
-db = client.freifunk
+from ffmap.mysqltools import FreifunkMySQL
 
-# create capped collection
-db.create_collection("stats", capped=True, size=10*1024*1024, max=4320)
+mysql = FreifunkMySQL()
+
+mysql.execute("""
+	CREATE TABLE `stats_global` (
+		`time` int(11) NOT NULL,
+		`clients` mediumint(9) NOT NULL,
+		`online` smallint(6) NOT NULL,
+		`offline` smallint(6) NOT NULL,
+		`unknown` smallint(6) NOT NULL,
+		`orphaned` smallint(6) NOT NULL,
+		`rx` int(10) UNSIGNED DEFAULT NULL,
+		`tx` int(10) UNSIGNED DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `stats_global`
+		ADD PRIMARY KEY (`time`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `stats_gw` (
+		`time` int(11) NOT NULL,
+		`mac` bigint(20) UNSIGNED NOT NULL,
+		`clients` mediumint(9) NOT NULL,
+		`online` smallint(6) NOT NULL,
+		`offline` smallint(6) NOT NULL,
+		`unknown` smallint(6) NOT NULL,
+		`orphaned` smallint(6) NOT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `stats_gw`
+		ADD PRIMARY KEY (`time`,`mac`),
+		ADD KEY `mac` (`mac`)
+""")
+
+mysql.execute("""
+	CREATE TABLE `stats_hood` (
+		`time` int(11) NOT NULL,
+		`hood` smallint(5) UNSIGNED NOT NULL,
+		`clients` mediumint(9) NOT NULL,
+		`online` smallint(6) NOT NULL,
+		`offline` smallint(6) NOT NULL,
+		`unknown` smallint(6) NOT NULL,
+		`orphaned` smallint(6) NOT NULL,
+		`rx` int(10) UNSIGNED DEFAULT NULL,
+		`tx` int(10) UNSIGNED DEFAULT NULL
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `stats_hood`
+		ADD PRIMARY KEY (`time`,`hood`),
+		ADD KEY `hood` (`hood`)
+""")
+
+mysql.commit()
+
+mysql.close()

ffmap/db/users.py

@@ -1,10 +1,38 @@
 #!/usr/bin/python3
 
-from pymongo import MongoClient
-client = MongoClient()
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 
-db = client.freifunk
+from ffmap.mysqltools import FreifunkMySQL
 
-# create db indexes
-db.users.create_index("email")
-db.users.create_index("nickname")
+mysql = FreifunkMySQL()
+
+mysql.execute("""
+	CREATE TABLE `users` (
+		`id` int(11) NOT NULL,
+		`nickname` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+		`password` varchar(250) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
+		`token` varchar(250) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
+		`email` varchar(200) COLLATE utf8_unicode_ci NOT NULL,
+		`created` datetime NOT NULL,
+		`admin` tinyint(1) NOT NULL DEFAULT '0',
+		`abuse` tinyint(1) NOT NULL DEFAULT '0'
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+""")
+
+mysql.execute("""
+	ALTER TABLE `users`
+		ADD PRIMARY KEY (`id`),
+		ADD UNIQUE KEY `nickname` (`nickname`),
+		ADD UNIQUE KEY `email` (`email`)
+""")
+
+mysql.execute("""
+	ALTER TABLE `users`
+		MODIFY `id` int(11) NOT NULL AUTO_INCREMENT
+""")
+
+mysql.commit()
+
+mysql.close()

ffmap/dbtools.py

@@ -1,15 +0,0 @@
-#!/usr/bin/python3
-
-from pymongo import MongoClient
-
-class FreifunkDB(object):
-	client = None
-	db = None
-
-	@classmethod
-	def handle(cls):
-		if not cls.client:
-			cls.client = MongoClient(tz_aware=True, connect=False)
-		if not cls.db:
-			cls.db = cls.client.freifunk
-		return cls.db

ffmap/gwtools.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+from ffmap.misc import *
+from ffmap.config import CONFIG
+from flask import request, url_for
+
+import datetime
+import time
+
+def import_gw_data(mysql, gw_data):
+	if "hostname" in gw_data and "netifs" in gw_data:
+		time = utcnow().strftime('%Y-%m-%d %H:%M:%S')
+		stats_page = gw_data.get("stats_page","")
+		version = gw_data.get("version","")
+
+		# Make None if empty (gw_data.get() only checks for existing key)
+		if not stats_page:
+			stats_page = None
+		if not version:
+			version = None
+		newid = mysql.findone("SELECT id FROM gw WHERE name = %s LIMIT 1",(gw_data["hostname"],),"id")
+		if newid:
+			mysql.execute("""
+				UPDATE gw
+				SET stats_page = %s, version = %s, last_contact = %s
+				WHERE id = %s
+			""",(stats_page,version,time,newid,))
+			mysql.execute("""
+				UPDATE gw_netif
+				SET ipv4 = NULL, ipv6 = NULL, dhcpstart = NULL, dhcpend = NULL
+				WHERE gw = %s
+			""",(newid,))
+		else:
+			mysql.execute("""
+				INSERT INTO gw (name, stats_page, version, last_contact)
+				VALUES (%s, %s, %s, %s)
+			""",(gw_data["hostname"],stats_page,version,time,))
+			newid = mysql.cursor().lastrowid
+		
+		nmacs = {}
+		for n in gw_data["netifs"]:
+			nmacs[n["netif"]] = n["mac"]
+		
+		ndata = []
+		for n in gw_data["netifs"]:
+			if len(n["mac"])<17 or len(n["mac"])>17:
+				continue
+			if n["netif"].startswith("l2tp"): # Filter l2tp interfaces
+				continue
+			if "vpnif" in n and n["vpnif"]:
+				n["vpnmac"] = nmacs.get(n["vpnif"],None)
+			else:
+				n["vpnmac"] = None
+			if not "ipv4" in n or not n["ipv4"]:
+				n["ipv4"] = None
+			if not "ipv6" in n or not n["ipv6"]:
+				n["ipv6"] = None
+			if not "dhcpstart" in n or not n["dhcpstart"]:
+				n["dhcpstart"] = None
+			if not "dhcpend" in n or not n["dhcpend"]:
+				n["dhcpend"] = None
+			
+			ndata.append((newid,mac2int(n["mac"]),n["netif"],mac2int(n["vpnmac"]),n["ipv4"],n["ipv6"],n["dhcpstart"],n["dhcpend"],time,))
+		
+		mysql.executemany("""
+			INSERT INTO gw_netif (gw, mac, netif, vpnmac, ipv4, ipv6, dhcpstart, dhcpend, last_contact)
+			VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+			ON DUPLICATE KEY UPDATE
+				gw=VALUES(gw),
+				netif=VALUES(netif),
+				vpnmac=VALUES(vpnmac),
+				ipv4=VALUES(ipv4),
+				ipv6=VALUES(ipv6),
+				dhcpstart=VALUES(dhcpstart),
+				dhcpend=VALUES(dhcpend),
+				last_contact=VALUES(last_contact)
+		""",ndata)
+
+		adata = []
+		aid = 0
+		for a in gw_data["admins"]:
+			aid += 1
+			adata.append((newid,a,aid,))
+		
+		mysql.execute("DELETE FROM gw_admin WHERE gw = %s",(newid,))
+		mysql.executemany("""
+			INSERT INTO gw_admin (gw, name, prio)
+			VALUES (%s, %s, %s)
+			ON DUPLICATE KEY UPDATE
+				prio=VALUES(prio)
+		""",adata)
+	else:
+		writelog(CONFIG["debug_dir"] + "/fail_gwinfo.txt", "{} - Corrupted file.".format(request.environ['REMOTE_ADDR']))
+
+def gw_name(gw):
+	if gw["gw"] and gw["gwif"]:
+		s = gw["gw"] + " (" + gw["gwif"] + ")"
+	else:
+		s = int2mac(gw["mac"])
+	return s
+
+def gw_bat(gw):
+	if gw["batif"] and gw["batmac"]:
+		s = int2mac(gw["batmac"]) + " (" + gw["batif"] + ")"
+	else:
+		s = "---"
+	return s
+
+def delete_unlinked_gws(mysql):
+	# Delete entries in gw_* tables without corresponding gw in master table
+	
+	tables = ["gw_admin","gw_netif"]
+	
+	for t in tables:
+		start_time = time.time()
+		mysql.execute("""
+			DELETE d FROM {} AS d
+			LEFT JOIN gw AS g ON g.id = d.gw
+			WHERE g.id IS NULL
+		""".format(t))
+		print("--- Deleted %i rows from %s: %.3f seconds ---" % (mysql.cursor().rowcount,t,time.time() - start_time))
+	mysql.commit()
+

ffmap/hoodtools.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+
+import urllib.request, urllib.error, json
+import math
+
+def update_hoods_v2(mysql):
+	try:
+		with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
+			hoodskx = json.loads(url.read().decode())
+		
+		kx_keys = []
+		kx_data = []
+		for kx in hoodskx:
+			kx_keys.append(kx["id"])
+			kx_data.append((kx["id"],kx["name"],kx["net"],kx.get("lat",None),kx.get("lon",None),))
+
+		# Delete entries in DB where hood is missing in KeyXchange
+		db_keys = mysql.fetchall("SELECT id FROM hoodsv2",(),"id")
+		for n in db_keys:
+			if n in kx_keys:
+				continue
+			mysql.execute("DELETE FROM hoodsv2 WHERE id = %s",(n,))
+
+		# Create/update entries from KeyXchange to DB
+		mysql.executemany("""
+			INSERT INTO hoodsv2 (id, name, net, lat, lng)
+			VALUES (%s, %s, %s, %s, %s)
+			ON DUPLICATE KEY UPDATE
+				name=VALUES(name),
+				net=VALUES(net),
+				lat=VALUES(lat),
+				lng=VALUES(lng)
+		""",kx_data)
+
+	except urllib.error.HTTPError as e:
+		return
+
+def update_hoods_poly(mysql):
+	try:
+		#with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
+		with urllib.request.urlopen("https://lauch.org/keyxchange/hoods.php") as url:
+			hoodskx = json.loads(url.read().decode())
+
+		mysql.execute("DELETE FROM polygons",())
+		mysql.execute("DELETE FROM polyhoods",())
+
+		for kx in hoodskx:
+			for polygon in kx.get("polygons",()):
+				mysql.execute("""
+					INSERT INTO polyhoods (hoodid)
+					VALUES (%s)
+				""",(kx["id"],))
+				newid = mysql.cursor().lastrowid
+				vertices = []
+				for p in polygon:
+					vertices.append((newid,p["lat"],p["lon"],))
+				mysql.executemany("""
+					INSERT INTO polygons (polyid, lat, lon)
+					VALUES (%s, %s, %s)
+				""",vertices)
+
+	except urllib.error.HTTPError as e:
+		return

ffmap/mapnik/dynmapnik.py

@@ -1,4 +1,7 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
+
+import sys
+sys.path.insert(0,'/data/fff/TileStache')
 
 import os
 import logging

ffmap/mapnik/hoods_poly.xml

@@ -3,26 +3,26 @@
 <Map background-color="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over">
   <Style name="hoodpoint">
 	<Rule>
-	  <TextSymbolizer face-name="DejaVu Sans Book" size="12" fill="#1e42ff" halo-radius="2" text-transform="capitalize">[name]</TextSymbolizer>
+	  <TextSymbolizer face-name="DejaVu Sans Book" size="12" fill="#b200b2" halo-radius="2">[name]</TextSymbolizer>
 	</Rule>
   </Style>
   <Style name="hoodborder">
 	<Rule>
-	  <LineSymbolizer stroke-width="3" stroke="#1e42ff" stroke-linecap="butt" stroke-dasharray="6, 2" clip="false" />
+	  <LineSymbolizer stroke-width="3" stroke="#b200b2" stroke-linecap="butt" stroke-dasharray="6, 2" clip="false" />
 	</Rule>
   </Style>
   <Layer name="borders" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
     <StyleName>hoodborder</StyleName>
     <Datasource>
       <Parameter name="type">csv</Parameter>
-      <Parameter name="file">csv/hoods.csv</Parameter>
+      <Parameter name="file">csv/hoods_poly.csv</Parameter>
     </Datasource>
   </Layer>
   <Layer name="points" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
     <StyleName>hoodpoint</StyleName>
     <Datasource>
       <Parameter name="type">csv</Parameter>
-      <Parameter name="file">csv/hood-points.csv</Parameter>
+      <Parameter name="file">csv/hood-points-poly.csv</Parameter>
     </Datasource>
   </Layer>
 </Map>

ffmap/mapnik/hoods_v2.xml

@@ -3,7 +3,7 @@
 <Map background-color="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over">
   <Style name="hoodpoint">
 	<Rule>
-	  <TextSymbolizer face-name="DejaVu Sans Book" size="12" fill="#1e42ff" halo-radius="2" text-transform="capitalize">[name]</TextSymbolizer>
+	  <TextSymbolizer face-name="DejaVu Sans Book" size="12" fill="#1e42ff" halo-radius="2">[name]</TextSymbolizer>
 	</Rule>
   </Style>
   <Style name="hoodborder">
@@ -15,7 +15,7 @@
     <StyleName>hoodborder</StyleName>
     <Datasource>
       <Parameter name="type">csv</Parameter>
-      <Parameter name="file">csv/hoodsv2.csv</Parameter>
+      <Parameter name="file">csv/hoods_v2.csv</Parameter>
     </Datasource>
   </Layer>
   <Layer name="points" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">

ffmap/mapnik/mkcsv.py

@@ -1,131 +0,0 @@
-#!/usr/bin/python3
-
-import math
-import numpy as np
-from scipy.spatial import Voronoi
-
-import urllib.request, json
-
-from pymongo import MongoClient
-client = MongoClient()
-
-db = client.freifunk
-
-EARTH_RADIUS = 6378137.0
-
-def merc_sphere(lng, lat):
-	x = math.radians(lng) * EARTH_RADIUS
-	y = math.log(math.tan(math.pi/4 + math.radians(lat)/2)) * EARTH_RADIUS
-	return (x,y)
-
-def merc_sphere_inv(x, y):
-	lng = math.degrees(x / EARTH_RADIUS)
-	lat = math.degrees(2*math.atan(math.exp(y / EARTH_RADIUS)) - math.pi/2)
-	return (lng,lat)
-
-def draw_voronoi_lines(csv, hoods):
-	points = np.array(hoods)
-	vor = Voronoi(points)
-	#mp = voronoi_plot_2d(vor)
-	#mp.show()
-
-	lines = [vor.vertices[line] for line in vor.ridge_vertices if -1 not in line]
-
-	for line in lines:
-		x = [line[0][0], line[1][0]]
-		y = [line[0][1], line[1][1]]
-		for i in range(len(x)-1):
-			# convert mercator coordinates back into lng/lat
-			lng1, lat1 = merc_sphere_inv(x[i], y[i])
-			lng2, lat2 = merc_sphere_inv(x[i+1], y[i+1])
-			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2))
-
-	ptp_bound = np.array(merc_sphere(180, 360))
-	center = vor.points.mean(axis=0)
-
-	for pointidx, simplex in zip(vor.ridge_points, vor.ridge_vertices):
-		simplex = np.asarray(simplex)
-		if np.any(simplex < 0):
-			i = simplex[simplex >= 0][0]  # finite end Voronoi vertex
-
-			t = vor.points[pointidx[1]] - vor.points[pointidx[0]]  # tangent
-			t /= np.linalg.norm(t)
-			n = np.array([-t[1], t[0]])  # normal
-
-			midpoint = vor.points[pointidx].mean(axis=0)
-			direction = np.sign(np.dot(midpoint - center, n)) * n
-			far_point = vor.vertices[i] + direction * ptp_bound.max()
-
-			# convert mercator coordinates back into lng/lat
-			lng1, lat1 = merc_sphere_inv(vor.vertices[i,0], vor.vertices[i,1])
-			lng2, lat2 = merc_sphere_inv(far_point[0], far_point[1])
-			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2))
-
-
-with open("csv/routers.csv", "w") as csv:
-	csv.write("lng,lat,status\n")
-	for router in db.routers.find({"position.coordinates": {"$exists": True}}):
-		csv.write("%f,%f,%s\n" % (
-			router["position"]["coordinates"][0],
-			router["position"]["coordinates"][1],
-			router["status"]
-		))
-
-with open("csv/links.csv", "w") as csv:
-	csv.write("WKT,quality\n")
-	for router in db.routers.find({"position.coordinates": {"$exists": True}, "neighbours": {"$exists": True}}):
-		for neighbour in router["neighbours"]:
-			if "position" in neighbour:
-				csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % (
-					router["position"]["coordinates"][0],
-					router["position"]["coordinates"][1],
-					neighbour["position"]["coordinates"][0],
-					neighbour["position"]["coordinates"][1],
-					neighbour["quality"]
-				))
-
-with open("csv/hood-points.csv", "w", encoding="UTF-8") as csv:
-	csv.write("lng,lat,name\n")
-	for hood in db.hoods.find({"position": {"$exists": True}}):
-		csv.write("%f,%f,\"%s\"\n" % (
-			hood["position"]["coordinates"][0],
-			hood["position"]["coordinates"][1],
-			hood["name"]
-		))
-
-with open("csv/hoods.csv", "w") as csv:
-	csv.write("WKT\n")
-	hoods = []
-	for hood in db.hoods.find({"position": {"$exists": True}}):
-		# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
-		x, y = merc_sphere(hood["position"]["coordinates"][0], hood["position"]["coordinates"][1])
-		hoods.append([x, y])
-	draw_voronoi_lines(csv, hoods)
-
-with open("csv/hood-points-v2.csv", "w", encoding="UTF-8") as csv:
-	csv.write("lng,lat,name\n")
-	with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
-		data = json.loads(url.read().decode())
-	for hood in data:
-		if not ( 'lon' in hood and 'lat' in hood ):
-			continue
-		csv.write("%f,%f,\"%s\"\n" % (
-			hood["lon"],
-			hood["lat"],
-			hood["name"]
-		))
-
-with open("csv/hoodsv2.csv", "w") as csv:
-	csv.write("WKT\n")
-	hoods = []
-	with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
-		data = json.loads(url.read().decode())
-	
-	for hood in data:
-		if not ( 'lon' in hood and 'lat' in hood ):
-			continue
-		# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
-		x, y = merc_sphere(hood["lon"], hood["lat"])
-		hoods.append([x, y])
-
-	draw_voronoi_lines(csv, hoods)

ffmap/mapnik/routers.xml

@@ -17,8 +17,32 @@
 	  <Filter>([status] = 'unknown')</Filter>
 	  <PointSymbolizer file="static/img/router_yellow.svg" allow-overlap="true" />
 	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned')</Filter>
+	  <PointSymbolizer file="static/img/router_grey.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'online_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_green_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_red_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_yellow_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_grey_white.svg" allow-overlap="true" />
+	</Rule>
   </Style>
   <Style name="color" filter-mode="first">
+	<Rule>
+	  <Filter>([quality] &lt; 1)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#008c00" stroke-linecap="butt" clip="false" />
+	</Rule>
 	<Rule>
 	  <Filter>([quality] &lt; 105)</Filter>
 	  <LineSymbolizer stroke-width="3" stroke="#ff1e1e" stroke-linecap="butt" clip="false" />

ffmap/mapnik/routers_local.xml

@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE Map>
+<Map background-color="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over">
+  <Style name="routerpoint" filter-mode="first">
+	<Rule>
+	  <Filter>([status] = 'online')</Filter>
+	  <!-- For directed antenna
+		<PointSymbolizer file="static/img/router_direct_green.svg" allow-overlap="true" transform="rotate(45)" />
+	  -->
+	  <PointSymbolizer file="static/img/router_green_v2.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline')</Filter>
+	  <PointSymbolizer file="static/img/router_red_v2.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown')</Filter>
+	  <PointSymbolizer file="static/img/router_yellow.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned')</Filter>
+	  <PointSymbolizer file="static/img/router_grey.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'online_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_green_v2_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_red_v2_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_yellow_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_grey_white.svg" allow-overlap="true" />
+	</Rule>
+  </Style>
+  <Style name="color" filter-mode="first">
+	<Rule>
+	  <Filter>([quality] &lt; 1)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#008c00" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 105)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff1e1e" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 130)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff4949" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 155)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff6a6a" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 180)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ffac53" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 205)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ffeb79" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 230)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#79ff7c" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 300)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#04ff0a" stroke-linecap="butt" clip="false" />
+	</Rule>
+  </Style>
+  <Style name="l3_color" filter-mode="first">
+	<Rule>
+	  <LineSymbolizer stroke-width="3" stroke="#0684c4" stroke-linecap="butt" clip="false" />
+	</Rule>
+  </Style>
+  <Style name="shadow1">
+	<Rule>
+	  <LineSymbolizer stroke-width="4" stroke="#333333" stroke-linecap="round" stroke-opacity="0.5" />
+	</Rule>
+  </Style>
+  
+  <Layer name="links" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>shadow1</StyleName>
+    <StyleName>color</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/links_local.csv</Parameter>
+    </Datasource>
+  </Layer>
+  <Layer name="l3_links" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>shadow1</StyleName>
+    <StyleName>l3_color</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/l3_links_local.csv</Parameter>
+    </Datasource>
+  </Layer>
+  <Layer name="routers" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>routerpoint</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/routers_local.csv</Parameter>
+    </Datasource>
+  </Layer>
+</Map>

ffmap/mapnik/routers_v2.xml

@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE Map>
+<Map background-color="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over">
+  <Style name="routerpoint" filter-mode="first">
+	<Rule>
+	  <Filter>([status] = 'online')</Filter>
+	  <!-- For directed antenna
+		<PointSymbolizer file="static/img/router_direct_green.svg" allow-overlap="true" transform="rotate(45)" />
+	  -->
+	  <PointSymbolizer file="static/img/router_green_v2.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline')</Filter>
+	  <PointSymbolizer file="static/img/router_red_v2.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown')</Filter>
+	  <PointSymbolizer file="static/img/router_yellow.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned')</Filter>
+	  <PointSymbolizer file="static/img/router_grey.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'online_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_green_v2_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_red_v2_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_yellow_white.svg" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'orphaned_wan')</Filter>
+	  <PointSymbolizer file="static/img/router_grey_white.svg" allow-overlap="true" />
+	</Rule>
+  </Style>
+  <Style name="color" filter-mode="first">
+	<Rule>
+	  <Filter>([quality] &lt; 1)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#008c00" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 105)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff1e1e" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 130)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff4949" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 155)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ff6a6a" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 180)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ffac53" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 205)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#ffeb79" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 230)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#79ff7c" stroke-linecap="butt" clip="false" />
+	</Rule>
+	<Rule>
+	  <Filter>([quality] &lt; 300)</Filter>
+	  <LineSymbolizer stroke-width="3" stroke="#04ff0a" stroke-linecap="butt" clip="false" />
+	</Rule>
+  </Style>
+  <Style name="l3_color" filter-mode="first">
+	<Rule>
+	  <LineSymbolizer stroke-width="3" stroke="#0684c4" stroke-linecap="butt" clip="false" />
+	</Rule>
+  </Style>
+  <Style name="shadow1">
+	<Rule>
+	  <LineSymbolizer stroke-width="4" stroke="#333333" stroke-linecap="round" stroke-opacity="0.5" />
+	</Rule>
+  </Style>
+  
+  <Layer name="links" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>shadow1</StyleName>
+    <StyleName>color</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/links_v2.csv</Parameter>
+    </Datasource>
+  </Layer>
+  <Layer name="l3_links" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>shadow1</StyleName>
+    <StyleName>l3_color</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/l3_links_v2.csv</Parameter>
+    </Datasource>
+  </Layer>
+  <Layer name="routers" srs="+proj=latlong +ellps=WGS84 +datum=WGS84 +no_defs">
+    <StyleName>routerpoint</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="file">csv/routers_v2.csv</Parameter>
+    </Datasource>
+  </Layer>
+</Map>

ffmap/mapnik/run.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
-liteserv.py links_and_routers.xml --processes=5 &
-liteserv.py hoods.xml -p 8001 --processes=5 &
-liteserv.py hoodsv2.xml -p 8002 --processes=5
+liteserv.py routers.xml --processes=5 &
+liteserv.py routers_v2.xml -p 8003 --processes=5 &
+liteserv.py routers_local.xml -p 8004 --processes=5 &
+liteserv.py hoods_v2.xml -p 8002 --processes=5
+liteserv.py hoods_poly.xml -p 8005 --processes=5
 
 killall liteserv.py

ffmap/mapnik/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 from distutils.core import setup
 setup(name='dynmapnik',

ffmap/mapnik/tilestache.cfg

@@ -4,31 +4,51 @@
 		"path": "/var/cache/ffmap/tiles/"
 	},
 	"layers": {
-		"tiles/links_and_routers": {
+		"tiles/routers": {
 			"provider": {
 				"class": "dynmapnik:DynMapnik",
 				"kwargs": {
-					"mapfile": "/usr/share/ffmap/links_and_routers.xml"
+					"mapfile": "/usr/share/ffmap/routers.xml"
 				}
 			},
 			"metatile": {"buffer": 128},
 			"cache lifespan": 300
 		},
-		"tiles/hoods": {
+		"tiles/routers_v2": {
 			"provider": {
 				"class": "dynmapnik:DynMapnik",
 				"kwargs": {
-					"mapfile": "/usr/share/ffmap/hoods.xml"
+					"mapfile": "/usr/share/ffmap/routers_v2.xml"
 				}
 			},
 			"metatile": {"buffer": 128},
 			"cache lifespan": 300
 		},
-		"tiles/hoodsv2": {
+		"tiles/routers_local": {
 			"provider": {
 				"class": "dynmapnik:DynMapnik",
 				"kwargs": {
-					"mapfile": "/usr/share/ffmap/hoodsv2.xml"
+					"mapfile": "/usr/share/ffmap/routers_local.xml"
+				}
+			},
+			"metatile": {"buffer": 128},
+			"cache lifespan": 300
+		},
+		"tiles/hoods_v2": {
+			"provider": {
+				"class": "dynmapnik:DynMapnik",
+				"kwargs": {
+					"mapfile": "/usr/share/ffmap/hoods_v2.xml"
+				}
+			},
+			"metatile": {"buffer": 128},
+			"cache lifespan": 300
+		},
+		"tiles/hoods_poly": {
+			"provider": {
+				"class": "dynmapnik:DynMapnik",
+				"kwargs": {
+					"mapfile": "/usr/share/ffmap/hoods_poly.xml"
 				}
 			},
 			"metatile": {"buffer": 128},

ffmap/maptools.py

@@ -4,19 +4,13 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
 
-from ffmap.dbtools import FreifunkDB
+from ffmap.mysqltools import FreifunkMySQL
+from ffmap.config import CONFIG
 
 import math
 import numpy as np
 from scipy.spatial import Voronoi
-
-import urllib.request, json
-
-db = FreifunkDB().handle()
-
-CONFIG = {
-	"csv_dir": "/var/lib/ffmap/csv"
-}
+import urllib.request, urllib.error, json
 
 EARTH_RADIUS = 6378137.0
 
@@ -73,109 +67,216 @@ def draw_voronoi_lines(csv, hoods):
 			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2))
 
 
-def update_mapnik_csv():
-	with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv:
-		csv.write("lng,lat,status\n")
-		for router in db.routers.find({"position.coordinates": {"$exists": True}}, {"status": 1, "position": 1}):
-			csv.write("%f,%f,%s\n" % (
-				router["position"]["coordinates"][0],
-				router["position"]["coordinates"][1],
-				router["status"]
-			))
+def update_mapnik_csv(mysql):
+	routers = mysql.fetchall("""
+		SELECT router.status, router.lat, router.lng, router.wan_uplink, v2, local FROM router
+		WHERE router.lat IS NOT NULL AND router.lng IS NOT NULL
+	""")
 
+	rv1 = "lng,lat,status\n"
+	rv2 = "lng,lat,status\n"
+	rvlocal = "lng,lat,status\n"
+	
+	for router in routers:
+		tmpstatus = router["status"]
+		if router["wan_uplink"]:
+			tmpstatus += "_wan";
+		tmp = "%f,%f,%s\n" % (
+			router["lng"],
+			router["lat"],
+			tmpstatus
+		)
+		if router["local"]:
+			rvlocal += tmp
+		elif router["v2"]:
+			rv2 += tmp
+		else:
+			rv1 += tmp
+
+	with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv:
+		csv.write(rv1)
+	with open(os.path.join(CONFIG["csv_dir"], "routers_v2.csv"), "w") as csv:
+		csv.write(rv2)
+	with open(os.path.join(CONFIG["csv_dir"], "routers_local.csv"), "w") as csv:
+		csv.write(rvlocal)
+
+	dblinks = mysql.fetchall("""
+		SELECT r1.id AS rid, r2.id AS nid, r1.lat AS rlat, r1.lng AS rlng, r2.lat AS nlat, r2.lng AS nlng, n.netif AS netif, n.type AS type, MAX(quality) AS quality, r1.v2, r1.local
+		FROM router AS r1
+		INNER JOIN router_neighbor AS n ON r1.id = n.router
+		INNER JOIN (
+			SELECT router, mac FROM router_netif GROUP BY mac, router
+			) AS net ON n.mac = net.mac
+		INNER JOIN router AS r2 ON net.router = r2.id
+		WHERE r1.lat IS NOT NULL AND r1.lng IS NOT NULL AND r2.lat IS NOT NULL AND r2.lng IS NOT NULL
+		AND r1.status = 'online'
+		GROUP BY r1.id, r1.lat, r1.lng, r2.id, r2.lat, r2.lng, n.netif, n.type, r1.v2, r1.local
+	""")
+	links = []
+	linksl3 = []
+	linksv2 = []
+	linksl3v2 = []
+	linkslocal = []
+	linksl3local = []
+	dictl3 = {}
+	dictl2 = {}
+	# The following code is very ugly, but works and is not too slow. Maybe make it nicer at some point ...
+	for row in dblinks:
+		if row.get("type")=="l3":
+			# Check for duplicate
+			if row["nid"] in dictl3.keys() and row["rid"] in dictl3[row["nid"]]:
+				continue
+			# Write current set to dict
+			if not row["rid"] in dictl3.keys():
+				dictl3[row["rid"]] = []
+			dictl3[row["rid"]].append(row["nid"])
+			
+			tmp = (
+				row["rlng"],
+				row["rlat"],
+				row["nlng"],
+				row["nlat"],
+			)
+			if row["local"]:
+				linksl3local.append(tmp)
+			elif row["v2"]:
+				linksl3v2.append(tmp)
+			else:
+				linksl3.append(tmp)
+		else:
+			# Check for duplicate
+			if row["nid"] in dictl2.keys() and row["rid"] in dictl2[row["nid"]].keys():
+				oldqual = dictl2[row["nid"]][row["rid"]]["data"][4]
+				# - Check for ethernet (ethernet always wins)
+				# - Take maximum quality (thus continue if current is lower)
+				if oldqual == 0 or oldqual > row["quality"]:
+					continue
+				# Delete old entry, as we create a new one with averaged quality
+				del dictl2[row["nid"]][row["rid"]]
+			if row["rid"] in dictl2.keys() and row["nid"] in dictl2[row["rid"]].keys():
+				oldqual = dictl2[row["rid"]][row["nid"]]["data"][4]
+				# - Check for ethernet (ethernet always wins)
+				# - Take maximum quality (thus continue if current is lower)
+				if oldqual == 0 or oldqual > row["quality"]:
+					continue
+				# No need to delete, since we overwrite later
+			# Write current set to dict
+			if not row["rid"] in dictl2.keys():
+				dictl2[row["rid"]] = {}
+			# Check for ethernet
+			if row["netif"].startswith("eth"):
+				row["quality"] = 0
+			
+			tmp = (
+				row["rlng"],
+				row["rlat"],
+				row["nlng"],
+				row["nlat"],
+				row["quality"],
+			)
+			dictl2[row["rid"]][row["nid"]] = {'v2':row["v2"],'local':row["local"],'data':tmp}
+	
+	for d1 in dictl2.values():
+		for d2 in d1.values():
+			if d2["local"]:
+				linkslocal.append(d2["data"])
+			elif d2["v2"]:
+				linksv2.append(d2["data"])
+			else:
+				links.append(d2["data"])
+	
 	with open(os.path.join(CONFIG["csv_dir"], "links.csv"), "w") as csv:
 		csv.write("WKT,quality\n")
-		links = []
-		for router in db.routers.find(
-			{
-				"position.coordinates": {"$exists": True},
-				"neighbours": {"$exists": True},
-				"status": "online"
-			},
-			{"position": 1, "neighbours": 1}
-		):
-			for neighbour in router["neighbours"]:
-				if "position" in neighbour and not neighbour.get("type"):
-					links.append((
-						router["position"]["coordinates"][0],
-						router["position"]["coordinates"][1],
-						neighbour["position"]["coordinates"][0],
-						neighbour["position"]["coordinates"][1],
-						neighbour["quality"]
-					))
 		for link in sorted(links, key=lambda l: l[4]):
 			csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link)
 
+	with open(os.path.join(CONFIG["csv_dir"], "links_v2.csv"), "w") as csv:
+		csv.write("WKT,quality\n")
+		for link in sorted(linksv2, key=lambda l: l[4]):
+			csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link)
+
+	with open(os.path.join(CONFIG["csv_dir"], "links_local.csv"), "w") as csv:
+		csv.write("WKT,quality\n")
+		for link in sorted(linkslocal, key=lambda l: l[4]):
+			csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % link)
+
 	with open(os.path.join(CONFIG["csv_dir"], "l3_links.csv"), "w") as csv:
 		csv.write("WKT\n")
-		for router in db.routers.find(
-			{
-				"position.coordinates": {"$exists": True},
-				"neighbours": {"$exists": True},
-				"status": "online"
-			},
-			{"position": 1, "neighbours": 1}
-		):
-			for neighbour in router["neighbours"]:
-				if "position" in neighbour and neighbour.get("type") and neighbour["type"] == "l3":
-					csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (
-						router["position"]["coordinates"][0],
-						router["position"]["coordinates"][1],
-						neighbour["position"]["coordinates"][0],
-						neighbour["position"]["coordinates"][1]
-					))
+		for link in linksl3:
+			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % link)
 
-	with open(os.path.join(CONFIG["csv_dir"], "hood-points.csv"), "w", encoding="UTF-8") as csv:
-		csv.write("lng,lat,name\n")
-		for hood in db.hoods.find({"position": {"$exists": True}}):
-			csv.write("%f,%f,\"%s\"\n" % (
-				hood["position"]["coordinates"][0],
-				hood["position"]["coordinates"][1],
-				hood["name"]
-			))
-
-	with open(os.path.join(CONFIG["csv_dir"], "hoods.csv"), "w") as csv:
+	with open(os.path.join(CONFIG["csv_dir"], "l3_links_v2.csv"), "w") as csv:
 		csv.write("WKT\n")
-		hoods = []
-		for hood in db.hoods.find({"position": {"$exists": True}}):
-			# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
-			x, y = merc_sphere(hood["position"]["coordinates"][0], hood["position"]["coordinates"][1])
-			hoods.append([x, y])
-		draw_voronoi_lines(csv, hoods)
+		for link in linksl3v2:
+			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % link)
 
+	with open(os.path.join(CONFIG["csv_dir"], "l3_links_local.csv"), "w") as csv:
+		csv.write("WKT\n")
+		for link in linksl3local:
+			csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % link)
+
+	dbhoodsv2 = mysql.fetchall("""
+		SELECT name, lat, lng FROM hoodsv2
+		WHERE lat IS NOT NULL AND lng IS NOT NULL
+	""")
 	with open(os.path.join(CONFIG["csv_dir"], "hood-points-v2.csv"), "w", encoding="UTF-8") as csv:
 		csv.write("lng,lat,name\n")
-		
-		with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
-			data = json.loads(url.read().decode())
-		
-		for hood in data:
-			if not ( 'lon' in hood and 'lat' in hood ):
-				continue
+		for hood in dbhoodsv2:
 			csv.write("%f,%f,\"%s\"\n" % (
-				hood["lon"],
+				hood["lng"],
 				hood["lat"],
 				hood["name"]
 			))
 
-	with open(os.path.join(CONFIG["csv_dir"], "hoodsv2.csv"), "w") as csv:
+	with open(os.path.join(CONFIG["csv_dir"], "hoods_v2.csv"), "w") as csv:
 		csv.write("WKT\n")
 		hoods = []
-		with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
-			data = json.loads(url.read().decode())
 		
-		for hood in data:
-			if not ( 'lon' in hood and 'lat' in hood ):
-				continue
+		for hood in dbhoodsv2:
 			# convert coordinates info marcator sphere as voronoi doesn't work with lng/lat
-			x, y = merc_sphere(hood["lon"], hood["lat"])
+			x, y = merc_sphere(hood["lng"], hood["lat"])
 			hoods.append([x, y])
 		draw_voronoi_lines(csv, hoods)
 
+	# Poly-Hoods
+	with urllib.request.urlopen("http://keyserver.freifunk-franken.de/v2/hoods.php") as url:
+		dbhoodspoly = json.loads(url.read().decode())
+	with open(os.path.join(CONFIG["csv_dir"], "hood-points-poly.csv"), "w", encoding="UTF-8") as csv:
+		csv.write("lng,lat,name\n")
+		for hood in dbhoodspoly:
+			for polygon in hood.get("polygons",()):
+				avlon = 0
+				avlat = 0
+				for p in polygon:
+					avlon += p["lon"]
+					avlat += p["lat"]
+				avlon /= len(polygon)
+				avlat /= len(polygon)
+				csv.write("%f,%f,\"%s\"\n" % (
+					avlon,
+					avlat,
+					hood["name"]
+				))
+
+	with open(os.path.join(CONFIG["csv_dir"], "hoods_poly.csv"), "w") as csv:
+		csv.write("WKT\n")
+		for hood in dbhoodspoly:
+			for polygon in hood.get("polygons",()):
+				oldlon = None
+				oldlat = None
+				for p in polygon:
+					if oldlon and oldlat:
+						csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (oldlon, oldlat, p["lon"], p["lat"]))
+					oldlon = p["lon"]
+					oldlat = p["lat"]
+				csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (oldlon, oldlat, polygon[0]["lon"], polygon[0]["lat"]))
+
 	# touch mapnik XML files to trigger tilelite watcher
-	touch("/usr/share/ffmap/hoods.xml")
-	touch("/usr/share/ffmap/hoodsv2.xml")
-	touch("/usr/share/ffmap/links_and_routers.xml")
+	touch("/usr/share/ffmap/hoods_v2.xml")
+	touch("/usr/share/ffmap/hoods_poly.xml")
+	touch("/usr/share/ffmap/routers.xml")
+	touch("/usr/share/ffmap/routers_v2.xml")
+	touch("/usr/share/ffmap/routers_local.xml")
 
 if __name__ == '__main__':
 	update_mapnik_csv()

ffmap/misc.py

@@ -1,6 +1,165 @@
 #!/usr/bin/python3
 
+import time
 import datetime
 
+from ffmap.config import CONFIG
+#from socket import inet_pton, inet_ntop, AF_INET6
+from ipaddress import IPv4Address, IPv6Address
+
+ipv6local = IPv6Address('fc00::')
+
 def utcnow():
 	return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
+
+def int2mac(data,keys=None):
+	if keys:
+		for k in keys:
+			data[k] = int2mac(data[k])
+		return data
+	if data:
+		return ':'.join(format(s, '02x') for s in data.to_bytes(6,byteorder='big'))
+		#return ':'.join(format(s, '02x') for s in bytes.fromhex('{0:x}'.format(data)))
+	else:
+		return ''
+
+def int2shortmac(data,keys=None):
+	if keys:
+		for k in keys:
+			data[k] = int2shortmac(data[k])
+		return data
+	if data:
+		return '{:012x}'.format(data)
+	else:
+		return ''
+
+def shortmac2mac(data):
+	if data:
+		return ':'.join(format(s, '02x') for s in bytes.fromhex(data.replace(':','')))
+	else:
+		return ''
+
+def mac2int(data):
+	if data:
+		return int(data.replace(":",""),16)
+	else:
+		return None
+
+def int2mactuple(data,index=None):
+	if index:
+		for r in data:
+			r[index] = int2mac(r[index])
+	else:
+		for r in data:
+			r = int2mac(r)
+	return data
+
+def ipv6tobin(data):
+	if data:
+		return IPv6Address(data).packed
+		#return inet_pton(AF_INET6,data)
+	else:
+		return None
+
+def ipv6tobinmasked(data):
+	if data:
+		ip = IPv6Address(data)
+		if ip >= ipv6local:
+			return ip.packed
+		else:
+			li = list(ip.packed)
+			# mask 1234:1234:ffff:ffff:ffff:ffff:ffff:ff34
+			li[4:15] = [255,255,255,255,255,255,255,255,255,255,255]
+		return IPv6Address(bytes(li)).packed
+	else:
+		return None
+
+def bintoipv6(data):
+	if data:
+		return IPv6Address(data).compressed
+		#return inet_ntop(AF_INET6,data)
+	else:
+		return ''
+
+def ipv4toint(data):
+	if data:
+		return int(IPv4Address(data))
+		#return inet_pton(AF_INET,data)
+	else:
+		return None
+def inttoipv4(data):
+	if data:
+		return str(IPv4Address(data))
+		#return inet_ntop(AF_INET,data)
+	else:
+		return ''
+
+def writelog(path, content):
+	with open(path, "a") as csv:
+		csv.write(time.strftime('{%Y-%m-%d %H:%M:%S}') + " - " + content + "\n")
+
+def writefulllog(content):
+	with open(CONFIG["debug_dir"] + "/fulllog.log", "a") as csv:
+		csv.write(time.strftime('{%Y-%m-%d %H:%M:%S}') + " - " + content + "\n")
+
+def neighbor_color(quality,netif,rt_protocol):
+	color = "#04ff0a"
+	if rt_protocol=="BATMAN_V":
+		if quality < 10:
+			color = "#ff1e1e"
+		elif quality < 20:
+			color = "#ff4949"
+		elif quality < 40:
+			color = "#ff6a6a"
+		elif quality < 80:
+			color = "#ffac53"
+		elif quality < 1000:
+			color = "#ffeb79"
+	else:
+		if quality < 105:
+			color = "#ff1e1e"
+		elif quality < 130:
+			color = "#ff4949"
+		elif quality < 155:
+			color = "#ff6a6a"
+		elif quality < 180:
+			color = "#ffac53"
+		elif quality < 205:
+			color = "#ffeb79"
+		elif quality < 230:
+			color = "#79ff7c"
+	if netif.startswith("eth"):
+		#color = "#999999"
+		color = "#008c00"
+	if quality < 0:
+		color = "#06a4f4"
+	return color
+
+def defrag_table(mysql,table,sleep):
+	minustime=0
+	allrows=0
+	start_time = time.time()
+
+	qry = "ALTER TABLE `%s` ENGINE = InnoDB" % (table)
+	mysql.execute(qry)
+	mysql.commit()
+
+	end_time = time.time()
+	if sleep > 0:
+		time.sleep(sleep)
+
+	writelog(CONFIG["debug_dir"] + "/deletetime.txt", "Defragmented table %s: %.3f seconds" % (table,end_time - start_time))
+	print("--- Defragmented table %s: %.3f seconds ---" % (table,end_time - start_time))
+
+def defrag_all(mysql,doall=False):
+	alltables = ('gw','gw_admin','gw_netif','hoods','hoodsv2','netifs','router','router_events','router_gw','router_ipv6','router_neighbor','router_netif','users')
+	stattables = ('router_stats','router_stats_gw','router_stats_neighbor','router_stats_netif','stats_global','stats_gw','stats_hood')
+
+	for t in alltables:
+		defrag_table(mysql,t,1)
+
+	if doall:
+		for t in stattables:
+			defrag_table(mysql,t,60)
+
+	writelog(CONFIG["debug_dir"] + "/deletetime.txt", "-------")

ffmap/mysqlconfig.example.py

@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+
+mysq = {
+	"host":"localhost",
+	"user":"root",
+	"passwd":"password",
+	"db":"dbname"
+	}

ffmap/mysqltools.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python3
+
+import MySQLdb
+from ffmap.mysqlconfig import mysq
+from ffmap.misc import *
+import datetime
+
+#import pytz
+
+class FreifunkMySQL:
+	
+	db = None
+	cur = None
+
+	def __init__(self):
+		#global mysq
+		self.db = MySQLdb.connect(host=mysq["host"], user=mysq["user"], passwd=mysq["passwd"], db=mysq["db"], charset="utf8")
+		#self.db.set_character_set('utf8')
+		self.cur = self.db.cursor(MySQLdb.cursors.DictCursor)
+
+	def close(self):
+		self.db.close()
+
+	def cursor(self):
+		return self.cur
+
+	def commit(self):
+		self.db.commit()
+	
+	def fetchall(self,str,tup=(),key=None):
+		self.cur.execute(str,tup)
+		result = self.cur.fetchall()
+		if len(result) > 0:
+			if key:
+				rnew = []
+				for r in result:
+					rnew.append(r[key])
+				return rnew
+			else:
+				return result
+		else:
+			return ()
+	
+	def fetchdict(self,str,tup,key,value=None):
+		self.cur.execute(str,tup)
+		dict = {}
+		for d in self.cur.fetchall():
+			if value:
+				dict[d[key]] = d[value]
+			else:
+				dict[d[key]] = d
+		return dict
+	
+	def findone(self,str,tup,sel=None):
+		self.cur.execute(str,tup)
+		result = self.cur.fetchall()
+		if len(result) > 0:
+			if sel:
+				return result[0][sel]
+			else:
+				return result[0]
+		else:
+			return False
+	
+	def executemany(self,a,b):
+		if not b:
+			return 0
+		return self.cur.executemany(a,b)
+	
+	def execute(self,a,b=None):
+		if b:
+			return self.cur.execute(a,b)
+		else:
+			return self.cur.execute(a)
+	
+	def utcnow(self):
+		return utcnow().strftime('%Y-%m-%d %H:%M:%S')
+	
+	def utctimestamp(self):
+		return int(utcnow().timestamp())
+	
+	def formatdt(self,dt):
+		return dt.strftime('%Y-%m-%d %H:%M:%S')
+	
+	def formattimestamp(self,t):
+		return int(t.timestamp())
+	
+	def utcawareint(self,data,keys=None):
+		if keys:
+			for k in keys:
+				data[k] = datetime.datetime.fromtimestamp(data[k],datetime.timezone.utc)
+		else:
+			data = datetime.datetime.fromtimestamp(data,datetime.timezone.utc)
+		return data
+	
+	def utcawaretupleint(self,data,index=None):
+		if index:
+			for r in data:
+				r[index] = datetime.datetime.fromtimestamp(r[index],datetime.timezone.utc)
+		else:
+			for r in data:
+				r = datetime.datetime.fromtimestamp(r,datetime.timezone.utc)
+		return data
+	
+	def utcaware(self,data,keys=None):
+		if keys:
+			for k in keys:
+				#self.utcaware(data[k])
+				#data[k] = pytz.utc.localize(data[k])
+				data[k] = data[k].replace(tzinfo=datetime.timezone.utc)
+		else:
+			#data = pytz.utc.localize(data)
+			data = data.replace(tzinfo=datetime.timezone.utc)
+		return data
+	
+	def utcawaretuple(self,data,index=None):
+		if index:
+			for r in data:
+				#self.utcaware(r[index])
+				#r[index] = pytz.utc.localize(r[index])
+				r[index] = r[index].replace(tzinfo=datetime.timezone.utc)
+		else:
+			for r in data:
+				#self.utcaware(r)
+				#r = pytz.utc.localize(r)
+				r = r.replace(tzinfo=datetime.timezone.utc)
+		return data

ffmap/stattools.py

@@ -4,91 +4,520 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
 
-from ffmap.dbtools import FreifunkDB
+from ffmap.mysqltools import FreifunkMySQL
+from ffmap.gwtools import gw_name, gw_bat
 from ffmap.misc import *
+from ffmap.config import CONFIG
 
-db = FreifunkDB().handle()
+from collections import OrderedDict
 
-def total_clients():
-	r = db.routers.aggregate([{"$group": {
-		"_id": None,
-		"clients": {"$sum": "$system.clients"}
-	}}])
-	return next(r)["clients"]
+def total_clients(mysql,selecthood=None):
+	if selecthood:
+		return mysql.findone("""
+			SELECT SUM(clients) AS clients
+			FROM router
+			WHERE hood = %s
+		""",(selecthood,),"clients")
+	else:
+		return mysql.findone("""
+			SELECT SUM(clients) AS clients
+			FROM router
+		""",(),"clients")
 
-def router_status():
-	r = db.routers.aggregate([{"$group": {
-		"_id": "$status",
-		"count": {"$sum": 1}
-	}}])
+def router_status(mysql,selecthood=None):
+	if selecthood:
+		tmp = mysql.fetchdict("""
+			SELECT status, COUNT(id) AS count
+			FROM router
+			WHERE hood = %s
+			GROUP BY status
+		""",(selecthood,),"status","count")
+	else:
+		tmp = mysql.fetchdict("""
+			SELECT status, COUNT(id) AS count
+			FROM router
+			GROUP BY status
+		""",(),"status","count")
+	tmp["sum"] = sum(tmp.values())
+	return tmp
+
+def router_traffic(mysql,selecthood=None):
+	# rx and tx are exchanged for bat0, since we want to get client traffic, which is the mirror of bat0 traffic
+	if selecthood:
+		tmp = mysql.findone("""
+			SELECT SUM(rx) AS tx, SUM(tx) AS rx FROM router_netif
+			INNER JOIN router ON router_netif.router = router.id
+			WHERE hood = %s AND gateway = FALSE AND netif = 'bat0'
+		""",(selecthood,))
+		gw = mysql.findone("""
+			SELECT SUM(rx) AS rx, SUM(tx) AS tx FROM router_netif
+			INNER JOIN router ON router_netif.router = router.id
+			WHERE hood = %s AND gateway = TRUE AND netif IN ('eth0.1','eth1.1','w2ap','w5ap')
+		""",(selecthood,))
+	else:
+		tmp = mysql.findone("""
+			SELECT SUM(rx) AS tx, SUM(tx) AS rx FROM router_netif
+			INNER JOIN router ON router_netif.router = router.id
+			WHERE gateway = FALSE AND netif = 'bat0'
+		""",())
+		gw = mysql.findone("""
+			SELECT SUM(rx) AS rx, SUM(tx) AS tx FROM router_netif
+			INNER JOIN router ON router_netif.router = router.id
+			WHERE gateway = TRUE AND netif IN ('eth0.1','eth1.1','w2ap','w5ap')
+		""",())
+	if "rx" in gw and gw["rx"]:
+		tmp["rx"] += gw["rx"]
+	if "tx" in gw and gw["tx"]:
+		tmp["tx"] += gw["tx"]
+	return tmp
+
+def total_clients_hood(mysql):
+	return mysql.fetchdict("""
+		SELECT hood, SUM(clients) AS clients
+		FROM router
+		GROUP BY hood
+	""",(),"hood","clients")
+
+def router_status_hood(mysql):
+	data = mysql.fetchall("""
+		SELECT hood, status, COUNT(id) AS count
+		FROM router
+		GROUP BY hood, status
+	""")
+	dict = {}
+	for d in data:
+		if not d["hood"] in dict:
+			dict[d["hood"]] = {}
+		dict[d["hood"]][d["status"]] = d["count"]
+	return dict
+
+def router_traffic_hood(mysql):
+	# rx and tx are exchanged for bat0, since we want to get client traffic, which is the mirror of bat0 traffic
+	dict = mysql.fetchdict("""
+		SELECT hood, SUM(rx) AS tx, SUM(tx) AS rx FROM router_netif
+		INNER JOIN router ON router_netif.router = router.id
+		WHERE gateway = FALSE AND netif = 'bat0'
+		GROUP BY hood
+	""",(),"hood")
+	gw = mysql.fetchall("""
+		SELECT hood, SUM(rx) AS rx, SUM(tx) AS tx FROM router_netif
+		INNER JOIN router ON router_netif.router = router.id
+		WHERE gateway = TRUE AND netif IN ('eth0.1','eth1.1','w2ap','w5ap')
+		GROUP BY hood
+	""")
+	allhoods = mysql.fetchall("""
+		SELECT hood
+		FROM router
+		GROUP BY hood
+	""")
+	for d in gw:
+		if not d["hood"] in dict:
+			dict[d["hood"]] = d
+		else:
+			dict[d["hood"]]["rx"] += d["rx"]
+			dict[d["hood"]]["tx"] += d["tx"]
+	for h in allhoods:
+		if not h["hood"] in dict:
+			dict[h["hood"]] =  {"hood": h["hood"], "rx": 0, "tx": 0}
+	return dict
+
+def total_clients_gw(mysql):
+	return mysql.fetchdict("""
+		SELECT router_gw.mac, SUM(clients) AS clients
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE
+		GROUP BY router_gw.mac
+	""",(),"mac","clients")
+
+def router_status_gw(mysql):
+	data = mysql.fetchall("""
+		SELECT router_gw.mac, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE
+		GROUP BY router_gw.mac, router.status
+	""")
+	dict = {}
+	for d in data:
+		if not d["mac"] in dict:
+			dict[d["mac"]] = {}
+		dict[d["mac"]][d["status"]] = d["count"]
+	return dict
+
+def router_models(mysql,selecthood=None,selectgw=None):
+	if selecthood:
+		return mysql.fetchdict("""
+			SELECT hardware, COUNT(id) AS count, SUM(clients) AS clients
+			FROM router
+			WHERE hood = %s
+			GROUP BY hardware
+			ORDER BY hardware
+		""",(selecthood,),"hardware")
+	elif selectgw:
+		return mysql.fetchdict("""
+			SELECT hardware, COUNT(router_gw.router) AS count, SUM(clients) AS clients
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE mac = %s
+			GROUP BY hardware
+			ORDER BY hardware
+		""",(mac2int(selectgw),),"hardware")
+	else:
+		return mysql.fetchdict("""
+			SELECT hardware, COUNT(id) AS count, SUM(clients) AS clients
+			FROM router
+			GROUP BY hardware
+			ORDER BY hardware
+		""",(),"hardware")
+
+def router_firmwares(mysql,selecthood=None,selectgw=None):
+	if selecthood:
+		return mysql.fetchdict("""
+			SELECT firmware, COUNT(id) AS count
+			FROM router
+			WHERE hood = %s
+			GROUP BY firmware
+			ORDER BY firmware
+		""",(selecthood,),"firmware","count")
+	elif selectgw:
+		return mysql.fetchdict("""
+			SELECT firmware, COUNT(router_gw.router) AS count
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE mac = %s
+			GROUP BY firmware
+			ORDER BY firmware
+		""",(mac2int(selectgw),),"firmware","count")
+	else:
+		return mysql.fetchdict("""
+			SELECT firmware, COUNT(id) AS count
+			FROM router
+			GROUP BY firmware
+			ORDER BY firmware
+		""",(),"firmware","count")
+
+def hoods(mysql,selectgw=None):
+	data = mysql.fetchall("""
+		SELECT hoods.id AS hoodid, hoods.name AS hood, status, COUNT(router.id) AS count
+		FROM router
+		LEFT JOIN hoods ON router.hood = hoods.id
+		GROUP BY hoods.id, hoods.name, status
+		ORDER BY hoods.name
+	""")
 	result = {}
-	for rs in r:
-		result[rs["_id"]] = rs["count"]
+	for rs in data:
+		if not rs["hood"]:
+			rs["hoodid"] = 1
+			rs["hood"] = "NoHood"
+		if not rs["hoodid"] in result:
+			result[rs["hoodid"]] = {'name':rs["hood"]}
+		result[rs["hoodid"]][rs["status"]] = rs["count"]
 	return result
 
-def router_models():
-	r = db.routers.aggregate([{"$group": {
-		"_id": "$hardware.name",
-		"count": {"$sum": 1}
-	}}])
+def hoods_sum(mysql,selectgw=None):
+	data = mysql.fetchall("""
+		SELECT hood, COUNT(id) AS count, SUM(clients) AS clients, MAX(v2) AS v2, MAX(local) AS local
+		FROM router
+		GROUP BY hood
+	""")
 	result = {}
-	for rs in r:
-		result[rs["_id"]] = rs["count"]
+	for rs in data:
+		if not rs["hood"]:
+			rs["hood"] = "Default"
+		result[rs["hood"]] = {"routers": rs["count"], "clients": rs["clients"], "v2": rs["v2"], "local": rs["local"]}
 	return result
 
-def router_firmwares():
-	r = db.routers.aggregate([{"$group": {
-		"_id": "$software.firmware",
-		"count": {"$sum": 1}
-	}}])
+def hoods_gws(mysql):
+	data = mysql.fetchall("""
+		SELECT hood, COUNT(sub.mac) AS count
+		FROM (
+			SELECT hood, router_gw.mac, COUNT(router.id) AS routers
+			FROM router
+			INNER JOIN router_gw ON router.id = router_gw.router
+			WHERE router.status = 'online'
+			GROUP BY hood, router_gw.mac
+		) AS sub
+		WHERE routers > 1
+		GROUP BY hood
+	""")
 	result = {}
-	for rs in r:
-		result[rs["_id"]] = rs["count"]
+	for rs in data:
+		if not rs["hood"]:
+			rs["hood"] = "Default"
+		result[rs["hood"]] = rs["count"]
 	return result
 
-def hoods():
-	r = db.routers.aggregate([{"$group": {
-		"_id": {"hood": "$hood", "status": "$status"},
-		"count": {"$sum": 1},
-	}}])
-	result = {}
-	for rs in r:
-		if not "hood" in rs["_id"]:
-			rs["_id"]["hood"] = "default"
-		if not rs["_id"]["hood"] in result:
-			result[rs["_id"]["hood"]] = {}
-		result[rs["_id"]["hood"]][rs["_id"]["status"]] = rs["count"]
+def gateways(mysql):
+	macs = mysql.fetchall("""
+		SELECT router_gw.mac, gw.name, gw.id AS gw, gw.version, gw_netif.netif
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		LEFT JOIN (gw_netif INNER JOIN gw ON gw_netif.gw = gw.id)
+		ON router_gw.mac = gw_netif.mac
+		WHERE router.status <> 'orphaned' AND NOT ISNULL(gw.name)
+		GROUP BY router_gw.mac
+		ORDER BY gw.name ASC, gw_netif.netif ASC, router_gw.mac ASC
+	""")
+	selected = mysql.fetchall("""
+		SELECT gw_netif.gw, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		INNER JOIN gw_netif ON gw_netif.mac = router_gw.mac
+		WHERE router_gw.selected = TRUE AND router.status <> 'orphaned'
+		GROUP BY gw_netif.gw, router.status
+	""")
+	others = mysql.fetchall("""
+		SELECT gw_netif.gw, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		INNER JOIN gw_netif ON gw_netif.mac = router_gw.mac
+		WHERE router_gw.selected = FALSE AND router.status <> 'orphaned'
+		GROUP BY gw_netif.gw, router.status
+	""")
+	
+	result = OrderedDict()
+	for m in macs:
+		if not m["gw"] in result:
+			result[m["gw"]] = {"name":m["name"],"version":m["version"],"macs":[],"selected":{},"others":{}}
+		result[m["gw"]]["macs"].append(m["mac"])
+	for rs in selected:
+		result[rs["gw"]]["selected"][rs["status"]] = rs["count"]
+	for rs in others:
+		result[rs["gw"]]["others"][rs["status"]] = rs["count"]
 	return result
 
-def hoods_sum():
-	r = db.routers.aggregate([{"$group": {
-		"_id": "$hood",
-		"count": {"$sum": 1},
-		"clients": {"$sum": "$system.clients"}
-	}}])
-	result = {}
-	for rs in r:
-		if not rs["_id"]:
-			rs["_id"] = "default"
-		result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]}
+def gws_ipv4(mysql):
+	data = mysql.fetchall("""
+		SELECT name, n1.ipv4, n1.netif AS batif, n2.netif AS vpnif, n2.mac FROM gw
+		INNER JOIN gw_netif AS n1 ON gw.id = n1.gw
+		LEFT JOIN gw_netif AS n2 ON n2.mac = n1.vpnmac AND n1.gw = n2.gw
+		WHERE n1.ipv4 IS NOT NULL
+		GROUP BY name, n1.ipv4, n1.netif, n2.netif, n2.mac
+		ORDER BY n1.ipv4
+	""")
+	
+	return data
+
+def gws_ipv6(mysql):
+	data = mysql.fetchall("""
+		SELECT name, n1.ipv6, n1.netif AS batif, n2.netif AS vpnif, n2.mac FROM gw
+		INNER JOIN gw_netif AS n1 ON gw.id = n1.gw
+		LEFT JOIN gw_netif AS n2 ON n2.mac = n1.vpnmac AND n1.gw = n2.gw
+		WHERE n1.ipv6 IS NOT NULL
+		GROUP BY name, n1.ipv6, n1.netif, n2.netif, n2.mac
+		ORDER BY n1.ipv6
+	""")
+	
+	return data
+
+def gws_dhcp(mysql):
+	data = mysql.fetchall("""
+		SELECT name, n1.dhcpstart, n1.dhcpend, n1.netif AS batif, n2.netif AS vpnif, n2.mac FROM gw
+		INNER JOIN gw_netif AS n1 ON gw.id = n1.gw
+		LEFT JOIN gw_netif AS n2 ON n2.mac = n1.vpnmac AND n1.gw = n2.gw
+		WHERE n1.dhcpstart IS NOT NULL
+		GROUP BY name, n1.dhcpstart, n1.dhcpend, n1.netif, n2.netif, n2.mac
+		ORDER BY n1.dhcpstart
+	""")
+	
+	return data
+
+def gws_ifs(mysql,selecthood=None):
+	if selecthood:
+		where = " AND hood=%s"
+		tup = (selecthood,)
+	else:
+		where = ""
+		tup = ()
+	
+	macs = mysql.fetchall("""
+		SELECT router_gw.mac, CONCAT(ISNULL(gw.name),'-',IF(NOT ISNULL(gw.name),CONCAT(gw.name,'-',gw_netif.netif),router_gw.mac)) AS sort
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		LEFT JOIN (gw_netif INNER JOIN gw ON gw_netif.gw = gw.id)
+		ON router_gw.mac = gw_netif.mac
+		WHERE router.status <> 'orphaned' {}
+		GROUP BY router_gw.mac
+		ORDER BY ISNULL(gw.name), gw.name ASC, gw_netif.netif ASC, router_gw.mac ASC
+	""".format(where),tup)
+	selected = mysql.fetchall("""
+		SELECT router_gw.mac, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE AND router.status <> 'orphaned' {}
+		GROUP BY router_gw.mac, router.status
+	""".format(where),tup)
+	others = mysql.fetchall("""
+		SELECT router_gw.mac, router.status, COUNT(router_gw.router) AS count
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = FALSE AND router.status <> 'orphaned' {}
+		GROUP BY router_gw.mac, router.status
+	""".format(where),tup)
+	
+	result = OrderedDict()
+	for m in macs:
+		result[m["mac"]] = {"selected":{},"others":{},"sort":m["sort"]}
+	for rs in selected:
+		result[rs["mac"]]["selected"][rs["status"]] = rs["count"]
+	for rs in others:
+		result[rs["mac"]]["others"][rs["status"]] = rs["count"]
 	return result
 
-def record_global_stats():
-	db.stats.insert_one({
-		"time": utcnow(),
-		"router_status": router_status(),
-		"total_clients": total_clients()
-	})
-
-
-def router_user_sum():
-	r = db.routers.aggregate([{"$group": {
-		"_id": "$user.nickname",
-		"count": {"$sum": 1},
-		"clients": {"$sum": "$system.clients"}
-	}}])
+def gws_sum(mysql,selecthood=None):
+	if selecthood:
+		where = " AND hood=%s"
+		tup = (selecthood,)
+	else:
+		where = ""
+		tup = ()
+	
+	data = mysql.fetchall("""
+		SELECT router_gw.mac, COUNT(router_gw.router) AS count, SUM(router.clients) AS clients
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		WHERE router_gw.selected = TRUE AND router.status <> 'orphaned' {}
+		GROUP BY router_gw.mac
+	""".format(where),tup)
 	result = {}
-	for rs in r:
-		if rs["_id"]:
-			result[rs["_id"]] = {"routers": rs["count"], "clients": rs["clients"]}
+	for rs in data:
+		result[rs["mac"]] = {"routers": rs["count"], "clients": rs["clients"]}
+	return result
+
+def gws_info(mysql,selecthood=None):
+	if selecthood:
+		where = "WHERE hood=%s"
+		tup = (selecthood,)
+	else:
+		where = ""
+		tup = ()
+	
+	data = mysql.fetchdict("""
+		SELECT router_gw.mac AS mac, gw.name AS gw, stats_page, version, n1.netif AS gwif, n2.netif AS batif, n2.mac AS batmac, n2.ipv4 AS ipv4, n2.ipv6 AS ipv6, n2.dhcpstart AS dhcpstart, n2.dhcpend AS dhcpend
+		FROM router
+		INNER JOIN router_gw ON router.id = router_gw.router
+		LEFT JOIN (
+			gw_netif AS n1
+			INNER JOIN gw ON n1.gw = gw.id
+			LEFT JOIN gw_netif AS n2 ON n1.mac = n2.vpnmac AND n1.gw = n2.gw
+		) ON router_gw.mac = n1.mac
+		{}
+		GROUP BY router_gw.mac, n2.netif, n2.mac
+	""".format(where),tup,"mac")
+	for d in data.values():
+		d["label"] = gw_name(d)
+		d["batX"] = gw_bat(d)
+	return data
+
+def gws_admin(mysql,selectgw):
+	if not selectgw:
+		return None
+	
+	data = mysql.fetchall("""
+		SELECT gw_admin.name
+		FROM gw_netif
+		INNER JOIN gw_admin ON gw_netif.gw = gw_admin.gw
+		WHERE mac = %s
+		ORDER BY prio ASC
+	""",(mac2int(selectgw),),"name")
+	return data
+
+def record_global_stats(mysql):
+	threshold=(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"])).timestamp()
+	time = mysql.utctimestamp()
+	status = router_status(mysql)
+	traffic = router_traffic(mysql)
+
+	mysql.execute("""
+		INSERT INTO stats_global (time, clients, online, offline, unknown, orphaned, rx, tx)
+		VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+		ON DUPLICATE KEY UPDATE
+			clients=VALUES(clients),
+			online=VALUES(online),
+			offline=VALUES(offline),
+			unknown=VALUES(unknown),
+			orphaned=VALUES(orphaned),
+			rx=VALUES(rx),
+			tx=VALUES(tx)
+	""",(time,total_clients(mysql),status.get("online",0),status.get("offline",0),status.get("unknown",0),status.get("orphaned",0),traffic["rx"],traffic["tx"],))
+
+	mysql.execute("""
+		DELETE FROM stats_global
+		WHERE time < %s
+	""",(threshold,))
+
+	mysql.commit()
+
+def record_hood_stats(mysql):
+	threshold=(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"])).timestamp()
+	time = mysql.utctimestamp()
+	status = router_status_hood(mysql)
+	clients = total_clients_hood(mysql)
+	traffic = router_traffic_hood(mysql)
+
+	hdata = []
+	for hood in clients.keys():
+		if not hood:
+			hood = "Default"
+		hdata.append((time,hood,clients[hood],status[hood].get("online",0),status[hood].get("offline",0),status[hood].get("unknown",0),status[hood].get("orphaned",0),traffic[hood]["rx"],traffic[hood]["tx"],))
+
+	mysql.executemany("""
+		INSERT INTO stats_hood (time, hood, clients, online, offline, unknown, orphaned, rx, tx)
+		VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+		ON DUPLICATE KEY UPDATE
+			clients=VALUES(clients),
+			online=VALUES(online),
+			offline=VALUES(offline),
+			unknown=VALUES(unknown),
+			orphaned=VALUES(orphaned),
+			rx=VALUES(rx),
+			tx=VALUES(tx)
+	""",hdata)
+
+	mysql.execute("""
+		DELETE FROM stats_hood
+		WHERE time < %s
+	""",(threshold,))
+
+	mysql.commit()
+
+def record_gw_stats(mysql):
+	threshold=(utcnow() - datetime.timedelta(days=CONFIG["global_stat_days"])).timestamp()
+	time = mysql.utctimestamp()
+	status = router_status_gw(mysql)
+	clients = total_clients_gw(mysql)
+	
+	gdata = []
+	for mac in clients.keys():
+		gdata.append((time,mac,clients[mac],status[mac].get("online",0),status[mac].get("offline",0),status[mac].get("unknown",0),status[mac].get("orphaned",0),))
+
+	mysql.executemany("""
+		INSERT INTO stats_gw (time, mac, clients, online, offline, unknown, orphaned)
+		VALUES (%s, %s, %s, %s, %s, %s, %s)
+		ON DUPLICATE KEY UPDATE
+			clients=VALUES(clients),
+			online=VALUES(online),
+			offline=VALUES(offline),
+			unknown=VALUES(unknown),
+			orphaned=VALUES(orphaned)
+	""",gdata)
+	
+	mysql.execute("""
+		DELETE FROM stats_gw
+		WHERE time < %s
+	""",(threshold,))
+
+	mysql.commit()
+
+def router_user_sum(mysql):
+	data = mysql.fetchall("""
+		SELECT contact, COUNT(id) AS count, SUM(clients) AS clients
+		FROM router
+		GROUP BY contact
+	""")
+	result = {}
+	for rs in data:
+		if rs["contact"]:
+			result[rs["contact"].lower()] = {"routers": rs["count"], "clients": rs["clients"]}
 	return result

ffmap/systemd/uwsgi-ffmap.service

@@ -3,7 +3,7 @@ Description=FF-MAP Web UI
 After=syslog.target
 
 [Service]
-ExecStart=/usr/bin/uwsgi_python3 -s 127.0.0.1:3031 -w ffmap.web.application:app --master --processes 4 --enable-threads --uid www-data --gid www-data --catch-exceptions
+ExecStart=/usr/bin/uwsgi_python3 -s 127.0.0.1:3031 -w ffmap.web.application:app --master --processes 4 --enable-threads --uid www-data --gid www-data --catch-exceptions --disable-logging --log-4xx --log-5xx
 Restart=always
 KillSignal=SIGQUIT
 Type=notify

ffmap/systemd/uwsgi-tiles.service

@@ -3,7 +3,7 @@ Description=FF-MAP Tiles
 After=syslog.target
 
 [Service]
-ExecStart=/usr/bin/uwsgi_python -s 127.0.0.1:3032 --eval 'import TileStache; application = TileStache.WSGITileServer("/usr/share/ffmap/tilestache.cfg")' --master --processes 4 --uid www-data --gid www-data --enable-threads
+ExecStart=/usr/bin/uwsgi_python3 -s 127.0.0.1:3032 --eval 'import sys; sys.path.insert(0,"/data/fff/TileStache"); import TileStache; application = TileStache.WSGITileServer("/usr/share/ffmap/tilestache.cfg")' --master --processes 4 --uid www-data --gid www-data --enable-threads --disable-logging --log-4xx --log-5xx
 Restart=always
 KillSignal=SIGQUIT
 Type=notify

ffmap/usertools.py

@@ -4,12 +4,13 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
 
-from ffmap.dbtools import FreifunkDB
+from ffmap.mysqltools import FreifunkMySQL
 from ffmap.misc import *
 
 from werkzeug.security import generate_password_hash, check_password_hash
 
-db = FreifunkDB().handle()
+class AccountWithEmptyField(Exception):
+	pass
 
 class AccountWithEmailExists(Exception):
 	pass
@@ -24,67 +25,141 @@ class InvalidToken(Exception):
 	pass
 
 def register_user(nickname, email, password):
-	user_with_nick  = db.users.find_one({"nickname": nickname})
-	user_with_email = db.users.find_one({"email": email})
+	if not nickname or not email:
+		raise AccountWithEmptyField()
+
+	mysql = FreifunkMySQL()
+	user_with_nick  = mysql.findone("SELECT id, email FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+	user_with_email  = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id")
+	pw = generate_password_hash(password)
 	if user_with_email:
+		mysql.close()
 		raise AccountWithEmailExists()
-	elif user_with_nick and "email" in user_with_nick:
+	elif user_with_nick and user_with_nick["email"]:
+		mysql.close()
 		raise AccountWithNicknameExists()
 	else:
-		user_update = {
-			"nickname": nickname,
-			"password": generate_password_hash(password),
-			"email": email,
-			"created": utcnow()
-		}
+		time = mysql.utcnow()
 		if user_with_nick:
-			db.users.update_one({"_id": user_with_nick["_id"]}, {"$set": user_update})
-			return user_with_nick["_id"]
+			mysql.execute("""
+				UPDATE users
+				SET password = %s, email = %s, created = %s, token = NULL
+				WHERE id = %s
+				LIMIT 1
+			""",(pw,email,time,user_with_nick["id"],))
+			mysql.commit()
+			mysql.close()
+			return user_with_nick["id"]
 		else:
-			return db.users.insert_one(user_update).inserted_id
+			mysql.execute("""
+				INSERT INTO users (nickname, password, email, created, token)
+				VALUES (%s, %s, %s, %s, NULL)
+			""",(nickname,pw,email,time,))
+			userid = mysql.cursor().lastrowid
+			mysql.commit()
+			mysql.close()
+			return userid
 
 def check_login_details(nickname, password):
-	user = db.users.find_one({"nickname": nickname})
+	mysql = FreifunkMySQL()
+	user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+	userbymail = mysql.findone("SELECT * FROM users WHERE email = %s LIMIT 1",(nickname,))
+	mysql.close()
+	
 	if user and check_password_hash(user.get('password', ''), password):
 		return user
-	else:
-		return False
+	elif userbymail and check_password_hash(userbymail.get('password', ''), password):
+		return userbymail
+	return False
 
-def reset_user_password(email, token=None, password=None):
-	user = db.users.find_one({"email": email})
+def reset_user_password(mysql, email, token=None, password=None):
+	user = mysql.findone("SELECT id, nickname, token FROM users WHERE email = %s LIMIT 1",(email,))
 	if not user:
 		raise AccountNotExisting()
 	elif password:
 		if user.get("token") == token:
-			db.users.update_one({"_id": user["_id"]}, {
-				"$set": {"password": generate_password_hash(password)},
-				"$unset": {"token": 1},
-			})
+			mysql.execute("""
+				UPDATE users
+				SET password = %s, token = NULL
+				WHERE id = %s
+				LIMIT 1
+			""",(generate_password_hash(password),user["id"],))
+			mysql.commit()
 		else:
 			raise InvalidToken()
 	elif token:
-		db.users.update_one({"_id": user["_id"]}, {"$set": {"token": token}})
+		mysql.execute("""
+			UPDATE users
+			SET token = %s
+			WHERE id = %s
+			LIMIT 1
+		""",(token,user["id"],))
+		mysql.commit()
+	return user
 
-def set_user_password(nickname, password):
-	user = db.users.find_one({"nickname": nickname})
-	if not user:
+def set_user_password(mysql, nickname, password):
+	userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id")
+	if not userid:
 		raise AccountNotExisting()
 	elif password:
-		db.users.update_one({"_id": user["_id"]}, {
-			"$set": {"password": generate_password_hash(password)},
-		})
+		mysql.execute("""
+			UPDATE users
+			SET password = %s
+			WHERE id = %s
+			LIMIT 1
+		""",(generate_password_hash(password),userid,))
+		mysql.commit()
 
-def set_user_email(nickname, email):
-	user = db.users.find_one({"nickname": nickname})
-	user_with_email = db.users.find_one({"email": email})
-	if user_with_email:
+def set_user_email(mysql, nickname, email):
+	userid = mysql.findone("SELECT id FROM users WHERE nickname = %s LIMIT 1",(nickname,),"id")
+	useridemail = mysql.findone("SELECT id FROM users WHERE email = %s LIMIT 1",(email,),"id")
+	if useridemail:
 		raise AccountWithEmailExists()
-	if not user:
+	if not userid:
 		raise AccountNotExisting()
 	elif email:
-		db.users.update_one({"_id": user["_id"]}, {
-			"$set": {"email": email},
-		})
+		mysql.execute("""
+			UPDATE users
+			SET email = %s
+			WHERE id = %s
+			LIMIT 1
+		""",(email,userid,))
+		mysql.commit()
+
+def set_user_admin(mysql, nickname, admin):
+	mysql.execute("""
+		UPDATE users
+		SET admin = %s
+		WHERE nickname = %s
+		LIMIT 1
+	""",(admin,nickname,))
+	mysql.commit()
+
+def set_user_abuse(mysql, nickname, abuse):
+	mysql.execute("""
+		UPDATE users
+		SET abuse = %s
+		WHERE nickname = %s
+		LIMIT 1
+	""",(abuse,nickname,))
+	mysql.commit()
+
+def users_v2(mysql):
+	data = mysql.fetchall("""
+		SELECT contact, COUNT(id) AS count, v2
+		FROM router
+		GROUP BY contact, v2
+	""")
+
+	datasort = {}
+	for d in data:
+		contact = d["contact"].lower()
+		if not contact in datasort:
+			datasort[contact] = {"v2":0, "v1":0}
+		if d["v2"]:
+			datasort[contact]["v2"] = d["count"]
+		else:
+			datasort[contact]["v1"] = d["count"]
+
+	return datasort
 
-def set_user_admin(nickname, admin):
-	db.users.update({"nickname": nickname}, {"$set": {"admin": admin}})

ffmap/web/api.py

@@ -1,113 +1,339 @@
 #!/usr/bin/python3
 
 from ffmap.routertools import *
+from ffmap.gwtools import *
 from ffmap.maptools import *
-from ffmap.dbtools import FreifunkDB
-from ffmap.stattools import record_global_stats
+from ffmap.mysqltools import FreifunkMySQL
+from ffmap.stattools import record_global_stats, record_hood_stats
+from ffmap.config import CONFIG
+from ffmap.misc import *
 
 from flask import Blueprint, request, make_response, redirect, url_for, jsonify, Response
-from pymongo import MongoClient
 from bson.json_util import dumps as bson2json
 import json
 
 from operator import itemgetter
 
+import datetime
+import time
+import traceback
+
 api = Blueprint("api", __name__)
 
-db = FreifunkDB().handle()
+# Load router netif statistics
+@api.route('/load_netif_stats/<dbid>')
+def load_netif_stats(dbid):
+	netif = request.args.get("netif","")
+	mysql = FreifunkMySQL()
+	netiffetch = mysql.fetchall("""
+		SELECT netifs.name AS netif, rx, tx, time
+		FROM router_stats_netif
+		INNER JOIN netifs ON router_stats_netif.netif = netifs.id
+		WHERE router = %s AND netifs.name = %s
+	""",(dbid,netif,))
+	mysql.close()
+
+	for ns in netiffetch:
+		ns["time"] = {"$date": int(mysql.utcawareint(ns["time"]).timestamp()*1000)}
+
+	r = make_response(json.dumps(netiffetch))
+	r.mimetype = 'application/json'
+	return r
+
+# Load router neighbor statistics
+@api.route('/load_neighbor_stats/<dbid>')
+def load_neighbor_stats(dbid):
+	mysql = FreifunkMySQL()
+	neighfetch = mysql.fetchall("""
+		SELECT quality, mac, time FROM router_stats_neighbor WHERE router = %s
+	""",(dbid,))
+	mysql.close()
+
+	neighdata = {}
+
+	for ns in neighfetch:
+		ns["time"] = {"$date": int(mysql.utcawareint(ns["time"]).timestamp()*1000)}
+		if not ns["mac"] in neighdata:
+			neighdata[ns["mac"]] = []
+		neighdata[ns["mac"]].append(ns)
+
+	r = make_response(json.dumps(neighdata))
+	r.mimetype = 'application/json'
+	return r
 
 # map ajax
 @api.route('/get_nearest_router')
 def get_nearest_router():
-	res_router = db.routers.find_one(
-		{"position": {"$near": {"$geometry": {
-			"type": "Point",
-			"coordinates": [float(request.args.get("lng")), float(request.args.get("lat"))]
-		}}}},
-		{
-			"hostname": 1,
-			"neighbours": 1,
-			"position": 1,
-			"description": 1,
-		}
-	)
-	r = make_response(bson2json(res_router))
+	lng = float(request.args.get("lng"))
+	lat = float(request.args.get("lat"))
+	
+	wherelist = []
+	if request.args.get("v1") == "on":
+		wherelist.append("(v2 = FALSE AND local = FALSE)")
+	if request.args.get("v2") == "on":
+		wherelist.append("(v2 = TRUE AND local = FALSE)")
+	if request.args.get("local") == "on":
+		wherelist.append("local = TRUE")
+	if wherelist:
+		where = " AND ( " + ' OR '.join(wherelist) + " ) "
+	else:
+		r = make_response(bson2json(None))
+		r.mimetype = 'application/json'
+		return r
+	
+	mysql = FreifunkMySQL()
+	router = mysql.findone("""
+		SELECT r.id, r.hostname, r.lat, r.lng, r.description, r.routing_protocol,
+			( acos(  cos( radians(%s) )
+						  * cos( radians( r.lat ) )
+						  * cos( radians( r.lng ) - radians(%s) )
+						  + sin( radians(%s) ) * sin( radians( r.lat ) )
+						 )
+			) AS distance
+		FROM
+			router AS r
+		WHERE r.lat IS NOT NULL AND r.lng IS NOT NULL """ + where + """ 
+		ORDER BY
+			distance ASC
+		LIMIT 1
+	""",(lat,lng,lat,))
+	if not router:
+		r = make_response(bson2json(None))
+		r.mimetype = 'application/json'
+		return r
+	
+	router["neighbours"] = mysql.fetchall("""
+		SELECT nb.mac, nb.netif, nb.quality, r.hostname, r.id
+		FROM router_neighbor AS nb
+		INNER JOIN (
+			SELECT router, mac FROM router_netif GROUP BY mac, router
+			) AS net ON nb.mac = net.mac
+		INNER JOIN router as r ON net.router = r.id
+		WHERE nb.router = %s""",(router["id"],))
+	mysql.close()
+	for n in router["neighbours"]:
+		n["color"] = neighbor_color(n["quality"],n["netif"],router["routing_protocol"])
+	
+	r = make_response(bson2json(router))
 	r.mimetype = 'application/json'
 	return r
 
 # router by mac (link from router webui)
 @api.route('/get_router_by_mac/<mac>')
 def get_router_by_mac(mac):
-	res_routers = db.routers.find({"netifs.mac": mac.lower()}, {"_id": 1})
-	if res_routers.count() != 1:
-		return redirect(url_for("router_list", q="netifs.mac:%s" % mac))
+	mysql = FreifunkMySQL()
+	res_routers = mysql.fetchall("""
+		SELECT id
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		WHERE mac = %s
+		GROUP BY mac, id
+	""",(mac2int(mac),))
+	mysql.close()
+	if len(res_routers) != 1:
+		return redirect(url_for("router_list", q="mac:%s" % mac))
 	else:
-		return redirect(url_for("router_info", dbid=next(res_routers)["_id"]))
+		return redirect(url_for("router_info", dbid=res_routers[0]["id"]))
 
+# Read alfred data WITH surrounding {"64":"<data>"}
 @api.route('/alfred', methods=['GET', 'POST'])
 def alfred():
-	#set_alfred_data = {65: "hallo", 66: "welt"}
-	set_alfred_data = {}
-	r = make_response(json.dumps(set_alfred_data))
-	#import cProfile, pstats, io
-	#pr = cProfile.Profile()
-	#pr.enable()
-	if request.method == 'POST':
-		alfred_data = request.get_json()
-		if alfred_data:
-			# load router status xml data
-			for mac, xml in alfred_data.get("64", {}).items():
-				import_nodewatcher_xml(mac, xml)
-			r.headers['X-API-STATUS'] = "ALFRED data imported"
-		detect_offline_routers()
-		delete_orphaned_routers()
-		record_global_stats()
-		update_mapnik_csv()
-	#pr.disable()
-	#s = io.StringIO()
-	#sortby = 'cumulative'
-	#ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
-	#ps.print_stats()
-	#print(s.getvalue())
-	r.mimetype = 'application/json'
-	return r
+	try:
+		start_time = time.time()
+		mysql = FreifunkMySQL()
+		r = make_response(json.dumps({}))
+		r.mimetype = 'application/json'
+		#import cProfile, pstats, io
+		#pr = cProfile.Profile()
+		#pr.enable()
+		banned = mysql.fetchall("""
+			SELECT mac FROM banned
+		""",(),"mac")
+		hoodsv2 = mysql.fetchall("""
+			SELECT name FROM hoodsv2
+		""",(),"name")
+		statstime = utcnow()
+		netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id")
+		hoodsdict = mysql.fetchdict("SELECT id, name FROM hoods",(),"name","id")
+		if request.method == 'POST':
+			try:
+				alfred_data = request.get_json()
+			except Exception as e:
+				writelog(CONFIG["debug_dir"] + "/fail_alfred.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],'JSON parsing failed'))
+				writefulllog("Warning: Error converting ALFRED data to JSON:\n__%s" % (request.get_data(True,True).replace("\n", "\n__")))
+				r.headers['X-API-STATUS'] = "JSON parsing failed"
+				return r
+
+			if alfred_data:
+				# load router status xml data
+				i = 1
+				for mac, xml in alfred_data.get("64", {}).items():
+					import_nodewatcher_xml(mysql, mac, xml, banned, hoodsv2, netifdict, hoodsdict, statstime)
+					if (i%500 == 0):
+						mysql.commit()
+					i += 1
+				mysql.commit()
+				r.headers['X-API-STATUS'] = "ALFRED data imported"
+		mysql.close()
+		#pr.disable()
+		#s = io.StringIO()
+		#sortby = 'cumulative'
+		#ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
+		#ps.print_stats()
+		#print(s.getvalue())
+
+		writelog(CONFIG["debug_dir"] + "/apitime.txt", "%s - %.3f seconds" % (request.environ['REMOTE_ADDR'],time.time() - start_time))
+		return r
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_alfred.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],str(e)))
+		writefulllog("Warning: Error while processing ALFRED data: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
+		r.headers['X-API-STATUS'] = "ERROR processing ALFRED data"
+		return r
+
+# Read alfred data without surrounding {"64":"<data>"}, so just <data> can be sent
+@api.route('/alfred2', methods=['GET', 'POST'])
+def alfred2():
+	try:
+		start_time = time.time()
+		mysql = FreifunkMySQL()
+		banned = mysql.fetchall("""
+			SELECT mac FROM banned
+		""",(),"mac")
+		hoodsv2 = mysql.fetchall("""
+			SELECT name FROM hoodsv2
+		""",(),"name")
+		statstime = utcnow()
+		netifdict = mysql.fetchdict("SELECT id, name FROM netifs",(),"name","id")
+		hoodsdict = mysql.fetchdict("SELECT id, name FROM hoods",(),"name","id")
+
+		r = make_response(json.dumps({}))
+		r.mimetype = 'application/json'
+		if request.method == 'POST':
+			try:
+				alfred_data = request.get_json()
+			except Exception as e:
+				writelog(CONFIG["debug_dir"] + "/fail_alfred2.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],'JSON parsing failed'))
+				writefulllog("Warning: Error converting ALFRED2 data to JSON:\n__%s" % (request.get_data(True,True).replace("\n", "\n__")))
+				r.headers['X-API-STATUS'] = "JSON parsing failed"
+				return r
+
+			if alfred_data:
+				# load router status xml data
+				i = 1
+				for mac, xml in alfred_data.items():
+					import_nodewatcher_xml(mysql, mac, xml, banned, hoodsv2, netifdict, hoodsdict, statstime)
+					if (i%500 == 0):
+						mysql.commit()
+					i += 1
+				mysql.commit()
+				r.headers['X-API-STATUS'] = "ALFRED2 data imported"
+		mysql.close()
+
+		writelog(CONFIG["debug_dir"] + "/apitime.txt", "%s - %.3f seconds (alfred2)" % (request.environ['REMOTE_ADDR'],time.time() - start_time))
+		return r
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_alfred.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],str(e)))
+		writefulllog("Warning: Error while processing ALFRED2 data: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
+		r.headers['X-API-STATUS'] = "ERROR processing ALFRED2 data"
+		return r
+
+@api.route('/gwinfo', methods=['GET', 'POST'])
+def gwinfo():
+	try:
+		start_time = time.time()
+		mysql = FreifunkMySQL()
+		#set_data = {65: "hallo", 66: "welt"}
+		set_data = {}
+		r = make_response(json.dumps(set_data))
+		if request.method == 'POST':
+			try:
+				gw_data = request.get_json()
+			except Exception as e:
+				writelog(CONFIG["debug_dir"] + "/fail_gwinfo.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],'JSON parsing failed'))
+				writefulllog("Warning: Error converting GWINFO data to JSON:\n__%s" % (request.get_data(True,True).replace("\n", "\n__")))
+				return
+			
+			if gw_data:
+				import_gw_data(mysql,gw_data)
+				mysql.commit()
+				r.headers['X-API-STATUS'] = "GW data imported"
+		mysql.close()
+		
+		writelog(CONFIG["debug_dir"] + "/gwtime.txt", "%s - %.3f seconds" % (request.environ['REMOTE_ADDR'],time.time() - start_time))
+		
+		r.mimetype = 'application/json'
+		return r
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_gwinfo.txt", "{} - {}".format(request.environ['REMOTE_ADDR'],str(e)))
+		writefulllog("Warning: Error while processing GWINFO data: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
 
 
 # https://github.com/ffansbach/de-map/blob/master/schema/nodelist-schema-1.0.0.json
 @api.route('/nodelist')
 def nodelist():
-	router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'system.clients', 'position.coordinates', 'last_contact'])
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT id, hostname, status, clients, last_contact, lat, lng
+		FROM router
+	""",())
+	router_data = mysql.utcawaretuple(router_data,"last_contact")
+	mysql.close()
 	nodelist_data = {'version': '1.0.0'}
 	nodelist_data['nodes'] = list()
 	for router in router_data:
 		nodelist_data['nodes'].append(
 			{
-				'id': str(router['_id']),
+				'id': str(router['id']),
 				'name': router['hostname'],
 				'node_type': 'AccessPoint',
-				'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']),
+				'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
 				'status': {
 					'online': router['status'] == 'online',
-					'clients': router['system']['clients'],
+					'clients': router['clients'],
 					'lastcontact': router['last_contact'].isoformat()
 				}
 			}
 		)
-		if 'position' in router:
+		if router['lat'] and router['lng']:
 			nodelist_data['nodes'][-1]['position'] = {
-				'lat': router['position']['coordinates'][1],
-				'long': router['position']['coordinates'][0]
+				'lat': router['lat'],
+				'long': router['lng']
 			}
 	return jsonify(nodelist_data)
 
 @api.route('/wifianal/<selecthood>')
 def wifianal(selecthood):
-	router_data = db.routers.find({'hood': selecthood}, projection=['hostname','netifs'])
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT hostname, mac, netif
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		INNER JOIN hoods ON router.hood = hoods.id
+		WHERE hoods.name = %s
+		GROUP BY router.id, netif
+	""",(selecthood,))
+	mysql.close()
 	
+	return wifianalhelper(router_data,"Hood: " + selecthood)
+
+@api.route('/wifianalall')
+def wifianalall():
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT hostname, mac, netif
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		GROUP BY id, netif
+	""",())
+	mysql.close()
+	
+	return wifianalhelper(router_data,"ALL hoods")
+
+def wifianalhelper(router_data, headline):
 	s = "#----------WifiAnalyzer alias file----------\n"
 	s += "# \n"
 	s += "#Freifunk Franken\n"
-	s += "#Hood: " + selecthood + "\n"
+	s += "#" + headline + "\n"
 	s += "# \n"
 	s += "#Encoding: UTF-8.\n"
 	s += "#The line starts with # is comment.\n"
@@ -118,166 +344,193 @@ def wifianal(selecthood):
 	s += "# \n"
 	
 	for router in router_data:
-		if not 'netifs' in router:
+		if not router['mac']:
 			continue
-		for netif in router['netifs']:
-			if not 'mac' in netif:
-				continue
-			if netif['name'] == 'br-mesh':
-				s += netif["mac"] + "|Mesh_" + router['hostname'] + "\n"
-			elif netif['name'] == 'w2ap':
-				s += netif["mac"] + "|" + router['hostname'] + "\n"
-			elif netif['name'] == 'w5ap':
-				s += netif["mac"] + "|W5_" + router['hostname'] + "\n"
-			elif netif['name'] == 'w5mesh':
-				s += netif["mac"] + "|W5Mesh_" + router['hostname'] + "\n"
+		if router["netif"] == 'br-mesh':
+			s += int2mac(router["mac"]) + "|Mesh_" + router['hostname'] + "\n"
+		elif router["netif"] == 'w2ap':
+			s += int2mac(router["mac"]) + "|" + router['hostname'] + "\n"
+		elif router["netif"] == 'w5ap':
+			s += int2mac(router["mac"]) + "|W5_" + router['hostname'] + "\n"
+		elif router["netif"] == 'w5mesh':
+			s += int2mac(router["mac"]) + "|W5Mesh_" + router['hostname'] + "\n"
 	
 	return Response(s,mimetype='text/plain')
 
-@api.route('/routers')
-def routers():
-	router_data = db.routers.find(projection=['_id', 'hostname', 'status', 'hood', 'user.nickname', 'hardware.name', 'software.firmware', 'system.clients', 'position.coordinates', 'last_contact', 'netifs'])
-	nodelist_data = {'version': '1.0.0'}
+@api.route('/dnslist')
+def dnslist():
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT hostname, mac, MIN(ipv6) AS fd43
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		INNER JOIN router_ipv6 ON router.id = router_ipv6.router AND router_netif.netif = router_ipv6.netif
+		WHERE LEFT(HEX(ipv6),4) = 'fd43'
+		GROUP BY hostname, mac
+	""",())
+	mysql.close()
+
+	s = ""
+	for router in router_data:
+		s += int2shortmac(router["mac"]) + "\t" + bintoipv6(router["fd43"]) + "\n"
+
+	return Response(s,mimetype='text/plain')
+
+@api.route('/dnsentries')
+def dnsentries():
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT hostname, mac, MIN(ipv6) AS fd43
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		INNER JOIN router_ipv6 ON router.id = router_ipv6.router AND router_netif.netif = router_ipv6.netif
+		WHERE LEFT(HEX(ipv6),4) = 'fd43'
+		GROUP BY hostname, mac
+	""",())
+	mysql.close()
+
+	s = ""
+	for router in router_data:
+		s += int2shortmac(router["mac"]) + ".fff.community.  300  IN  AAAA  " + bintoipv6(router["fd43"]) + "    ; " + router["hostname"] + "\n"
+
+	return Response(s,mimetype='text/plain')
+
+def nodelist_helper(where = "",data=()):
+	# Suppresses routers without br-mesh
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT router.id, hostname, status, hoods.id AS hoodid, hoods.name AS hood, contact, nickname, hardware, firmware, clients, lat, lng, last_contact, mac, sys_loadavg, fe80_addr
+		FROM router
+		INNER JOIN hoods ON router.hood = hoods.id
+		INNER JOIN router_netif ON router.id = router_netif.router
+		LEFT JOIN users ON router.contact = users.email
+		WHERE netif = 'br-mesh' {}
+		ORDER BY hostname ASC
+	""".format(where),data)
+	router_data = mysql.utcawaretuple(router_data,"last_contact")
+	router_net = mysql.fetchall("""
+		SELECT id, netif, COUNT(router) AS count
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		GROUP BY id, netif
+	""")
+	mysql.close()
+
+	net_dict = {}
+	for rs in router_net:
+		if not rs["id"] in net_dict:
+			net_dict[rs["id"]] = []
+		net_dict[rs["id"]].append(rs["netif"])
+	nodelist_data = {'version': '1.1.0'}
 	nodelist_data['nodes'] = list()
 	
 	for router in router_data:
-		hood = ""
-		user = ""
-		firmware = ""
-		mac = ""
 		fastd = 0
 		l2tp = 0
 		
-		if 'hood' in router:
-			hood = router['hood']
-		if 'user' in router:
-			user = router['user']['nickname']
-		if 'software' in router:
-			firmware = router['software']['firmware']
-		
-		if 'netifs' in router:
-			for netif in router['netifs']:
-				if netif['name'] == 'fffVPN':
+		if router["id"] in net_dict:
+			for netif in net_dict[router["id"]]:
+				if netif == 'fffVPN':
 					fastd += 1
-				elif netif['name'].startswith('l2tp'):
+				elif netif.startswith('l2tp'):
 					l2tp += 1
-				elif netif['name'] == 'br-mesh' and 'mac' in netif:
-					mac = netif["mac"]
+				#elif netif['netif'] == 'br-mesh' and 'mac' in netif:
+				#	mac = netif["mac"]
+		
+		if not router['mac']:
+			continue
 		
 		nodelist_data['nodes'].append(
 			{
-				'id': str(router['_id']),
+				'id': str(router['id']),
 				'name': router['hostname'],
-				'mac': mac,
-				'hood': hood,
+				'mac': int2mac(router['mac']),
+				'hoodid': router['hoodid'],
+				'hood': router['hood'],
 				'status': router['status'],
-				'user': user,
-				'hardware': router['hardware']['name'],
-				'firmware': firmware,
-				'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']),
-				'clients': router['system']['clients'],
+				'user': router['nickname'],
+				'hardware': router['hardware'],
+				'firmware': router['firmware'],
+				'loadavg': router['sys_loadavg'],
+				'href': 'https://monitoring.freifunk-franken.de/mac/' + int2shortmac(router['mac']),
+				'clients': router['clients'],
 				'lastcontact': router['last_contact'].isoformat(),
+				'fe80_addr': bintoipv6(router['fe80_addr']),
 				'uplink': {
 					'fastd': fastd,
 					'l2tp': l2tp
 				}
 			}
 		)
-		if 'position' in router:
-			nodelist_data['nodes'][-1]['position'] = {
-				'lat': router['position']['coordinates'][1],
-				'lng': router['position']['coordinates'][0]
-			}
-	return jsonify(nodelist_data)
+		nodelist_data['nodes'][-1]['position'] = {
+			'lat': router['lat'],
+			'lng': router['lng']
+		}
+	return nodelist_data
 
 @api.route('/nopos')
 def no_position():
-    router_data = db.routers.find(filter={'position': { '$exists': False}}, projection=['_id', 'hostname', 'system.contact', 'user.nickname', 'software.firmware'])
-    #nodelist_data = dict()
-    nodelist_data = list()
-    for router in router_data:
-        nodelist_data.append(
-            {
-                'name': router['hostname'],
-                'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']),
-                'firmware': router['software']['firmware']
-            }
-        )
-        if 'system' in router and 'contact' in router['system']:
-            nodelist_data[-1]['contact'] = router['system']['contact']
-        if 'user' in router and 'nickname' in router['user']:
-            nodelist_data[-1]['owner'] = router['user']['nickname']
-        else:
-            nodelist_data[-1]['owner'] = ''
+	mysql = FreifunkMySQL()
+	router_data = mysql.fetchall("""
+		SELECT router.id, hostname, contact, nickname, firmware
+		FROM router
+		LEFT JOIN users ON router.contact = users.email
+		WHERE lat IS NULL OR lng IS NULL
+	""")
+	mysql.close()
+	#nodelist_data = dict()
+	nodelist_data = list()
+	for router in router_data:
+		nick = router['nickname']
+		if not nick:
+			nick = ""
+		nodelist_data.append(
+			{
+				'name': router['hostname'],
+				'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['id']),
+				'firmware': router['firmware'],
+				'contact': router['contact'],
+				'owner': nick
+			}
+		)
 
+	nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False)
+	nodes = dict()
+	nodes['nodes'] = list(nodelist_data2)
 
-    nodelist_data2 = sorted(nodelist_data, key=itemgetter('owner'), reverse=False)
-    nodes = dict()
-    nodes['nodes'] = list(nodelist_data2)
+	return jsonify(nodes)
 
-    return jsonify(nodes)
+@api.route('/routers')
+def routers():
+	# Suppresses routers without br-mesh
+	return jsonify(nodelist_helper())
 
-import pymongo
 @api.route('/routers_by_nickname/<nickname>')
 def get_routers_by_nickname(nickname):
-    try:
-        user = db.users.find_one({"nickname": nickname})
-        assert user
-    except AssertionError:
-        return "User not found"
-
-    nodelist_data = dict()
-    nodelist_data['nodes'] = list()
-    routers=db.routers.find({"user._id": user["_id"]}, {"hostname": 1, "netifs": 1, "_id": 1}).sort("hostname", pymongo.ASCENDING)
-    for router in routers:
-        #print(router['hostname'])
-        for netif in router['netifs']:
-            if netif['name'] == 'br-mesh':
-                #print(netif['ipv6_fe80_addr'])
-                nodelist_data['nodes'].append(
-                {
-                        'name': router['hostname'],
-                        'oid': str(router['_id']),
-                        'ipv6_fe80_addr': netif['ipv6_fe80_addr']
-                    }
-                )
-    return jsonify(nodelist_data)
+	mysql = FreifunkMySQL()
+	users = mysql.fetchall("""
+		SELECT id
+		FROM users
+		WHERE nickname = %s
+		LIMIT 1
+	""",(nickname,))
+	mysql.close()
+	if len(users)==0:
+		return "User not found"
 
+	return jsonify(nodelist_helper("AND nickname = %s",(nickname,)))
 
 @api.route('/routers_by_keyxchange_id/<keyxchange_id>')
 def get_routers_by_keyxchange_id(keyxchange_id):
-    try:
-        hood = db.hoods.find_one({"keyxchange_id": int(keyxchange_id)})
-        assert hood
-    except AssertionError:
-        return "Hood not found"
-    nodelist_data = dict()
-    nodelist_data['nodes'] = list()
-    routers = db.routers.find({"hood": hood["name"]}, {"hostname": 1, "hardware": 1, "netifs": 1, "_id": 1, "software": 1, "position": 1, "system": 1, "position_comment": 1, "description": 1}).sort("hostname", pymongo.ASCENDING)
-    for router in routers:
-        for netif in router['netifs']:
-            if netif['name'] == 'br-mesh':
-                if 'ipv6_fe80_addr' not in netif:
-                    continue
-                nodelist_data['nodes'].append(
-                    {
-                        'name': router['hostname'],
-                        'ipv6_fe80_addr': netif['ipv6_fe80_addr'],
-                        'href': 'https://monitoring.freifunk-franken.de/routers/' + str(router['_id']),
-                        'firmware': router['software']['firmware'],
-                        'hardware': router['hardware']['name']
-                    }
-                )
-                if 'position' in router:
-                    nodelist_data['nodes'][-1]['position'] = {
-                        'lat': router['position']['coordinates'][1],
-                        'long': router['position']['coordinates'][0]
-                    }
-                if 'system' in router and 'contact' in router['system']:
-                    nodelist_data['nodes'][-1]['contact'] = router['system']['contact']
-                if 'description' in router:
-                    nodelist_data['nodes'][-1]['description'] = router['description']
+	mysql = FreifunkMySQL()
+	hood = mysql.findone("""
+		SELECT name
+		FROM hoodsv2
+		WHERE id = %s
+		LIMIT 1
+	""",(int(keyxchange_id),),"name")
+	mysql.close()
+	if not hood:
+		return "Hood not found"
 
-                if 'position_comment' in router:
-                    nodelist_data['nodes'][-1]['position']['comment'] = router['position_comment']
-    return jsonify(nodelist_data)
+	return jsonify(nodelist_helper('AND hoods.name = %s',(hood,)))

ffmap/web/application.py

@@ -6,28 +6,32 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 
 from ffmap.web.api import api
 from ffmap.web.filters import filters
-from ffmap.dbtools import FreifunkDB
+from ffmap.mysqltools import FreifunkMySQL
 from ffmap import stattools
 from ffmap.usertools import *
+from ffmap.routertools import delete_router, ban_router
+from ffmap.gwtools import gw_name, gw_bat
 from ffmap.web.helpers import *
+from ffmap.config import CONFIG
+from ffmap.misc import *
 
 from flask import Flask, render_template, request, Response, redirect, url_for, flash, session
 import bson
-import pymongo
 from bson.json_util import dumps as bson2json
 from bson.objectid import ObjectId
 import base64
+import datetime
 
 app = Flask(__name__)
 app.register_blueprint(api, url_prefix='/api')
 app.register_blueprint(filters)
 
-db = FreifunkDB().handle()
-
 tileurls = {
-	"links_and_routers": "/tiles/links_and_routers",
-	"hoods": "/tiles/hoods",
-	"hoodsv2": "/tiles/hoodsv2",
+	"routers": "/tiles/routers",
+	"routers_v2": "/tiles/routers_v2",
+	"routers_local": "/tiles/routers_local",
+	"hoods_v2": "/tiles/hoods_v2",
+	"hoods_poly": "/tiles/hoods_poly"
 }
 
 @app.route('/')
@@ -44,132 +48,599 @@ def router_map():
 
 @app.route('/routers')
 def router_list():
-	query, query_str = parse_router_list_search_query(request.args)
-	return render_template("router_list.html", query_str=query_str, routers=db.routers.find(query, {
-		"hostname": 1,
-		"status": 1,
-		"hood": 1,
-		"user.nickname": 1,
-		"hardware.name": 1,
-		"created": 1,
-		"system.uptime": 1,
-		"system.clients": 1,
-	}).sort("hostname", pymongo.ASCENDING))
+	where, tuple, query_str = parse_router_list_search_query(request.args)
+	mysql = FreifunkMySQL()
+	
+	routers = mysql.fetchall("""
+		SELECT router.id, hostname, status, hoods.id AS hoodid, hoods.name AS hood, contact, nickname, hardware, router.created, sys_uptime, last_contact, clients, router.lat, router.lng, reset, blocked, v2, local
+		FROM router
+		INNER JOIN hoods ON router.hood = hoods.id
+		LEFT JOIN users ON router.contact = users.email
+		LEFT JOIN (
+			SELECT router, blocked.mac AS blocked FROM router_netif
+			INNER JOIN blocked ON router_netif.mac = blocked.mac
+			WHERE netif = 'br-mesh'
+		) AS b
+		ON router.id = b.router
+		{}
+		ORDER BY hostname ASC
+	""".format(where),tuple)
+	mysql.close()
+	routers = mysql.utcawaretuple(routers,"created")
+	routers = mysql.utcawaretuple(routers,"last_contact")
+	
+	return render_template("router_list.html", query_str=query_str, routers=routers, numrouters=len(routers))
+
+# test
+@app.route('/v2routers')
+def v2_routers():
+	try:
+		mysql = FreifunkMySQL()
+		statsv2 = mysql.fetchall("""
+			SELECT time, CAST(SUM(clients) AS SIGNED) clients, CAST(SUM(online) AS SIGNED) online, CAST(SUM(offline) AS SIGNED) offline, CAST(SUM(unknown) AS SIGNED) unknown, CAST(SUM(orphaned) AS SIGNED) orphaned, CAST(SUM(rx) AS SIGNED) rx, CAST(SUM(tx) AS SIGNED) tx
+			FROM stats_hood
+			INNER JOIN hoods ON hoods.id = stats_hood.hood
+			LEFT JOIN hoodsv2 ON hoodsv2.name = hoods.name
+			WHERE time > 1531612800 AND ( hoodsv2.id IS NOT NULL OR hoods.name REGEXP '[vV]2$' )
+			GROUP BY time
+		""")
+		statsv1 = mysql.fetchall("""
+			SELECT time, CAST(SUM(clients) AS SIGNED) clients, CAST(SUM(online) AS SIGNED) online, CAST(SUM(offline) AS SIGNED) offline, CAST(SUM(unknown) AS SIGNED) unknown, CAST(SUM(orphaned) AS SIGNED) orphaned, CAST(SUM(rx) AS SIGNED) rx, CAST(SUM(tx) AS SIGNED) tx
+			FROM stats_hood
+			INNER JOIN hoods ON hoods.id = stats_hood.hood
+			WHERE time > 1531612800 AND ( hoods.id > 9999 AND hoods.id < 11000 )
+			GROUP BY time
+		""")
+		mysql.close()
+		statsv2 = mysql.utcawaretupleint(statsv2,"time")
+		statsv1 = mysql.utcawaretupleint(statsv1,"time")
+
+		return render_template("v2routers.html",statsv2 = statsv2,statsv1 = statsv1)
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_v2.txt", str(e))
+		import traceback
+		writefulllog("Warning: Failed to display v2 page: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
+
+
+# router by mac (short link version)
+@app.route('/mac/<mac>', methods=['GET'])
+def router_mac(mac):
+	mysql = FreifunkMySQL()
+	res_routers = mysql.fetchall("""
+		SELECT id
+		FROM router
+		INNER JOIN router_netif ON router.id = router_netif.router
+		WHERE mac = %s
+		GROUP BY mac, id
+	""",(mac2int(mac),))
+	mysql.close()
+	if len(res_routers) != 1:
+		return redirect(url_for("router_list", q="mac:%s" % mac))
+	elif request.args.get('fffconfig', None) != None:
+		return redirect(url_for("router_info", dbid=res_routers[0]["id"], fffconfig=1))
+	elif request.args.get('json', None) != None:
+		return redirect(url_for("router_info", dbid=res_routers[0]["id"], json=1))
+	else:
+		return redirect(url_for("router_info", dbid=res_routers[0]["id"]))
 
 @app.route('/routers/<dbid>', methods=['GET', 'POST'])
 def router_info(dbid):
 	try:
-		router = db.routers.find_one({"_id": ObjectId(dbid)})
-		assert router
-		if request.method == 'POST':
-			if request.form.get("act") == "delete":
-				user = None
-				# a router may not have a owner, but admin users still can delete it
-				if ("user" in router) and ("nickname" in router["user"]):
-					user = router["user"]["nickname"]
-				if is_authorized(user, session):
-					db.routers.delete_one({"_id": ObjectId(dbid)})
-					flash("<b>Router <i>%s</i> deleted!</b>" % router["hostname"], "success")
-					return redirect(url_for("index"))
-				else:
-					flash("<b>You are not authorized to perform this action!</b>", "danger")
-	except (bson.errors.InvalidId, AssertionError):
-		return "Router not found"
-	if request.args.get('json', None) != None:
-		del router["stats"]
-		#FIXME: Only as admin
-		return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json')
-	else:
-		return render_template("router.html", router=router, tileurls=tileurls)
+		mysql = FreifunkMySQL()
+		router = mysql.findone("""
+			SELECT router.*, hoods.id AS hoodid, hoods.name AS hoodname FROM router
+			INNER JOIN hoods ON router.hood = hoods.id
+			WHERE router.id = %s LIMIT 1
+		""",(dbid,))
+		mac = None
+		
+		if router:
+			if request.args.get('fffconfig', None) != None:
+				mysql.close()
+				s = "\nconfig fff 'system'\n"
+				s += "        option hostname '{}'\n".format(router["hostname"])
+				s += "        option description '{}'\n".format(router["description"])
+				s += "        option latitude '{}'\n".format(router["lat"] if router["lat"] else "")
+				s += "        option longitude '{}'\n".format(router["lng"] if router["lng"] else "")
+				s += "        option position_comment '{}'\n".format(router["position_comment"])
+				s += "        option contact '{}'\n".format(router["contact"])
+				return Response(s,mimetype='text/plain')
+
+			router = mysql.utcaware(router,["created","last_contact"])
+
+			router["user"] = mysql.findone("SELECT nickname FROM users WHERE email = %s",(router["contact"],),"nickname")
+			router["netifs"] = mysql.fetchall("""SELECT * FROM router_netif WHERE router = %s""",(dbid,))
+			
+			netifs = []
+			for n in router["netifs"]:
+				n["ipv6_addrs"] = mysql.fetchall("""SELECT ipv6 FROM router_ipv6 WHERE router = %s AND netif = %s""",(dbid,n["netif"],),"ipv6")
+				if n["netif"]=="br-mesh":
+					mac = n["mac"]
+				netifs.append(n["netif"])
+			
+			router["neighbours"] = mysql.fetchall("""
+				SELECT nb.mac, nb.netif, nb.quality, r.hostname, r.id
+				FROM router_neighbor AS nb
+				LEFT JOIN (
+					SELECT router, mac FROM router_netif GROUP BY mac, router
+					) AS net ON nb.mac = net.mac
+				LEFT JOIN router as r ON net.router = r.id
+				WHERE nb.router = %s
+				ORDER BY nb.quality DESC
+			""",(dbid,))
+			# FIX SQL: only one from router_netif
+			
+			router["gws"] = mysql.fetchall("""
+				SELECT router_gw.mac AS mac, quality, router_gw.netif AS netif, gw_class, selected, gw.name AS gw, n1.netif AS gwif, n2.netif AS batif, n2.mac AS batmac
+				FROM router_gw
+				LEFT JOIN (
+					gw_netif AS n1
+					INNER JOIN gw ON n1.gw = gw.id
+					LEFT JOIN gw_netif AS n2 ON n1.mac = n2.vpnmac AND n1.gw = n2.gw
+				) ON router_gw.mac = n1.mac
+				WHERE router = %s
+			""",(dbid,))
+			for gw in router["gws"]:
+				gw["label"] = gw_name(gw)
+				gw["batX"] = gw_bat(gw)
+			
+			router["events"] = mysql.fetchall("""SELECT * FROM router_events WHERE router = %s""",(dbid,))
+			router["events"] = mysql.utcawaretuple(router["events"],"time")
+			
+			## Create json with all data except stats
+			if request.args.get('json', None) != None:
+				mysql.close()
+				return Response(bson2json(router, sort_keys=True, indent=4), mimetype='application/json')
+			
+			cwan = "blue"
+			cclient = "orange"
+			cbatman = "#29c329"
+			cvpn = "red"
+			chidden = "gray"
+			
+			## Label netifs AFTER json if clause
+			for n in router["netifs"]:
+				netif = n["netif"];
+				desc = None
+				color = None
+				if netif == 'br-mesh':
+					desc = "Bridge"
+				elif netif.endswith('.1'):
+					desc = "Clients via Ethernet"
+					color = cclient
+				elif netif.endswith('.2'):
+					desc = "WAN"
+					color = cwan
+				elif netif.endswith('.3'):
+					desc = "Mesh via Ethernet"
+					color = cbatman
+				elif netif == "w2ap":
+					desc = "Clients @ 2.4 GHz"
+					color = cclient
+				elif netif == "w2mesh" or netif == "w2ibss":
+					desc = "Mesh @ 2.4 GHz"
+					color = cbatman
+				elif netif == "w2configap":
+					desc = "Config @ 2.4 GHz"
+					color = chidden
+				elif netif == "w5ap":
+					desc = "Clients @ 5 GHz"
+					color = cclient
+				elif netif == "w5mesh" or netif == "w5ibss":
+					desc = "Mesh @ 5 GHz"
+					color = cbatman
+				elif netif == "w5configap":
+					desc = "Config @ 5 GHz"
+					color = chidden
+				elif netif == "fffVPN":
+					desc = "Fastd VPN Tunnel"
+					color = cvpn
+				elif netif.startswith("l2tp"):
+					desc = "L2TP VPN Tunnel"
+					color = cvpn
+				elif netif.startswith("bat"):
+					desc = "Batman Interface"
+				elif netif.startswith("eth") and any(item.startswith("{}.".format(netif)) for item in netifs):
+					desc = "Switch"
+				elif netif == "eth1":
+					# already known from above: no switch; no one-port, as there must be eth0
+					if not "eth0" in netifs or any(item.startswith("eth0.") for item in netifs):
+						desc = "WAN"
+						color = cwan
+					else:
+						# Second port of Nanostation M2
+						desc = "Ethernet Multi-Port"
+				elif netif == "eth0":
+					if any(item.startswith("eth1.") for item in netifs):
+						# already known from above: no switch
+						desc = "WAN"
+						color = cwan
+					else:
+						# First port of Nanostation M2 or ONE-Port
+						desc = "Ethernet Multi-Port"
+				n["description"] = desc
+				n["color"] = color
+
+			## Set color for neighbors AFTER json if clause
+			for n in router["neighbours"]:
+				n["color"] = neighbor_color(n["quality"],n["netif"],router["routing_protocol"])
+
+			router["stats"] = mysql.fetchall("""SELECT * FROM router_stats WHERE router = %s""",(dbid,))
+			for s in router["stats"]:
+				s["time"] = mysql.utcawareint(s["time"])
+
+			threshold_neighstats = (utcnow() - datetime.timedelta(hours=24)).timestamp()
+			neighfetch = mysql.fetchall("""
+				SELECT quality, mac, time FROM router_stats_neighbor WHERE router = %s AND time > %s
+			""",(dbid,threshold_neighstats,))
+
+			neighdata = {}
+			for ns in neighfetch:
+				ns["time"] = {"$date": int(mysql.utcawareint(ns["time"]).timestamp()*1000)}
+				if not ns["mac"] in neighdata:
+					neighdata[ns["mac"]] = []
+				neighdata[ns["mac"]].append(ns)
+
+			neighident = mysql.fetchall("""
+				SELECT snb.mac, r.hostname, n.netif
+				FROM router_stats_neighbor AS snb
+				INNER JOIN router_netif AS n ON snb.mac = n.mac
+				INNER JOIN router AS r ON n.router = r.id
+				WHERE snb.router = %s AND n.netif <> 'w2ap' AND n.netif <> 'w5ap'
+				GROUP BY snb.mac, r.hostname, n.netif
+			""",(dbid,))
+			neighlabel = {}
+			for ni in neighident:
+				label = ni["hostname"]
+				# add network interface when there are multiple links to same node
+				for ni2 in neighident:
+					if label == ni2["hostname"] and ni["mac"] != ni2["mac"]:
+						# This shows the NEIGHBOR'S interface name
+						label += "@" + ni["netif"]
+				append = " (old)"
+				for nnn in router["neighbours"]:
+					if nnn["mac"] == ni["mac"]:
+						append = ""
+				neighlabel[ni["mac"]] = label + append
+
+			gwfetch = mysql.fetchall("""
+				SELECT quality, mac, time FROM router_stats_gw WHERE router = %s
+			""",(dbid,))
+			
+			for ns in gwfetch:
+				ns["time"] = mysql.utcawareint(ns["time"])
+
+			if request.method == 'POST':
+				if request.form.get("act") == "delete":
+					# a router may not have a owner, but admin users still can delete it
+					if is_authorized(router["user"], session):
+						delete_router(mysql,dbid)
+						flash("<b>Router <i>%s</i> deleted!</b>" % router["hostname"], "success")
+						mysql.close()
+						return redirect(url_for("index"))
+					else:
+						flash("<b>You are not authorized to perform this action!</b>", "danger")
+				elif request.form.get("act") == "ban":
+					if session.get('admin'):
+						if mac:
+							ban_router(mysql,dbid)
+							delete_router(mysql,dbid)
+							flash("<b>Router <i>%s</i> banned!</b>" % router["hostname"], "success")
+							mysql.close()
+							return redirect(url_for("index"))
+						else:
+							flash("<b>Router has no br-mesh and thus cannot be banned!</b>", "danger")
+					else:
+						flash("<b>You are not authorized to perform this action!</b>", "danger")
+				elif request.form.get("act") == "changeblocked" and mac:
+					if session.get('admin'):
+						if request.form.get("blocked") == "true":
+							added = mysql.utcnow()
+							mysql.execute("INSERT INTO blocked (mac, added) VALUES (%s, %s)",(mac,added,))
+							mysql.execute("""
+								INSERT INTO router_events (router, time, type, comment)
+								VALUES (%s, %s, %s, %s)
+							""",(dbid,mysql.utcnow(),"admin","Marked as blocked",))
+							mysql.commit()
+						else:
+							mysql.execute("DELETE FROM blocked WHERE mac = %s",(mac,))
+							mysql.execute("""
+								INSERT INTO router_events (router, time, type, comment)
+								VALUES (%s, %s, %s, %s)
+							""",(dbid,mysql.utcnow(),"admin","Removed blocked status",))
+							mysql.commit()
+						router["events"] = mysql.fetchall("""SELECT * FROM router_events WHERE router = %s""",(dbid,))
+						router["events"] = mysql.utcawaretuple(router["events"],"time")
+					else:
+						flash("<b>You are not authorized to perform this action!</b>", "danger")
+				elif request.form.get("act") == "report":
+					abusemails = mysql.fetchall("SELECT email FROM users WHERE abuse = 1")
+					for a in abusemails:
+						send_email(
+							recipient = a["email"],
+							subject   = "Monitoring: Router %s reported" % router["hostname"],
+							content   = "Hello Admin,\n\n" +
+									"The router with hostname %s has been reported as abusive by a user.\n" % router["hostname"] +
+									"Please take care:\n" +
+									"%s\n\n" % url_for("router_info", dbid=dbid, _external=True) +
+									"Regards,\nFreifunk Franken Monitoring System"
+						)
+					flash("<b>Router reported to administrators!</b>", "success")
+		else:
+			mysql.close()
+			return "Router not found"
+		
+		router["blocked"] = mysql.findone("""
+			SELECT blocked.mac
+			FROM router_netif AS n
+			LEFT JOIN blocked ON n.mac = blocked.mac
+			WHERE n.router = %s AND n.netif = 'br-mesh'
+		""",(dbid,),"mac")
+		mysql.close()
+		
+		return render_template("router.html",
+			router = router,
+			mac = mac,
+			tileurls = tileurls,
+			neighstats = neighdata,
+			neighlabel = neighlabel,
+			gwstats = gwfetch,
+			authuser = is_authorized(router["user"], session),
+			authadmin = session.get('admin')
+			)
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_router.txt", str(e))
+		import traceback
+		writefulllog("Warning: Failed to display router details page: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
 
 @app.route('/users')
 def user_list():
+	mysql = FreifunkMySQL()
+	users = mysql.fetchall("SELECT id, nickname, email, created, admin FROM users ORDER BY nickname COLLATE utf8_unicode_ci ASC")
+	user_routers = stattools.router_user_sum(mysql)
+	usersv2 = users_v2(mysql)
+	mysql.close()
+	users = mysql.utcawaretuple(users,"created")
+
 	return render_template("user_list.html",
-		user_routers = stattools.router_user_sum(),
-		users = db.users.find({}, {"nickname": 1, "email": 1, "created": 1, "admin": 1}).sort("nickname", pymongo.ASCENDING)
+		user_routers = user_routers,
+		users = users,
+		users_count = len(users),
+		users_v2 = usersv2
 	)
 
 @app.route('/users/<nickname>', methods=['GET', 'POST'])
 def user_info(nickname):
-	try:
-		user = db.users.find_one({"nickname": nickname})
-		assert user
-	except AssertionError:
+	mysql = FreifunkMySQL()
+	user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+	user["created"] = mysql.utcaware(user["created"])
+	if not user:
+		mysql.close()
 		return "User not found"
 	if request.method == 'POST':
-		if is_authorized(user["nickname"], session):
-			if request.form.get("action") == "changepw":
+		if request.form.get("action") == "changepw":
+			if is_authorized(user["nickname"], session):
 				if request.form["password"] != request.form["password_rep"]:
 					flash("<b>Passwords did not match!</b>", "danger")
 				elif request.form["password"] == "":
 					flash("<b>Password must not be empty!</b>", "danger")
 				else:
-					set_user_password(user["nickname"], request.form["password"])
+					set_user_password(mysql, user["nickname"], request.form["password"])
 					flash("<b>Password changed!</b>", "success")
-			elif request.form.get("action") == "changemail":
+			else:
+				flash("<b>You are not authorized to perform this action!</b>", "danger")
+		elif request.form.get("action") == "changemail":
+			if is_authorized(user["nickname"], session):
 				if request.form["email"] != request.form["email_rep"]:
 					flash("<b>E-Mail addresses do not match!</b>", "danger")
 				elif not "@" in request.form["email"]:
 					flash("<b>Invalid E-Mail addresse!</b>", "danger")
 				else:
 					try:
-						set_user_email(user["nickname"], request.form["email"])
+						set_user_email(mysql, user["nickname"], request.form["email"])
 						flash("<b>E-Mail changed!</b>", "success")
 						if not session.get('admin'):
 							password = base64.b32encode(os.urandom(10)).decode()
-							set_user_password(user["nickname"], password)
+							set_user_password(mysql, user["nickname"], password)
 							send_email(
 								recipient = request.form['email'],
 								subject   = "Password for %s" % user['nickname'],
 								content   = "Hello %s,\n\n" % user["nickname"] +
-								            "You changed your email address on https://monitoring.freifunk-franken.de/\n" +
-									    "To verify your new email address your password was changed to %s\n" % password +
-									    "... and sent to your new address. Please log in and change it.\n\n" +
-									    "Regards,\nFreifunk Franken Monitoring System"
+											"You changed your email address on https://monitoring.freifunk-franken.de/\n" +
+										"To verify your new email address your password was changed to %s\n" % password +
+										"... and sent to your new address. Please log in and change it.\n\n" +
+										"Regards,\nFreifunk Franken Monitoring System"
 							)
+							mysql.close()
 							return logout()
 						else:
 							# force db data reload
-							user = db.users.find_one({"nickname": nickname})
+							user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+							user["created"] = mysql.utcaware(user["created"])
 					except AccountWithEmailExists:
 						flash("<b>There is already an account with this E-Mail Address!</b>", "danger")
-			elif request.form.get("action") == "changeadmin":
-				if session.get('admin'):
-					set_user_admin(nickname, request.form.get("admin") == "true")
-					# force db data reload
-					user = db.users.find_one({"nickname": nickname})
-			elif request.form.get("action") == "deleteaccount":
-				if session.get('admin'):
-					db.users.delete_one({"nickname": nickname})
-					flash("<b>User <i>%s</i> deleted!</b>" % nickname, "success")
-					return redirect(url_for("user_list"))
-		else:
-			flash("<b>You are not authorized to perform this action!</b>", "danger")
-	routers=db.routers.find({"user._id": user["_id"]}, {
-		"hostname": 1,
-		"status": 1,
-		"hood": 1,
-		"software.firmware": 1,
-		"hardware.name": 1,
-		"created": 1,
-		"system.uptime": 1,
-		"system.clients": 1,
-	}).sort("hostname", pymongo.ASCENDING)
-	return render_template("user.html", user=user, routers=routers)
+			else:
+				flash("<b>You are not authorized to perform this action!</b>", "danger")
+		elif request.form.get("action") == "changeadmin":
+			if session.get('admin'):
+				set_user_admin(mysql, nickname, request.form.get("admin") == "true")
+				# force db data reload
+				user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+				user["created"] = mysql.utcaware(user["created"])
+			else:
+				flash("<b>You are not authorized to perform this action!</b>", "danger")
+		elif request.form.get("action") == "changeabuse":
+			if session.get('admin'):
+				set_user_abuse(mysql, nickname, request.form.get("abuse") == "true")
+				# force db data reload
+				user = mysql.findone("SELECT * FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+				user["created"] = mysql.utcaware(user["created"])
+			else:
+				flash("<b>You are not authorized to perform this action!</b>", "danger")
+		elif request.form.get("action") == "deleteaccount":
+			if is_authorized(user["nickname"], session):
+				mysql.execute("DELETE FROM users WHERE nickname = %s LIMIT 1",(nickname,))
+				mysql.commit()
+				flash("<b>User <i>%s</i> deleted!</b>" % nickname, "success")
+				mysql.close()
+				if user["nickname"] == session.get("user"):
+					session.pop('user', None)
+				return redirect(url_for("user_list"))
+			else:
+				flash("<b>You are not authorized to perform this action!</b>", "danger")
+	routers = mysql.fetchall("""
+		SELECT router.id, hostname, status, hoods.id AS hoodid, hoods.name AS hood, firmware, hardware, created, sys_uptime, clients, router.lat, router.lng, reset, blocked, v2, local
+		FROM router
+		INNER JOIN hoods ON router.hood = hoods.id
+		LEFT JOIN (
+			SELECT router, blocked.mac AS blocked FROM router_netif
+			INNER JOIN blocked ON router_netif.mac = blocked.mac
+			WHERE netif = 'br-mesh'
+		) AS b
+		ON router.id = b.router
+		WHERE contact = %s
+		ORDER BY hostname ASC
+	""",(user["email"],))
+	mysql.close()
+	routers = mysql.utcawaretuple(routers,"created")
+	return render_template("user.html",
+		user=user,
+		routers=routers,
+		routers_count=len(routers),
+		authuser = is_authorized(user["nickname"], session),
+		authadmin = session.get('admin')
+	)
 
 @app.route('/statistics')
 def global_statistics():
-	hoods = stattools.hoods()
-	return render_template("statistics.html",
-		stats = db.stats.find({}, {"_id": 0}),
-		clients = stattools.total_clients(),
-		router_status = stattools.router_status(),
-		router_models = stattools.router_models(),
-		router_firmwares = stattools.router_firmwares(),
-		hoods = hoods,
-		hoods_sum = stattools.hoods_sum(),
-		newest_routers = db.routers.find({"hardware.name": {"$ne": "Legacy"}}, {"hostname": 1, "hood": 1, "created": 1}).sort("created", pymongo.DESCENDING).limit(len(hoods)+1)
-	)
+	mysql = FreifunkMySQL()
+	stats = mysql.fetchall("SELECT * FROM stats_global")
+	return helper_statistics(mysql,stats,None,None)
+
+@app.route('/hoodstatistics/<selecthood>')
+def global_hoodstatistics(selecthood):
+	selecthood = int(selecthood)
+	mysql = FreifunkMySQL()
+	stats = mysql.fetchall("SELECT * FROM stats_hood WHERE hood = %s",(selecthood,))
+	return helper_statistics(mysql,stats,selecthood,None)
+
+@app.route('/gwstatistics/<selectgw>')
+def global_gwstatistics(selectgw):
+	mysql = FreifunkMySQL()
+	stats = mysql.fetchall("SELECT * FROM stats_gw WHERE mac = %s",(mac2int(selectgw),))
+	selectgw = shortmac2mac(selectgw)
+	return helper_statistics(mysql,stats,None,selectgw)
+
+def helper_statistics(mysql,stats,selecthood,selectgw):
+	try:
+		hoods = stattools.hoods(mysql,selectgw)
+		gws = stattools.gws_ifs(mysql,selecthood)
+		
+		if selecthood:
+			selecthoodname = mysql.findone("SELECT name FROM hoods WHERE id = %s",(selecthood,),'name')
+		else:
+			selecthoodname = None
+		
+		if selectgw:
+			selectgwint = mac2int(selectgw)
+		else:
+			selectgwint = None
+		
+		if selecthood and not selecthoodname:
+			mysql.close()
+			return "Hood not found"
+		if selectgw and not selectgwint in gws:
+			mysql.close()
+			return "Gateway not found"
+		
+		stats = mysql.utcawaretupleint(stats,"time")
+		
+		numnew = len(hoods)-27
+		if numnew < 1:
+			numnew = 1
+		
+		if selectgw:
+			newest_routers = mysql.fetchall("""
+				SELECT router.id, hostname, hoods.id AS hoodid, hoods.name AS hood, created
+				FROM router
+				INNER JOIN hoods ON router.hood = hoods.id
+				INNER JOIN router_gw ON router.id = router_gw.router
+				WHERE hardware <> 'Legacy' AND mac = %s
+				ORDER BY created DESC
+				LIMIT %s
+			""",(mac2int(selectgw),numnew,))
+		else:
+			if selecthood:
+				where = " AND hoods.id = %s"
+				tup = (selecthood,numnew,)
+			else:
+				where = ""
+				tup = (numnew,)
+			newest_routers = mysql.fetchall("""
+				SELECT router.id, hostname, hoods.id AS hoodid, hoods.name AS hood, created
+				FROM router
+				INNER JOIN hoods ON router.hood = hoods.id
+				WHERE hardware <> 'Legacy' {}
+				ORDER BY created DESC
+				LIMIT %s
+			""".format(where),tup)
+		newest_routers = mysql.utcawaretuple(newest_routers,"created")
+		
+		clients = stattools.total_clients(mysql)
+		router_status = stattools.router_status(mysql)
+		router_models = stattools.router_models(mysql,selecthood,selectgw)
+		router_firmwares = stattools.router_firmwares(mysql,selecthood,selectgw)
+		hoods_sum = stattools.hoods_sum(mysql,selectgw)
+		hoods_gws = stattools.hoods_gws(mysql)
+		gws_sum = stattools.gws_sum(mysql,selecthood)
+		gws_info = stattools.gws_info(mysql,selecthood)
+		gws_admin = stattools.gws_admin(mysql,selectgw)
+		mysql.close()
+		
+		return render_template("statistics.html",
+			selecthood = selecthood,
+			selecthoodname = selecthoodname,
+			selectgw = selectgw,
+			selectgwint = selectgwint,
+			stats = stats,
+			clients = clients,
+			router_status = router_status,
+			router_models = router_models,
+			router_firmwares = router_firmwares,
+			hoods = hoods,
+			hoods_sum = hoods_sum,
+			hoods_gws = hoods_gws,
+			newest_routers = newest_routers,
+			gws = gws,
+			gws_sum = gws_sum,
+			gws_info = gws_info,
+			gws_admin = gws_admin
+		)
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_stats.txt", str(e))
+		import traceback
+		writefulllog("Warning: Failed to display stats page: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
+
+@app.route('/gateways')
+def gateways():
+	try:
+		mysql = FreifunkMySQL()
+		gws = stattools.gateways(mysql)
+		ipv4 = stattools.gws_ipv4(mysql)
+		ipv6 = stattools.gws_ipv6(mysql)
+		dhcp = stattools.gws_dhcp(mysql)
+		mysql.close()
+		
+		return render_template("gws.html",
+			gws = gws,
+			ipv4 = ipv4,
+			ipv6 = ipv6,
+			dhcp = dhcp
+		)
+	except Exception as e:
+		writelog(CONFIG["debug_dir"] + "/fail_gateways.txt", str(e))
+		import traceback
+		writefulllog("Warning: Failed to display gateways page: %s\n__%s" % (e, traceback.format_exc().replace("\n", "\n__")))
 
 @app.route('/register', methods=['GET', 'POST'])
 def register():
@@ -181,16 +652,18 @@ def register():
 				recipient = request.form['email'],
 				subject   = "Password for %s" % request.form['user'],
 				content   = "Hello %s,\n\n" % request.form['user'] +
-					    "You created an account on https://monitoring.freifunk-franken.de/\n" +
-					    "To verify your new email address your password was autogenerated to %s\n" % password +
-					    "... and sent to your address. Please log in and change it.\n\n" +
-					    "Regards,\nFreifunk Franken Monitoring System"
+						"You created an account on https://monitoring.freifunk-franken.de/\n" +
+						"To verify your new email address your password was autogenerated to %s\n" % password +
+						"... and sent to your address. Please log in and change it.\n\n" +
+						"Regards,\nFreifunk Franken Monitoring System"
 			)
 			flash("<b>Registration successful!</b> - Your password was sent to %s" % request.form['email'], "success")
 		except AccountWithEmailExists:
 			flash("<b>There is already an account with this E-Mail Address!</b>", "danger")
 		except AccountWithNicknameExists:
 			flash("<b>There is already an active account with this Nickname!</b>", "danger")
+		except AccountWithEmptyField:
+			flash("<b>Please fill all fields!</b>", "danger")
 	return render_template("register.html")
 
 @app.route('/resetpw', methods=['GET', 'POST'])
@@ -198,31 +671,33 @@ def resetpw():
 	try:
 		if request.method == 'POST':
 			token = base64.b32encode(os.urandom(10)).decode()
-			user = db.users.find_one({"email": request.form['email']})
-			reset_user_password(request.form['email'], token)
+			mysql = FreifunkMySQL()
+			user = reset_user_password(mysql, request.form['email'], token)
+			mysql.close()
 			send_email(
 				recipient = request.form['email'],
 				subject   = "Password reset link",
 				content   = "Hello %s,\n\n" % user["nickname"] +
-				            "You attemped to reset your password on https://monitoring.freifunk-franken.de/\n" +
-					    "To verify you a reset link was sent to you:\n" +
-				            "%s\n" % url_for('resetpw', email=request.form['email'], token=token, _external=True) +
-					    "Clicking this link will reset your password and send the new password to your email address.\n\n" +
-					    "Regards,\nFreifunk Franken Monitoring System"
+							"You attemped to reset your password on https://monitoring.freifunk-franken.de/\n" +
+							"To verify you a reset link was sent to you:\n" +
+							"%s\n" % url_for('resetpw', email=request.form['email'], token=token, _external=True) +
+							"Clicking this link will reset your password and send the new password to your email address.\n\n" +
+							"Regards,\nFreifunk Franken Monitoring System"
 			)
 			flash("<b>A password reset link was sent to %s</b>" % request.form['email'], "success")
 		elif "token" in request.args:
 			password = base64.b32encode(os.urandom(10)).decode()
-			reset_user_password(request.args['email'], request.args['token'], password)
-			user = db.users.find_one({"email": request.args['email']})
+			mysql = FreifunkMySQL()
+			user = reset_user_password(mysql, request.args['email'], request.args['token'], password)
+			mysql.close()
 			send_email(
 				recipient = request.args['email'],
 				subject   = "Your new Password",
 				content   = "Hello %s,\n\n" % user["nickname"] +
-				            "You attemped to reset your password on https://monitoring.freifunk-franken.de/\n" +
-				            "Your new Password: %s\n" % password +
-					    "Please log in and change it\n\n" +
-					    "Regards,\nFreifunk Franken Monitoring System"
+							"You attemped to reset your password on https://monitoring.freifunk-franken.de/\n" +
+							"Your new Password: %s\n" % password +
+							"Please log in and change it\n\n" +
+							"Regards,\nFreifunk Franken Monitoring System"
 			)
 			flash("<b>Password reset successful!</b> - Your password was sent to %s" % request.args['email'], "success")
 	except AccountNotExisting:
@@ -249,6 +724,7 @@ def login():
 @app.route('/logout')
 def logout():
 	session.pop('user', None)
+	session.pop('admin', None)
 	return redirect(request.referrer or url_for("index"))
 
 

ffmap/web/filters.py

@@ -8,32 +8,81 @@ import sys
 import json
 import datetime
 import re
-import pymongo
 import hashlib
+from ffmap.misc import int2mac, int2shortmac, inttoipv4, bintoipv6
+from ipaddress import ip_address
 
 sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))
 from ffmap.misc import *
 
 filters = Blueprint("filters", __name__)
 
-@filters.app_template_filter('neighbour_color')
-def neighbour_color(quality):
-	color = "#04ff0a"
-	if quality == -1:
-		color = "#0684c4"
-	elif quality < 105:
-		color = "#ff1e1e"
-	elif quality < 130:
-		color = "#ff4949"
-	elif quality < 155:
-		color = "#ff6a6a"
-	elif quality < 180:
-		color = "#ffac53"
-	elif quality < 205:
-		color = "#ffeb79"
-	elif quality < 230:
-		color = "#79ff7c"
-	return color
+@filters.app_template_filter('sumdict')
+def sumdict(d):
+	return sum(d.values())
+
+@filters.app_template_filter('v2userpercent')
+def v2formatpercent(d):
+	return "{:.0f}".format(v2numberpercent(d))
+
+def v2numberpercent(d):
+	if d.get("v1",0) > 0 or d.get("v2",0) > 0:
+		return d["v2"] * 100 / ( d["v1"] + d["v2"] )
+	else:
+		return 0.0
+
+@filters.app_template_filter('v2colorpercent')
+def v2colorpercent(d):
+	pc = v2numberpercent(d)
+	color = "000000"
+	if pc > 99:
+		color = "008800"
+	elif pc > 75:
+		color = "00d93d"
+	elif pc > 50:
+		color = "ffc926"
+	elif pc > 25:
+		color = "ff9326"
+	elif pc > 1:
+		color = "ff0000"
+	return "color:#" + color
+
+@filters.app_template_filter('longip')
+def longip(d):
+	if len(d) > 32:
+		return d.replace('::','::... ...::')
+	else:
+		return d
+
+@filters.app_template_filter('int2mac')
+def int2macfilter(d):
+	return int2mac(d)
+
+@filters.app_template_filter('int2shortmac')
+def int2shortmacfilter(d):
+	return int2shortmac(d)
+
+@filters.app_template_filter('int2ipv4')
+def int2ipv4filter(d):
+	return inttoipv4(d)
+
+@filters.app_template_filter('bin2ipv6')
+def bin2ipv6filter(d):
+	return bintoipv6(d)
+
+@filters.app_template_filter('ip2int')
+def ip2intfilter(d):
+	try:
+		return int(ip_address(d))
+	except ValueError as e:
+		return 0
+
+@filters.app_template_filter('ipnet2int')
+def ipnet2intfilter(d):
+	try:
+		return int(ip_address(d.split("/")[0]))
+	except ValueError as e:
+		return 0
 
 @filters.app_template_filter('utc2local')
 def utc2local(dt):
@@ -99,8 +148,6 @@ def bson_to_json(bsn):
 
 @filters.app_template_filter('statbson2json')
 def statbson_to_json(bsn):
-	if isinstance(bsn, pymongo.cursor.Cursor):
-		bsn = list(bsn)
 	for point in bsn:
 		point["time"] = {"$date": int(point["time"].timestamp()*1000)}
 	return json.dumps(bsn)
@@ -113,9 +160,18 @@ def nbsp(txt):
 def humanize_bytes(num, suffix='B'):
 	for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 		if abs(num) < 1024.0 and unit != '':
-			return "%3.1f%s%s" % (num, unit, suffix)
+			return "%3.1f %s%s" % (num, unit, suffix)
 		num /= 1024.0
-	return "%.1f%s%s" % (num, 'Yi', suffix)
+	return "%.1f %s%s" % (num, 'Yi', suffix)
+
+@filters.app_template_filter('bytes_to_bits')
+def bytes_to_bits(num, suffix='b'):
+	num *= 8.0
+	for unit in ['','k','M','G','T','P','E','Z']:
+		if abs(num) < 1000.0 and unit != '':
+			return "%3.1f %s%s" % (num, unit, suffix)
+		num /= 1000.0
+	return "%.1f %s%s" % (num, 'Y', suffix)
 
 @filters.app_template_filter('mac2fe80')
 def mac_to_ipv6_linklocal(mac):
@@ -124,7 +180,12 @@ def mac_to_ipv6_linklocal(mac):
 
 	# Remove the most common delimiters; dots, dashes, etc.
 	mac_bare = re.sub('[%s]+' % re.escape(' .:-'), '', mac)
-	mac_value = int(mac_bare, 16)
+	return macint_to_ipv6_linklocal(int(mac_bare, 16))
+
+@filters.app_template_filter('macint2fe80')
+def macint_to_ipv6_linklocal(mac_value):
+	if not mac_value:
+		return ''
 
 	# Split out the bytes that slot into the IPv6 address
 	# XOR the most significant byte with 0x02, inverting the
@@ -146,6 +207,8 @@ def status2css(status):
 		"created": "primary",
 		"netmon": "primary",
 		"update": "primary",
+		"orphaned": "default",
+		"admin": "warning",
 	}
 	return "label label-%s" % status_map.get(status, "default")
 
@@ -177,13 +240,23 @@ def gravatar_url(email):
 
 @filters.app_template_filter('webui_addr')
 def webui_addr(router_netifs):
-	try:
-		for br_mesh in filter(lambda n: n["name"] == "br-mesh", router_netifs):
-			for ipv6 in br_mesh["ipv6_addrs"]:
-				if ipv6.startswith("fd") and len(ipv6) > 25:
-					# This selects the first ULA address, if present
-					return ipv6
-				if ipv6.startswith("fdff") and len(ipv6) > 15 and len(ipv6) <= 20:
-					return ipv6
-	except (KeyError, TypeError):
-		return None
+	for br_mesh in filter(lambda n: n["netif"] == "br-mesh", router_netifs):
+		for ipv6 in br_mesh["ipv6_addrs"]:
+			ipv6 = bintoipv6(ipv6)
+			if not ipv6:
+				return None
+			if ipv6.startswith("fd43"):
+				# This selects the first ULA address, if present
+				return ipv6
+			if ipv6.startswith("fdff") and len(ipv6) > 10:
+				# This selects the first fdff address, if present (and skips fdff::1)
+				return ipv6
+	return None
+
+@filters.app_template_filter('format_airtime')
+def format_airtime(airtime):
+	return "%.0f %%" % (airtime*100)
+
+@filters.app_template_filter('format_query')
+def format_query(query):
+	return query.replace(" ","_").replace(".","\.").replace("(","\(").replace(")","\)")

ffmap/web/helpers.py

@@ -18,20 +18,33 @@ def format_query(query_usr):
 allowed_filters = (
 	'status',
 	'hood',
-	'community',
-	'user.nickname',
-	'hardware.name',
-	'software.firmware',
-	'netifs.mac',
-	'netifs.name',
-	'netmon_id',
+	'nickname',
+	'hardware',
+	'firmware',
+	'mac',
 	'hostname',
-	'system.contact',
+	'contact',
+	'community',
+	'neighbor',
+	'neighbour',
+	'gw',
+	'selected',
+	'bat',
+	'batselected',
+	'network',
+	'os',
+	'batman',
+	'kernel',
+	'nodewatcher',
 )
+
 def parse_router_list_search_query(args):
 	query_usr = bson.SON()
 	if "q" in args:
 		for word in args["q"].strip().split(" "):
+			if not word:
+				# Case of "q=" without arguments
+				break
 			if not ':' in word:
 				key = "hostname"
 				value = word
@@ -39,31 +52,91 @@ def parse_router_list_search_query(args):
 				key, value = word.split(':', 1)
 			if key in allowed_filters:
 				query_usr[key] = query_usr.get(key, "") + value
-	query = {}
+	s = ""
+	j = ""
+	t = []
+	i = 0
 	for key, value in query_usr.items():
-		if value == "EXISTS":
-			query[key] = {"$exists": True}
-		elif value == "EXISTS_NOT":
-			query[key] = {"$exists": False}
-		elif key == 'netifs.mac':
-			query[key] = value.lower()
-		elif key == 'netifs.name':
-			query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'}
-		elif key == 'hostname':
-			query[key] = {"$regex": value.replace('.', '\.'), "$options": 'i'}
-		elif key == 'hardware.name':
-			query[key] = {"$regex": value.replace('.', '\.').replace('_', ' '), "$options": 'i'}
-		elif key == 'netmon_id':
-			query[key] = int(value)
-		elif key == 'system.contact':
-			if not '\.' in value:
-				value = re.escape(value)
-			query[key] = {"$regex": value, "$options": 'i'}
-		elif value.startswith('!'):
-			query[key] = {"$ne": value.replace('!', '', 1)}
+		if i==0:
+			prefix = " WHERE "
 		else:
-			query[key] = value
-	return (query, format_query(query_usr))
+			prefix = " AND "
+		if value.startswith('!'):
+			no = "NOT "
+			value = value[1:]
+		else:
+			no = ""
+		
+		if value == "EXISTS":
+			k = key + ' <> "" AND ' + key + " IS NOT NULL"
+		elif value == "EXISTS_NOT":
+			k = key + ' = "" OR ' + key + " IS NULL"
+		elif key == 'mac':
+			j += " INNER JOIN ( SELECT router, mac FROM router_netif GROUP BY router, mac) AS j ON router.id = j.router "
+			k = "HEX(mac) {} REGEXP %s".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'gw'):
+			j += " INNER JOIN router_gw ON router.id = router_gw.router "
+			k = "HEX(router_gw.mac) {} REGEXP %s".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'selected'):
+			j += " INNER JOIN router_gw ON router.id = router_gw.router "
+			k = "HEX(router_gw.mac) {} REGEXP %s AND router_gw.selected = TRUE".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'bat'):
+			j += """ INNER JOIN router_gw ON router.id = router_gw.router
+				INNER JOIN (
+					gw_netif AS n1
+					INNER JOIN gw_netif AS n2 ON n1.mac = n2.vpnmac AND n1.gw = n2.gw
+				) ON router_gw.mac = n1.mac
+			"""
+			k = "HEX(n2.mac) {} REGEXP %s".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'batselected'):
+			j += """ INNER JOIN router_gw ON router.id = router_gw.router
+				INNER JOIN (
+					gw_netif AS n1
+					INNER JOIN gw_netif AS n2 ON n1.mac = n2.vpnmac AND n1.gw = n2.gw
+				) ON router_gw.mac = n1.mac
+			"""
+			k = "HEX(n2.mac) {} REGEXP %s AND router_gw.selected = TRUE".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'neighbor') or (key == 'neighbour'):
+			j += " INNER JOIN ( SELECT router, mac FROM router_neighbor GROUP BY router, mac) AS j ON router.id = j.router "
+			k = "HEX(mac) {} REGEXP %s".format(no)
+			t.append(value.replace(':',''))
+		elif (key == 'hood'):
+			k = "hoods.name {} REGEXP %s".format(no)
+			t.append(value.replace("_","."))
+		elif (key == 'hardware') or (key == 'nickname'):
+			k = key + " {} REGEXP %s".format(no)
+			t.append(value.replace("_","."))
+		elif (key == 'hostname') or (key == 'firmware'):
+			k = key + " {} REGEXP %s".format(no)
+			t.append(value)
+		elif key == 'contact':
+			k = "contact {} REGEXP %s".format(no)
+			t.append(value)
+		elif key == 'network':
+			# local hood included for v2
+			if value.lower() == 'local':
+				k = no + " (router.v2 = TRUE AND local = TRUE)"
+			elif value.lower() == 'v2':
+				k = no + " (router.v2 = TRUE AND local = FALSE)"
+			elif value.lower() == 'v1':
+				k = no + " router.v2 = FALSE"
+			else:
+				continue
+		elif key in ('os','batman','kernel','nodewatcher',):
+			k = key + " {} REGEXP %s".format(no)
+			t.append(value.replace("_","."))
+		else:
+			k = no + key + " = %s"
+			t.append(value)
+		i += 1
+		s += prefix + k
+	where = j + " " + s
+	return (where, tuple(t), format_query(query_usr))
 
 def send_email(recipient, subject, content, sender="FFF Monitoring <noreply@monitoring.freifunk-franken.de>"):
 	msg = MIMEText(content)

ffmap/web/static/css/style.css

@@ -14,6 +14,9 @@
 .popup-headline.with-neighbours {
 	border-bottom: 1px solid lightgray;
 }
+.popup-latlng {
+	font-size:14px;
+}
 table.neighbours td {
 	padding: 0 3px;
 }
@@ -34,3 +37,26 @@ table.neighbours {
 	padding-left: 5px;
 	padding-right: 7px;
 }
+.graph-pie {
+	height: 250px;
+	width: 100%;
+}
+.graph-pie .legendLabel {
+	padding-left: 5px;
+	padding-right: 7px;
+}
+
+.hoodv2 {
+	color: #2db200;
+}
+.hoodlocal {
+	color: #ffbf00;
+}
+
+.hoodv2 a {
+	color: #2db200;
+}
+.hoodlocal a {
+	color: #ffbf00;
+}
+

ffmap/web/static/img/router_blue_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_blue.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="45.254834"
+     inkscape:cx="6.1575261"
+     inkscape:cy="6.8281194"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#123cff;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_green.svg

@@ -56,7 +56,7 @@
      id="layer1"
      transform="translate(0,-1038.3621)">
     <circle
-       style="fill:#2fa034;fill-opacity:1"
+       style="fill:#0f7014;fill-opacity:1"
        id="path4690"
        cx="7"
        cy="1045.3621"

ffmap/web/static/img/router_green_v2.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_green_v2.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#2fbb34;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#000000;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_green_v2_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_green_v2.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#2fbb34;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_green_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_green.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#0f7014;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_grey.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_grey.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#999999;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#000000;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_grey_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_grey.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#999999;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_red.svg

@@ -56,7 +56,7 @@
      id="layer1"
      transform="translate(0,-1038.3621)">
     <circle
-       style="fill:#dd0c0c;fill-opacity:0.99607843"
+       style="fill:#cc0c0c;fill-opacity:0.99607843"
        id="path4690"
        cx="7"
        cy="1045.3621"

ffmap/web/static/img/router_red_v2.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_red_v2.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="11.524549"
+     inkscape:cy="7.6508884"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#ff6666;fill-opacity:0.99607843"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#000000;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_red_v2_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_red_v2.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="11.524549"
+     inkscape:cy="7.6508884"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#ff6666;fill-opacity:0.99607843"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_red_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_red.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="11.524549"
+     inkscape:cy="7.6508884"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#cc0c0c;fill-opacity:0.99607843"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/img/router_yellow_white.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="14"
+   height="14"
+   viewBox="0 0 14 14"
+   id="svg4142"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="router_yellow.svg">
+  <defs
+     id="defs4144" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="3.7021802"
+     inkscape:cy="7.5625001"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1436"
+     inkscape:window-height="858"
+     inkscape:window-x="0"
+     inkscape:window-y="19"
+     inkscape:window-maximized="1"
+     width="14in" />
+  <metadata
+     id="metadata4147">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1038.3621)">
+    <circle
+       style="fill:#ffea12;fill-opacity:1"
+       id="path4690"
+       cx="7"
+       cy="1045.3621"
+       r="6.6121397" />
+    <circle
+       style="fill:#ffffff;fill-opacity:1"
+       id="path4134"
+       cx="7"
+       cy="1045.3621"
+       r="2.0780513" />
+  </g>
+</svg>

ffmap/web/static/js/graph.js

@@ -1,5 +1,5 @@
 var points_per_px = 0.3;
-var controls_container = "<div style='right:60px;top:13px;position:absolute;display:none;' id='controls'></div>";
+var controls_container = "<div style='position:absolute;left:0;bottom:0;display:none;' id='controls'></div>";
 var reset_button = "<div class='btn btn-default btn-xs'>Reset</div>";
 
 function labelFormatter(label, series) {
@@ -37,7 +37,7 @@ function setup_plot_zoom(plot, pdata, num_data_points) {
 		plot.draw();
 		plot.clearSelection();
 		plot.getPlaceholder().children("#controls")
-			.css("top", (plot.getPlotOffset().top+5) + "px")
+			.css("bottom", (plot.getPlotOffset().bottom+5) + "px")
 			.css("left", (plot.getPlotOffset().left+5) + "px")
 			.css("display", "block");
 	});
@@ -55,29 +55,29 @@ function setup_plot_zoom(plot, pdata, num_data_points) {
 			plot.setupGrid();
 			plot.draw();
 			plot.getPlaceholder().children("#controls")
-				.css("top", (plot.getPlotOffset().top+5) + "px")
+				.css("bottom", (plot.getPlotOffset().bottom+5) + "px")
 				.css("left", (plot.getPlotOffset().left+5) + "px")
 				.css("display", "none");
 		});
 	plot.getPlaceholder().children("#controls")
-		.css("top", (plot.getPlotOffset().top+5) + "px")
+		.css("bottom", (plot.getPlotOffset().bottom+5) + "px")
 		.css("left", (plot.getPlotOffset().left+5) + "px");
 }
 
 // Per router statistics
 
-function network_graph(netif) {
-	var netstat = $("#netstat");
+function network_graph(netif_stats, field, tx_label, rx_label) {
+	var netstat = $("#"+field);
 	var tx = [], rx = [];
 	var len, i;
-	for (len=router_stats.length, i=0; i<len; i++) {
+	for (len=netif_stats.length, i=0; i<len; i++) {
 		try {
-			var tx_value = router_stats[i].netifs[netif].tx;
-			var rx_value = router_stats[i].netifs[netif].rx;
-			var date_value = router_stats[i].time.$date;
+			var tx_value = netif_stats[i].tx;
+			var rx_value = netif_stats[i].rx;
+			var date_value = netif_stats[i].time.$date;
 			if(tx_value != null && rx_value != null) {
-				tx.push([date_value, tx_value]);
-				rx.push([date_value, rx_value]);
+				tx.push([date_value, tx_value * 8]);
+				rx.push([date_value, rx_value * 8]);
 			}
 		}
 		catch(TypeError) {
@@ -85,40 +85,80 @@ function network_graph(netif) {
 		}
 	}
 	var pdata = [
-		{"label": "tx", "data": tx, "color": "#CB4B4B"},
-		{"label": "rx", "data": rx, "color": "#8CACC6"}
+		{"label": tx_label, "data": tx, "color": "#CB4B4B"},
+		{"label": rx_label, "data": rx, "color": "#8CACC6"}
 	]
 	var plot = $.plot(netstat, pdata, {
 		xaxis: {mode: "time", timezone: "browser"},
 		selection: {mode: "x"},
-		yaxis: {min: 0, mode: "byteRate"},
+		yaxis: {min: 0, mode: "bitRate"},
 		legend: {noColumns: 2, hideable: true},
 		series: {downsample: {threshold: Math.floor(netstat.width() * points_per_px)}}
 	});
 	setup_plot_zoom(plot, pdata, len);
 }
 
-function neighbour_graph(neighbours) {
+function neighbour_graph(neigh_label) {
 	var meshstat = $("#meshstat");
 	var pdata = [];
-	for (j=0; j<neighbours.length; j++) {
-		var label = neighbours[j].name;
+	var len, i;
+	
+	for (var j in neigh_stats) {
+		var dataset = neigh_stats[j];
+		var label = j;
+		var data = [];
+		if(j in neigh_label) {
+			label = neigh_label[j];
+		}
+		for (len=dataset.length, i=0; i<len; i++) {
+			try {
+				var quality = dataset[i].quality;
+				var date_value = dataset[i].time.$date;
+				if(quality == null) {
+					quality = 0;
+				}
+				data.push([date_value, Math.abs(quality)]);
+			}
+			catch(TypeError) {
+				// pass
+			}
+		}
+		pdata.push({"label": label, "data": data});
+	}
+	if(pdata.length == 0) { pdata.push({"label": "empty", "data": []}); }
+	var plot = $.plot(meshstat, pdata, {
+		xaxis: {mode: "time", timezone: "browser"},
+		selection: {mode: "x"},
+		yaxis: {min: 0, autoscaleMargin: 0.5},
+		legend: {noColumns: 3, hideable: true},
+		series: {downsample: {threshold: Math.floor(meshstat.width() * points_per_px)}}
+	});
+	setup_plot_zoom(plot, pdata, len);
+	return true;
+}
+
+function gw_graph(gws) {
+	var gwstat = $("#gwstat");
+	var pdata = [];
+	for (j=0; j<gws.length; j++) {
+		var label = gws[j].name;
 
 		// add network interface when there are multiple links to same node
 		var k;
-		for(k=0; k<neighbours.length; k++) {
-			if(label == neighbours[k].name && k != j) {
-				label += "@" + neighbours[j].net_if;
+		for(k=0; k<gws.length; k++) {
+			if(label == gws[k].name && k != j) {
+				label += "@" + gws[j].netif;
 			}
 		}
 
-		var mac = neighbours[j].mac;
+		var mac = gws[j].mac;
 		var data = [];
 		var len, i;
-		for (len=router_stats.length, i=0; i<len; i++) {
+		for (len=gw_stats.length, i=0; i<len; i++) {
+			if (gw_stats[i].mac != mac) { continue; }
 			try {
-				var quality = router_stats[i].neighbours[mac];
-				var date_value = router_stats[i].time.$date;
+				var quality = gw_stats[i].quality;
+				var date_value = gw_stats[i].time.$date;
 				if(quality == null) {
 					quality = 0;
 				}
@@ -130,12 +170,12 @@ function neighbour_graph(neighbours) {
 		}
 		pdata.push({"label": label, "data": data});
 	}
-	var plot = $.plot(meshstat, pdata, {
+	var plot = $.plot(gwstat, pdata, {
 		xaxis: {mode: "time", timezone: "browser"},
 		selection: {mode: "x"},
-		yaxis: {min: 0, max: 400},
+		yaxis: {min: 0, max: 350},
 		legend: {noColumns: 2, hideable: true},
-		series: {downsample: {threshold: Math.floor(meshstat.width() * points_per_px)}}
+		series: {downsample: {threshold: Math.floor(gwstat.width() * points_per_px)}}
 	});
 	setup_plot_zoom(plot, pdata, len);
 }
@@ -146,9 +186,9 @@ function memory_graph() {
 	var len, i;
 	for (len=router_stats.length, i=0; i<len; i++) {
 		try {
-			var free_value = router_stats[i].memory.free*1024;
-			var caching_value = router_stats[i].memory.caching*1024;
-			var buffering_value = router_stats[i].memory.buffering*1024;
+			var free_value = router_stats[i].sys_memfree*1024;
+			var caching_value = router_stats[i].sys_memcache*1024;
+			var buffering_value = router_stats[i].sys_membuff*1024;
 			var date_value = router_stats[i].time.$date;
 			if(free_value != null && caching_value != null && buffering_value != null) {
 				free.push([date_value, free_value]);
@@ -181,8 +221,8 @@ function process_graph() {
 		var len, i;
 		for (len=router_stats.length, i=0; i<len; i++) {
 			try {
-				var runnable_value = router_stats[i].processes.runnable;
-				var total_value = router_stats[i].processes.total;
+				var runnable_value = router_stats[i].sys_procrun;
+				var total_value = router_stats[i].sys_proctot;
 				var date_value = router_stats[i].time.$date;
 				if(runnable_value != null && total_value != null) {
 					runnable.push([date_value, runnable_value]);
@@ -209,28 +249,57 @@ function process_graph() {
 
 function client_graph() {
 	var clientstat = $("#clientstat");
-	var clients = [];
+	var clients = [], clients_eth = [], clients_w2 = [], clients_w5 = [];
 	var len, i;
 	for (len=router_stats.length, i=0; i<len; i++) {
 		try {
 			var client_value = router_stats[i].clients;
+			var client_eth = router_stats[i].clients_eth;
+			var client_w2 = router_stats[i].clients_w2;
+			var client_w5 = router_stats[i].clients_w5;
 			var date_value = router_stats[i].time.$date;
 			if(client_value != null) {
 				clients.push([date_value, client_value]);
 			}
+			if(client_eth != null) {
+				clients_eth.push([date_value, client_eth]);
+			}
+			if(client_w2 != null) {
+				clients_w2.push([date_value, client_w2]);
+			}
+			if(client_w5 != null) {
+				clients_w5.push([date_value, client_w5]);
+			}
 		}
 		catch(TypeError) {
 			// pass
 		}
 	}
-	var pdata = [
-		{"label": "clients", "data": clients, "color": "#8CACC6", lines: {fill: true}}
-	];
+	var pdata = [];
+	if (clients_w2.length > 0) {
+		pdata.push(
+			{"label": "2.4 GHz", "data": clients_w2, "color": "#CB4B4B"}
+		);
+	}
+	if (clients_w5.length > 0) {
+		pdata.push(
+			{"label": "5 GHz", "data": clients_w5, "color": "#EDC240"}
+		);
+	}
+	if (clients_eth.length > 0) {
+		pdata.push(
+			{"label": "Ethernet", "data": clients_eth, "color": "#4DA74A"}
+		);
+	}
+	pdata.push(
+		{"label": "Total", "data": clients, "color": "#8CACC6"}
+	);
+	
 	var plot = $.plot(clientstat, pdata, {
 		xaxis: {mode: "time", timezone: "browser"},
 		selection: {mode: "x"},
 		yaxis: {min: 0},
-		legend: {hideable: true},
+		legend: {noColumns: 4, hideable: true},
 		series: {downsample: {threshold: Math.floor(clientstat.width() * points_per_px)}}
 	});
 	setup_plot_zoom(plot, pdata, len);
@@ -265,17 +334,56 @@ function loadavg_graph() {
 	setup_plot_zoom(plot, pdata, len);
 }
 
+function airtime_graph() {
+	var airstat = $("#airstat");
+	var airtime2 = [];
+	var airtime5 = [];
+	var len, i;
+	for (len=router_stats.length, i=0; i<len; i++) {
+		try {
+			var air2_value = router_stats[i].airtime_w2;
+			var air5_value = router_stats[i].airtime_w5;
+			var date_value = router_stats[i].time.$date;
+			if(air2_value != null) {
+				airtime2.push([date_value, air2_value * 100]);
+			}
+			if(air5_value != null) {
+				airtime5.push([date_value, air5_value * 100]);
+			}
+		}
+		catch(TypeError) {
+			// pass
+		}
+	}
+	var pdata = [
+		{"label": "Airtime 2.4 GHz / %", "data": airtime2, "color": "#CB4B4B"}
+	];
+	if (airtime5.length > 0) {
+		pdata.push(
+			{"label": "Airtime 5 GHz / %", "data": airtime5, "color": "#EDC240"}
+		);
+	}
+	var plot = $.plot(airstat, pdata, {
+		xaxis: {mode: "time", timezone: "browser"},
+		selection: {mode: "x"},
+		yaxis: {min: 0, max: 100},
+		legend: {noColumns: 2, hideable: true},
+		series: {downsample: {threshold: Math.floor(airstat.width() * points_per_px)}}
+	});
+	setup_plot_zoom(plot, pdata, len);
+}
+
 
 // Global statistics
 
-function global_client_graph() {
-	var clientstat = $("#globclientstat");
+function global_client_graph(indata,field) {
+	var clientstat = $("#"+field);
 	var clients = [];
 	var len, i;
-	for (len=global_stats.length, i=0; i<len; i++) {
+	for (len=indata.length, i=0; i<len; i++) {
 		try {
-			var client_value = global_stats[i].total_clients;
-			var date_value = global_stats[i].time.$date;
+			var client_value = indata[i].clients;
+			var date_value = indata[i].time.$date;
 			if(client_value != null) {
 				clients.push([date_value, client_value]);
 			}
@@ -298,37 +406,43 @@ function global_client_graph() {
 	setup_plot_zoom(plot, pdata, len);
 }
 
-function global_router_graph() {
-	var memstat = $("#globrouterstat");
-	var offline = [], online = [], unknown = [];
+function global_router_graph(indata,field) {
+	var memstat = $("#"+field);
+	var offline = [], online = [], unknown = [], orphaned = [], total = [];
 	var len, i;
-	for (len=global_stats.length, i=0; i<len; i++) {
+	for (len=indata.length, i=0; i<len; i++) {
 		try {
-			var offline_value = global_stats[i].router_status.offline;
-			var online_value = global_stats[i].router_status.online;
-			var unknown_value = global_stats[i].router_status.unknown;
-			var date_value = global_stats[i].time.$date;
+			var offline_value = indata[i].offline;
+			var online_value = indata[i].online;
+			var unknown_value = indata[i].unknown;
+			var orphaned_value = indata[i].orphaned;
+			var date_value = indata[i].time.$date;
 			if (offline_value == null) offline_value = 0;
 			if (online_value == null) online_value = 0;
 			if (unknown_value == null) unknown_value = 0;
+			if (orphaned_value == null) orphaned_value = 0;
 			offline.push([date_value, offline_value]);
 			online.push([date_value, online_value]);
 			unknown.push([date_value, unknown_value]);
+			orphaned.push([date_value, orphaned_value]);
+			total.push([date_value, offline_value + online_value + unknown_value + orphaned_value]);
 		}
 		catch(TypeError) {
 			// pass
 		}
 	}
 	var pdata = [
+		{"label": "total", "data": total, "color": "#006DD9"},
 		{"label": "online", "data": online, "color": "#4DA74A"},
 		{"label": "offline", "data": offline, "color": "#CB4B4B"},
-		{"label": "unknown", "data": unknown, "color": "#EDC240"}
+		{"label": "unknown", "data": unknown, "color": "#EDC240"},
+		{"label": "orphaned", "data": orphaned, "color": "#666666"}
 	];
 	var plot = $.plot(memstat, pdata, {
 		xaxis: {mode: "time", timezone: "browser"},
 		selection: {mode: "x"},
-		yaxis: {min: 0, autoscaleMargin: 0.1},
-		legend: {noColumns: 3, hideable: true},
+		yaxis: {min: 0, autoscaleMargin: 0.15},
+		legend: {noColumns: 5, hideable: true},
 		series: {downsample: {threshold: Math.floor(memstat.width() * points_per_px)}}
 	});
 	setup_plot_zoom(plot, pdata, len);
@@ -346,40 +460,40 @@ function global_router_firmwares_graph() {
 	var plot = $.plot(placeholder, pdata, {
 		legend: {noColumns: 1, show: true, "labelFormatter": legendFormatter},
 		grid: {hoverable: true, clickable: true},
-		tooltip: {show: true, content: "<b>%s</b>: %p.0%", shifts: {x: 15, y: 5}, defaultTheme: true},
+		tooltip: {show: true, content: "<b>%s</b>: %p.1%", shifts: {x: 15, y: 5}, defaultTheme: true},
 		series: {pie: {
 			show: true, radius: 99/100, label: {show: true, formatter: labelFormatter, radius: 0.5, threshold: 0.10},
-			combine: {threshold: 0.009}
+			combine: {threshold: 0.005}
 		}}
 	});
 	placeholder.bind("plotclick", function(event, pos, obj) {
 		if (obj && obj.series.label != "Other") {
-			window.location.href = routers_page_url + "?q=software.firmware:" + obj.series.label;
+			window.location.href = routers_page_url + encodeURI("?q=firmware:^" + obj.series.label + "$ " + hoodstr);
 		}
 	});
 }
 
-function global_router_models_graph() {
-	var placeholder = $("#globroutermodelsstat");
+function global_router_models_graph(id,field) {
+	var placeholder = $("#"+id);
 	var pdata = [];
 	for (var mdname in router_models) {
 		pdata.push({
 			"label": mdname,
-			"data": [router_models[mdname]]
+			"data": [router_models[mdname][field]]
 		});
 	}
 	var plot = $.plot(placeholder, pdata, {
 		legend: {noColumns: 1, show: true, "labelFormatter": legendFormatter},
 		grid: {hoverable: true, clickable: true},
-		tooltip: {show: true, content: "<b>%s</b>: %p.0%", shifts: {x: 15, y: 5}, defaultTheme: true},
+		tooltip: {show: true, content: "<b>%s</b>: %p.1%", shifts: {x: 15, y: 5}, defaultTheme: true},
 		series: {pie: {
 			show: true, radius: 99/100, label: {show: true, formatter: labelFormatter, radius: 0.5, threshold: 0.2},
-			combine: {threshold: 0.009}
+			combine: {threshold: 0.019}
 		}}
 	});
 	placeholder.bind("plotclick", function(event, pos, obj) {
 		if (obj && obj.series.label != "Other") {
-			window.location.href = routers_page_url + "?q=hardware.name:" + obj.series.label.replace(/ /g, '_');
+			window.location.href = routers_page_url + encodeURI("?q=hardware:^" + obj.series.label.replace(/ /g, '_') + "$ " + hoodstr);
 		}
 	});
 }

ffmap/web/static/js/graph/jquery.flot.byte.js

@@ -87,6 +87,88 @@
 						}
 
 
+						if (typeof axis.rate !== "undefined") {
+							ext += "/s";
+						}
+
+						return (size.toFixed(axis.tickDecimals) + ext);
+					};
+				}
+				else if (opts.mode === "bit" || opts.mode === "bitRate") {
+					axis.tickGenerator = function (axis) {
+						var returnTicks = [],
+							tickSize = 2,
+							delta = axis.delta,
+							steps = 0,
+							tickMin = 0,
+							tickVal,
+							tickCount = 0;
+
+						//Set the reference for the formatter
+						if (opts.mode === "bitRate") {
+							axis.rate = true;
+						}
+
+						//Enforce maximum tick Decimals
+						if (typeof opts.tickDecimals === "number") {
+							axis.tickDecimals = opts.tickDecimals;
+						} else {
+							axis.tickDecimals = 0;
+						}
+
+						//Count the steps
+						while (Math.abs(delta) >= 1000) {
+							steps++;
+							delta /= 1000;
+						}
+
+						//Set the tick size relative to the remaining delta
+						while (tickSize <= 1000) {
+							if (delta <= tickSize) {
+								break;
+							}
+							tickSize *= 2;
+						}
+
+						//Tell flot the tickSize we've calculated
+						if (typeof opts.minTickSize !== "undefined" && tickSize < opts.minTickSize) {
+							axis.tickSize = opts.minTickSize;
+						} else {
+							axis.tickSize = tickSize * Math.pow(1000,steps);
+						}
+
+						//Calculate the new ticks
+						tickMin = floorInBase(axis.min, axis.tickSize);
+						do {
+							tickVal = tickMin + (tickCount++) * axis.tickSize;
+							returnTicks.push(tickVal);
+						} while (tickVal < axis.max);
+
+						return returnTicks;
+					};
+
+					axis.tickFormatter = function(size, axis) {
+						var ext, steps = 0;
+
+						while (Math.abs(size) >= 1000) {
+							steps++;
+							size /= 1000;
+						}
+
+
+						switch (steps) {
+							case 0: ext = " b";  break;
+							case 1: ext = " kb"; break;
+							case 2: ext = " Mb"; break;
+							case 3: ext = " Gb"; break;
+							case 4: ext = " Tb"; break;
+							case 5: ext = " Pb"; break;
+							case 6: ext = " Eb"; break;
+							case 7: ext = " Zb"; break;
+							case 8: ext = " Yb"; break;
+						}
+
+
 						if (typeof axis.rate !== "undefined") {
 							ext += "/s";
 						}

ffmap/web/static/js/map.js

@@ -5,7 +5,6 @@ var tilesosmorg = new L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y
 	maxNativeZoom: 19,
 	maxZoom: 22
 });
-map.addLayer(tilesosmorg);
 var tilesosmde = new L.TileLayer('https://{s}.osm.rrze.fau.de/osmde/{z}/{x}/{y}.png', {
 	attribution: '<a href="https://www.openstreetmap.org/copyright">&copy; Openstreetmap Contributors</a>',
 	maxNativeZoom: 19,
@@ -26,17 +25,23 @@ var overlay_config = {
 	maximumAge: 1000*3600*24*10
 }
 
-var links_and_routers = new L.TileLayer(tileurls.links_and_routers + '/{z}/{x}/{y}.png', overlay_config).addTo(map);
-var hoods = new L.TileLayer(tileurls.hoods + '/{z}/{x}/{y}.png', overlay_config);
-var hoodsv2 = new L.TileLayer(tileurls.hoodsv2 + '/{z}/{x}/{y}.png', overlay_config);
+var routers = new L.TileLayer(tileurls.routers + '/{z}/{x}/{y}.png', overlay_config);
+var routers_v2 = new L.TileLayer(tileurls.routers_v2 + '/{z}/{x}/{y}.png', overlay_config);
+var routers_local = new L.TileLayer(tileurls.routers_local + '/{z}/{x}/{y}.png', overlay_config);
+var hoods_v2 = new L.TileLayer(tileurls.hoods_v2 + '/{z}/{x}/{y}.png', overlay_config);
+var hoods_poly = new L.TileLayer(tileurls.hoods_poly + '/{z}/{x}/{y}.png', overlay_config);
+var popuplayer = new L.TileLayer('');
 layersControl = new L.Control.Layers({
 	"openstreetmap.org": tilesosmorg,
 	"openstreetmap.de": tilesosmde,
 	"Thunderforest Outdoors": tilestfod
 }, {
-	"Links & Routers": links_and_routers,
-	"Hoods": hoods,
-	"Hoods v2": hoodsv2
+	"Routers V1": routers,
+	"Routers V2": routers_v2,
+	"Local Routers": routers_local,
+	"Hoods V2": hoods_v2,
+	"Poly-Hoods": hoods_poly,
+	"Position-Popup": popuplayer
 });
 map.addControl(layersControl);
 
@@ -48,22 +53,74 @@ if (window.matchMedia("(min--moz-device-pixel-ratio: 1.5),(-o-min-device-pixel-r
 }
 
 var popup;
+var popupopen = false;
 
-function update_mappos_permalink() {
+function update_permalink() {
 	if (typeof mapurl != 'undefined') {
 		var pos = map.getCenter();
 		var zoom = map.getZoom();
-		window.history.replaceState({}, document.title, mapurl + '?mapcenter='+pos.lat.toFixed(5)+','+pos.lng.toFixed(5)+','+zoom);
+		window.history.replaceState({}, document.title,
+			mapurl + '?mapcenter=' + pos.lat.toFixed(5) + ',' + pos.lng.toFixed(5) + ',' + zoom
+			+ '&source=' + (map.hasLayer(tilesosmorg) ? 1 : (map.hasLayer(tilesosmde) ? 2 : 3))
+			+ '&layers=' + (map.hasLayer(routers)|0) + ','
+			+ (map.hasLayer(routers_v2)|0) + ','
+			+ (map.hasLayer(routers_local)|0) + ','
+			+ '0,'
+			+ (map.hasLayer(hoods_v2)|0) + ','
+			+ (map.hasLayer(hoods_poly)|0) + ','
+			+ (map.hasLayer(popuplayer)|0)
+			);
 	}
 }
 
+function initialMap() {
+	map.addLayer(tilesosmorg);
+}
+function initialLayers() {
+	routers.addTo(map);
+	routers_v2.addTo(map);
+	routers_local.addTo(map);
+}
+function setupMap(getargs) {
+	var getsrc = getargs.replace("source=", "");
+	if(getsrc==2) { map.addLayer(tilesosmde); }
+	else if(getsrc==3) { map.addLayer(tilestfod); }
+	else { map.addLayer(tilesosmorg); }
+}
+function setupLayers(getargs) {
+	var getlayers = getargs.replace("layers=", "").split(",");
+	if(getlayers[0]==1) { routers.addTo(map); }
+	if(getlayers[1]==1) { routers_v2.addTo(map); }
+	if(getlayers[2]==1) { routers_local.addTo(map); }
+	// getlayers[3] former hoods_v1 unused
+	if(getlayers[4]==1) { hoods_v2.addTo(map); }
+	if(getlayers[5]==1) { hoods_poly.addTo(map); }
+	if(getlayers[6]==1) { popuplayer.addTo(map); }
+}
 
-map.on('moveend', update_mappos_permalink);
-map.on('zoomend', update_mappos_permalink);
+map.on('moveend', update_permalink);
+map.on('zoomend', update_permalink);
+map.on('overlayadd', update_permalink);
+map.on('overlayremove', update_permalink);
+map.on('baselayerchange', update_permalink);
 
 map.on('click', function(pos) {
 	// height = width of world in px
 	var size_of_world_in_px = map.options.crs.scale(map.getZoom());
+	
+	layeropt = ""
+	if (map.hasLayer(routers)) {
+		console.debug("Looking for router in V1 ...");
+		layeropt += "&v1=on"
+	}
+	if (map.hasLayer(routers_v2)) {
+		console.debug("Looking for router in V2 ...");
+		layeropt += "&v2=on"
+	}
+	if (map.hasLayer(routers_local)) {
+		console.debug("Looking for router in local hoods ...");
+		layeropt += "&local=on"
+	}
 
 	var px_per_deg_lng = size_of_world_in_px / 360;
 	var px_per_deg_lat = size_of_world_in_px / 180;
@@ -73,22 +130,25 @@ map.on('click', function(pos) {
 	var lat = pos.latlng.lat;
 	if (lng > 180) { lng -= 360; }
 
-	ajax_get_request(url_get_nearest_router + "?lng=" + lng + "&lat=" + lat, function(router) {
-		// decide if router is close enough
-		var lng_delta = Math.abs(lng - router.position.coordinates[0])
-		var lat_delta = Math.abs(lat - router.position.coordinates[1])
+	ajax_get_request(url_get_nearest_router + "?lng=" + lng + "&lat=" + lat + layeropt, function(router) {
+		if (router) {
+			// decide if router is close enough
+			var lng_delta = Math.abs(lng - router.lng)
+			var lat_delta = Math.abs(lat - router.lat)
 
-		// convert degree distances into px distances on the map
-		var x_delta_px = lng_delta * px_per_deg_lng;
-		var y_delta_px = lat_delta * px_per_deg_lat;
+			// convert degree distances into px distances on the map
+			var x_delta_px = lng_delta * px_per_deg_lng;
+			var y_delta_px = lat_delta * px_per_deg_lat;
 
-		// use pythagoras to calculate distance
-		var px_distance = Math.sqrt(x_delta_px*x_delta_px + y_delta_px*y_delta_px);
+			// use pythagoras to calculate distance
+			var px_distance = Math.sqrt(x_delta_px*x_delta_px + y_delta_px*y_delta_px);
 
-		console.debug("Distance to closest router ("+router.hostname+"): " + px_distance+"px");
+			console.debug("Distance to closest router ("+router.hostname+"): " + px_distance+"px");
+		}
 
 		// check if mouse click was on the router icon
-		if (px_distance <= router_pointer_radius) {
+		if (router && px_distance <= router_pointer_radius) {
+			popupopen = true;
 			console.log("Click on '"+router.hostname+"' detected.");
 			console.log(router);
 			var popup_html = "";
@@ -99,19 +159,21 @@ map.on('click', function(pos) {
 				has_neighbours = false;
 				for (var i = 0; i < router.neighbours.length; i++) {
 					neighbour = router.neighbours[i];
-					if ('_id' in neighbour) {
+					if ('id' in neighbour) {
 						has_neighbours = true;
 					}
 				}
 			}
-						
+
 			if (has_neighbours) {
+				console.log("Has "+router.neighbours.length+" neighbours.");
 				popup_html += "<div class=\"popup-headline with-neighbours\">";
 			}
 			else {
+				console.log("Has no neighbours.");
 				popup_html += "<div class=\"popup-headline\">";
 			}
-			popup_html += '<b>Router <a href="' + url_router_info + router._id.$oid +'">'+router.hostname+'</a></b>';
+			popup_html += '<b>Router <a href="' + url_router_info + router.id +'">'+router.hostname+'</a></b>';
 			popup_html += "</div>"
 			if (has_neighbours) {
 				popup_html += '<table class="neighbours" style="width: 100%;">';
@@ -123,26 +185,33 @@ map.on('click', function(pos) {
 				for (var i = 0; i < router.neighbours.length; i++) {
 					neighbour = router.neighbours[i];
 					// skip unknown neighbours
-					if ('_id' in neighbour) {
-						var tr_color = "#04ff0a";
-						if      (neighbour.quality == -1) { tr_color = "#0684c4"; }
-						else if (neighbour.quality < 105) { tr_color = "#ff1e1e"; }
-						else if (neighbour.quality < 130) { tr_color = "#ff4949"; }
-						else if (neighbour.quality < 155) { tr_color = "#ff6a6a"; }
-						else if (neighbour.quality < 180) { tr_color = "#ffac53"; }
-						else if (neighbour.quality < 205) { tr_color = "#ffeb79"; }
-						else if (neighbour.quality < 230) { tr_color = "#79ff7c"; }
-						popup_html += "<tr style=\"background-color: "+tr_color+";\">";
-						popup_html += '<td><a href="'+url_router_info+neighbour._id.$oid+'" title="'+escapeHTML(neighbour.mac)+'">'+escapeHTML(neighbour.hostname)+'</a></td>';
+					if ('id' in neighbour) {
+						popup_html += "<tr style=\"background-color: "+neighbour.color+";\">";
+						popup_html += '<td><a href="'+url_router_info+neighbour.id+'" title="'+escapeHTML(neighbour.mac)+'" style="color:#000000">'+escapeHTML(neighbour.hostname)+'</a></td>'; // MACTODO
 						popup_html += "<td>"+neighbour.quality+"</td>";
-						popup_html += "<td>"+escapeHTML(neighbour.net_if)+"</td>";
+						popup_html += "<td>"+escapeHTML(neighbour.netif)+"</td>";
 						popup_html += "</tr>";
 					}
 				}
 				popup_html += "</table>";
 			}
 			popup = L.popup({offset: new L.Point(1, 1), maxWidth: 500})
-				.setLatLng([router.position.coordinates[1], router.position.coordinates[0]])
+				.setLatLng([router.lat, router.lng])
+				.setContent(popup_html)
+				.openOn(map);
+		} else if(popupopen) {
+			popupopen = false;
+		} else if(map.hasLayer(popuplayer)) {
+			popupopen = true;
+			console.log("Click on lat: "+lat+", lng: "+lng+" detected.");
+			var popup_html = "<div class=\"popup-headline\">";
+			
+			popup_html += '<b>Coordinates</b>';
+			popup_html += '<p class="popup-latlng" style="margin:0">Latitude: '+lat.toFixed(8)+'</p>';
+			popup_html += '<p class="popup-latlng" style="margin:0">Longitude: '+lng.toFixed(8)+'</p>';
+			popup_html += "</div>"
+			popup = L.popup({offset: new L.Point(1, 1), maxWidth: 500})
+				.setLatLng([lat, lng])
 				.setContent(popup_html)
 				.openOn(map);
 		}

ffmap/web/templates/apidoc.html

@@ -77,6 +77,15 @@
 			</td>
 		</tr>
 		<tr>
+			<td>
+				<h2>Routers without position</h2>
+			</td>
+			<td>
+				<p class="apilink">/api/nopos</p>
+				<p class="apidesc">Returns JSON file of all routers without coordinates set.</p>
+			</td>
+		</tr>
+		<tr class="uneven">
 			<td>
 				<h2>Extended router list</h2>
 			</td>
@@ -85,7 +94,25 @@
 				<p class="apidesc">Returns JSON file of all routers with the following information: <span style="font-style:italic">Monitoring ID, hostname, MAC address, hood, status, user nickname, hardware, firmware, Monitoring link, clients, last contact, uplink interfaces and coordinates.</span></p>
 			</td>
 		</tr>
+		<tr>
+			<td>
+				<h2>Routers of a specific user</h2>
+			</td>
+			<td>
+				<p class="apilink">/api/routers_by_nickname/&lt;user_nickname&gt;</p>
+				<p class="apidesc">Returns JSON file of all routers belonging to the specified &lt;user_nickname&gt;.</p>
+			</td>
+		</tr>
 		<tr class="uneven">
+			<td>
+				<h2>Routers of a hood by KeyXchange ID</h2>
+			</td>
+			<td>
+				<p class="apilink">/api/routers_by_keyxchange_id/&lt;hood_keyxchange_id&gt;</p>
+				<p class="apidesc">Returns JSON file of all routers belonging to the specified &lt;hood_keyxchange_id&gt;.</p>
+			</td>
+		</tr>
+		<tr>
 			<td>
 				<h2>Wifi Analyzer node list</h2>
 			</td>
@@ -95,6 +122,16 @@
 				<p class="apidesc">The file contains all routers of the selected &lt;hood&gt;.</p>
 			</td>
 		</tr>
+		<tr class="uneven">
+			<td>
+				<h2>Wifi Analyzer node list for all hoods</h2>
+			</td>
+			<td>
+				<p class="apilink">/api/wifianalall</p>
+				<p class="apidesc">Returns configuration file (text/plain) for the Wifi Analyzer app (<a href="https://play.google.com/store/apps/details?id=com.farproc.wifi.analyzer&hl=en">PlayStore</a>).</p>
+				<p class="apidesc">The file contains all routers of all hoods.</p>
+			</td>
+		</tr>
 		</table>
 	</div>
 {% endblock %}

ffmap/web/templates/bootstrap.html

@@ -1,9 +1,10 @@
 <!DOCTYPE html>
-<html>
+<html lang="de">
 <head>
 	{% block head %}
 	<meta charset="utf-8">
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="google" content="notranslate">
 	<meta name="viewport" content="{% block viewport %}width=device-width, initial-scale=1, maximum-scale=1.0, user-scalable=no{% endblock %}">
 
 	<title>{% block title %}FFF Monitoring{% endblock %}</title>
@@ -38,6 +39,7 @@
 					                      (["router_list", "router_info"], "Routers"),
 					                      (["user_list", "user_info"], "Users"),
 					                      (["global_statistics"], "Statistics"),
+					                      (["gateways"], "GWs"),
 					                      (["apidoc"], "API"),
 					] %}
 					<li class="{{ "active" if request.endpoint in fkt }}"><a href="{{ url_for(fkt[0]) }}">{{ text }}</a></li>
@@ -52,7 +54,7 @@
 				<a href="{{ url_for('user_info', nickname=session.user) }}" class="navbar-link">{{ session.user }}</a>
 				[<a href="{{ url_for('logout') }}" class="navbar-link">Logout</a>]
 				{%- else %}
-				<a href="{{ url_for('login') }}" class="navbar-link">Login</a>
+				<a href="{{ url_for('login',_external=True,_scheme='https') }}" class="navbar-link">Login</a>
 				{%- endif %}
 				</p>
 				{%- endblock %}

ffmap/web/templates/gws.html

@@ -0,0 +1,226 @@
+{% extends "bootstrap.html" %}
+{% block title %}{{super()}} :: Gateways{% endblock %}
+{% block head %}{{super()}}
+	<script src="{{ url_for('static', filename='js/graph/date.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.time.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.byte.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.selection.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.downsample.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.resize.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.hiddengraphs.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.pie.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.tooltip.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/jquery.dataTables.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/dataTables.bootstrap.min.js') }}"></script>
+	<link rel="stylesheet" href="{{ url_for('static', filename='css/datatables/dataTables.bootstrap.min.css') }}">
+	<style type="text/css">
+		.table-condensed {
+			margin-bottom: 0;
+		}
+		.table-condensed tr:last-child td, th {
+			border-bottom: 1px solid #ddd;
+		}
+		@media(max-width:500px) {
+			th {
+				padding-left: 2px !important;
+				padding-right: 2px !important;
+			}
+			td {
+				padding-left: 2px !important;
+				padding-right: 2px !important;
+			}
+			.panel-body {
+				padding-left: 3px !important;
+				padding-right: 3px !important;
+			}
+		}
+		.table-hoods th {
+			text-align: center;
+		}
+		.table-hoods td {
+			text-align: center;
+		}
+		.table-hoods .firstrow {
+			text-align: left;
+		}
+	</style>
+{% endblock %}
+
+{% block content %}
+	<div class="row">
+		<div class="col-xs-12 col-md-6">
+			<div class="panel panel-default">
+				<div class="panel-heading">Gateways (selected / others)</div>
+				<div class="panel-body">
+					<table id="gwlist" class="table table-condensed table-hoods">
+					<thead>
+						<tr>
+							<th class="firstrow">Gateway</th>
+							<th class="success" title="Online Routers">On</th>
+							<th class="danger" title="Offline Routers">Off</th>
+							<th class="warning" title="Unknown Routers">Unk.</th>
+							<th class="active" title="Total Routers">Sum</th>
+						</tr>
+					</thead>
+					<tbody>
+						{%- for gw, value in gws.items() %}
+						<tr>
+							<td class="firstrow"><p style="margin:0">{{ value["name"] }}{%- if value["version"] %} <span style="font-size:12px">({{ value["version"] }})</span>{%- endif %}</p></td>
+							<td class="success" data-order="{{ (value["selected"]["online"] or 0) + (value["others"]["online"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["online"] or 0 }}</span> / {{ value["others"]["online"] or 0 }}</td>
+							<td class="danger" data-order="{{ (value["selected"]["offline"] or 0) + (value["others"]["offline"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["offline"] or 0 }}</span> / {{ value["others"]["offline"] or 0 }}</td>
+							<td class="warning" data-order="{{ (value["selected"]["unknown"] or 0) + (value["others"]["unknown"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["unknown"] or 0 }}</span> / {{ value["others"]["unknown"] or 0 }}</td>
+							<td class="active" data-order="{{ (value["selected"]|sumdict if value["selected"] else 0) + (value["others"]|sumdict if value["others"] else 0) }}"><span style="font-weight:bold">{{ value["selected"]|sumdict if value["selected"] else 0 }}</span> / {{ value["others"]|sumdict if value["others"] else 0 }}</td>
+						</tr>
+						{%- endfor %}
+					</tbody>
+					</table>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">DHCP ranges</div>
+				<div class="panel-body">
+					<table id="dhcplist" class="table table-condensed table-hoods">
+					<thead>
+						<tr>
+							<th class="firstrow">Gateway</th>
+							<th class="warning" title="Interface1">VPN</th>
+							<th title="Interface2">batX</th>
+							<th class="success" title="IPv4">Range</th>
+						</tr>
+					</thead>
+					<tbody>
+						{%- for ip in dhcp %}
+						<tr>
+							<td class="firstrow">{{ ip["name"] }}</td>
+							<td class="warning" data-order="{{ ip["name"] }}_{{ ip["vpnif"] }}">{{ ip["vpnif"] }}</td>
+							<td data-order="{{ ip["name"] }}_{{ ip["batif"] }}">{{ ip["batif"] }}</td>
+							<td class="success" data-order="{{ ip["dhcpstart"]|ip2int }}">{{ ip["dhcpstart"] }} - {{ ip["dhcpend"] }}</td>
+						</tr>
+						{%- endfor %}
+					</tbody>
+					</table>
+				</div>
+			</div>
+		</div>
+		<div class="col-xs-12 col-md-6">
+			<div class="panel panel-default">
+				<div class="panel-heading">IPv4 List</div>
+				<div class="panel-body">
+					<table id="ipv4list" class="table table-condensed table-hoods">
+					<thead>
+						<tr>
+							<th class="firstrow">Gateway</th>
+							<th class="warning" title="Interface1">VPN</th>
+							<th title="Interface2">batX</th>
+							<th class="success" title="IPv4">IPv4</th>
+							<th class="stats">Stat</th>
+						</tr>
+					</thead>
+					<tbody>
+						{%- for ip in ipv4 %}
+						<tr>
+							<td class="firstrow">{{ ip["name"] }}</td>
+							<td class="warning" data-order="{{ ip["name"] }}_{{ ip["vpnif"] }}">{{ ip["vpnif"] }}</td>
+							<td data-order="{{ ip["name"] }}_{{ ip["batif"] }}">{{ ip["batif"] }}</td>
+							<td class="success" data-order="{{ ip["ipv4"]|ipnet2int }}">{{ ip["ipv4"] }}</td>
+							{%- if ip["mac"] %}
+							<td class="stats"><a href="{{ url_for('global_gwstatistics', selectgw='%s' % ip["mac"]|int2shortmac) }}">Stats</a></td>
+							{%- else %}
+							<td class="stats">&nbsp;</td>
+							{%- endif %}
+						</tr>
+						{%- endfor %}
+					</tbody>
+					</table>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">IPv6 List</div>
+				<div class="panel-body">
+					<table id="ipv6list" class="table table-condensed table-hoods">
+					<thead>
+						<tr>
+							<th class="firstrow">Gateway</th>
+							<th class="warning" title="Interface1">VPN</th>
+							<th title="Interface2">batX</th>
+							<th class="success" title="IPv4">IPv6</th>
+							<th class="stats">Stat</th>
+						</tr>
+					</thead>
+					<tbody>
+						{%- for ip in ipv6 %}
+						<tr>
+							<td class="firstrow">{{ ip["name"] }}</td>
+							<td class="warning" data-order="{{ ip["name"] }}_{{ ip["vpnif"] }}">{{ ip["vpnif"] }}</td>
+							<td data-order="{{ ip["name"] }}_{{ ip["batif"] }}">{{ ip["batif"] }}</td>
+							<td class="success" data-order="{{ ip["ipv6"]|ipnet2int }}">{{ ip["ipv6"]|longip }}</td>
+							{%- if ip["mac"] %}
+							<td class="stats"><a href="{{ url_for('global_gwstatistics', selectgw='%s' % ip["mac"]|int2shortmac) }}">Stats</a></td>
+							{%- else %}
+							<td class="stats">&nbsp;</td>
+							{%- endif %}
+						</tr>
+						{%- endfor %}
+					</tbody>
+					</table>
+				</div>
+			</div>
+		</div>
+	</div>
+	<script type="text/javascript">
+		$(document).ready(function() {
+			$("#gwlist").DataTable({
+				"order": [],
+				"paging": false,
+				"info": false,
+				"searching": false
+				/*"responsive": {
+					"details": false
+				},*/
+			});
+			
+			$("#ipv4list").DataTable({
+				"order": [[3,'asc']],
+				"paging": false,
+				"info": false,
+				"searching": false,
+				/*"responsive": {
+					"details": false
+				},*/
+				"columnDefs": [
+					{"orderable": false, "targets": 0},
+					{"orderable": false, "targets": -1}
+				]
+			});
+			
+			$("#ipv6list").DataTable({
+				"order": [[3,'asc']],
+				"paging": false,
+				"info": false,
+				"searching": false,
+				/*"responsive": {
+					"details": false
+				},*/
+				"columnDefs": [
+					{"orderable": false, "targets": 0},
+					{"orderable": false, "targets": -1}
+				]
+			});
+			
+			$("#dhcplist").DataTable({
+				"order": [[3,'asc']],
+				"paging": false,
+				"info": false,
+				"searching": false,
+				/*"responsive": {
+					"details": false
+				},*/
+				"columnDefs": [
+					{"orderable": false, "targets": 0}
+				]
+			});
+		});
+	</script>
+{% endblock %}

ffmap/web/templates/index.html

@@ -1,18 +1,154 @@
 {% extends "bootstrap.html" %}
 {% block title %}{{super()}} :: Home{% endblock %}
+{% block head %}{{super()}}
+	<style type="text/css">
+		.jumbotron .consolefont {
+			font-style: normal;
+			white-space: pre;
+			font-family: monospace;
+		}
+	</style>
+{% endblock %}
 {% block content %}
 	<div class="jumbotron">
 		<h1>Freifunk Franken Monitoring</h1>
 		<div class="row">
 			<div class="col-xs-2 col-sm-2">
-				<img src="{{ url_for('static', filename='img/freifunk.svg') }}" style="width: 100%;" />
+				<img src="{{ url_for('static', filename='img/freifunk.svg') }}" alt="Freifunk" style="width: 100%;" />
+			</div>
+			<div class="col-xs-8 col-sm-8">
+				<p style="margin-top:12px">Monitoring f&uuml;r das Freifunk Franken Funknetzwerk.<br />
+				Hier werden Daten von Routern ab Firmware 0.5.x angezeigt.<br />
+				Der Quellcode f&uuml;r diese Webanwendung steht unter der GPL und kann <a href="https://github.com/FreifunkFranken/fff-monitoring">hier</a>
+				heruntergeladen werden.</p>
+				<h2 style="margin-top:30px">Changelog (nur Features)</h2>
+				<h3>&Auml;nderungen bis 14.12.2018</h3>
+				<ul style="font-size:18px">
+				<li>Verhindere Fake-Hoodwechsel</li>
+				<li>Rückbau der KeyXchangeV1 Unterstützung</li>
+				<li>Maskiere Public IPv6-Adressen</li>
+				</ul>
+				<h3>&Auml;nderungen bis 24.11.2018</h3>
+				<ul style="font-size:18px">
+				<li>Fehlende Kontaktadressen werden in Router-Liste hervorgehoben</li>
+				<li>gwinfo-Version wird angezeigt (Gateway-Seite und Gateway-Details auf Statistik-Seite)</li>
+				<li>URL f&uuml;r Karte enth&auml;lt nun die aktivierten Layer (ggf. Cache leeren)</li>
+				<li>"Reset!"-Warnung wird nun immer angezeigt, wenn keine Koordinaten gesetzt sind</li>
+				<li>Neue API /api/alfred2 ohne Angabe der alfred ID und bessere Status-R&uuml;ckmeldung</li>
+				<li>Standardausschnitt der Karte zeigt nun "ganzes" Freifunk Franken Gebiet</li>
+				<li>Support f&uuml;r Polygon-Hoods (z. Zt. noch nicht am KeyXchange verf&uuml;gbar)</li>
+				<li>Vertauschen der Links f&uuml;r Statistik-Daten und Router-Liste in der Liste der Gateways und der Hoods</li>
+				<li>Filtern der Router-Liste nach "os", "batman", "kernel" und "nodewatcher" m&ouml;glich</li>
+				<li>Link zu Router-Liste f&uuml;r diverse Router-Daten auf Detailseite verf&uuml;gbar</li>
+				</ul>
+				<h3>&Auml;nderungen bis 21.11.2018</h3>
+				<ul style="font-size:18px">
+				<li>Uplink-Router werden durch wei&szlig;e Punktmitte in Karte erkennbar</li>
+				<li>Seite f&uuml;r V1/V2-Vergleich hinzugef&uuml;gt: <a href="{{ url_for('v2_routers') }}">V2-Statistik</a></li>
+				<li>Neue Rubrik "Gateways" mit IP-Adressen und DHCP-Ranges (sofern per gwinfo &uuml;bermittelt)</li>
+				<li>Hood-Grenzen in verschiedenen Farben</li>
+				<li>Hoods werden vom KeyXchange geladen (kein manuelles Eintragen mehr n&ouml;tig)</li>
+				<li>Erkennung von dezentralen Hoods und Kennzeichnung/farbliche Markierung f&uuml;r V1/V2/Dezentral</li>
+				<li>Filtern der Hood-Liste (Statistik-Seite) nach V1/V2/Dezentral m&ouml;glich</li>
+				<li>Filtern der Router-Liste nach V1/V2/Dezentral: "network:&lt;local|v1|v2&gt;"</li>
+				<li>V2-Router pro Nutzer werden als Prozentzahl in der User-Liste angezeigt (hier z&auml;hlt dezentral als V2)</li>
+				</ul>
+				<h3>&Auml;nderungen bis 03.07.2018</h3>
+				<ul style="font-size:18px">
+				<li>Globaler und Hood-spezifischer aggregierter Traffic</li>
+				<li>Gateways werden als solche auf der Detailseite erkannt</li>
+				<li>Ethernet-Nachbarn werden auf der Karte farblich erkennbar (dunkelgr&uuml;ne Linien)</li>
+				<li>"Beste" Verbindung wird f&uuml;r Links auf Karte verwendet</li>
+				</ul>
+				<h3>&Auml;nderungen bis 16.04.2018</h3>
+				<ul style="font-size:18px">
+				<li>Server-Migration (vielen Dank an den Spender)</li>
+				<li>Mehrere Backend-&Auml;nderungen sollten daf&uuml;r sorgen, dass die Routerdaten nun verl&auml;sslicher alle 5 Minuten aktualisiert werden</li>
+				<li>Drastische Erh&ouml;hung der angezeigten Router-Events: Es werden nun die letzten 250 Events angezeigt</li>
+				<li>Die Nachbar-Statistik wird nun nur f&uuml;r den letzten Tag geladen; die volle Statistik kann &uuml;ber eine Schaltfl&auml;che nachgeladen werden. Zusammen mit anderen &Auml;nderungen reduziert dies die Ladezeit der Routerdetailseite enorm</li>
+				<li>Die Nachbar-Statistik zeigt nun auch Daten f&uuml;r Knoten an, die gerade nicht verbunden sind, aber es in der Vergangenheit waren</li>
+				<li>Das Positions-Popup in der Karte muss nun als Layer aktiviert werden (Standard: aus)</li>
+				<li>Die Statisitik-Seite schl&uuml;sselt nun die Modelle pro Client auf</li>
+				</ul>
+				<h3>&Auml;nderungen bis 22.02.2018</h3>
+				<ul style="font-size:18px">
+				<li>V2-Hoods werden auf der Statistik-Seite farblich hervorgehoben</li>
+				<li>Das Setzen des "Blocked"-Status wird als Router-Event geloggt</li>
+				<li>Babel-Verbindungskosten werden im Nachbarplot erfasst (erfordert Anpassung der Firmware)</li>
+				<li>Die Router&uuml;bersicht zeigt nun auch "last contact" an; nach der "uptime" kann sortiert werden</li>
+				<li>Netzinterfaces verf&uuml;gen nun &uuml;ber eine Erkl&auml;rung sowie Farbhervorhebung entsprechend der Funktion</li>
+				<li>Traffic-Control-Status f&uuml;r Router mit aktualisierter Firmware</li>
+				<li>F&uuml;r alle Router ohne Kontaktadresse wird eine Warnung angezeigt</li>
+				<li>Das Restart-Event f&uuml;r V2-Hoods wird nun wieder korrekt geloggt</li>
+				</ul>
+				<h3>&Auml;nderungen bis 01.02.2018</h3>
+				<ul style="font-size:18px">
+				<li>Detaillierte Client-Statistiken (Ethernet, 2.4 GHz, 5 GHz) f&uuml;r Router mit aktualisierter Firmware</li>
+				<li>Airtime-Statistiken f&uuml;r Router mit aktualisierter Firmware</li>
+				<li>Netzwerk-Interface-Statistiken werden jetzt in Bit pro Sekunde angezeigt</li>
+				</ul>
+				<h3>&Auml;nderungen bis 18.01.2018</h3>
+				<ul style="font-size:18px">
+				<li>Netif-Statistiken werden nur noch 21 Tage erfasst, um die Ladezeit der Routerseite zu reduzieren</li>
+				<li>In der Hood-&Uuml;bersichtstabelle wird nun die Zahl der aktiven Gateways als Spalte angezeigt</li>
+				<li>Die Hood- und Gateway-&Uuml;bersichtstabellen k&ouml;nnen sortiert werden</li>
+				<li>Im KeyXchange gesperrte Router werden im Monitoring entsprechend markiert (diese Einstellung ist nicht live, sondern muss manuell von Admins vorgenommen werden)</li>
+				<li>F&uuml;r Mesh-Router ohne Koordinaten wurde die virtuelle Hood "NoCoordinates" geschaffen; in der Default-Hood werden nur noch die Router angezeigt, die per VPN wirklich in der Default-Hood sind</li>
+				</ul>
+				<h3>&Auml;nderungen bis 13.01.2018</h3>
+				<ul style="font-size:18px">
+				<li>Verbundene Gateways werden nun f&uuml;r jeden Router auf der Detailseite angezeigt</li>
+				<li>Alle Gateways werden nun ebenso wie die Hoods auf der Statistik-Seite angezeigt; wie f&uuml;r die Hoods sind selektive Statistiken m&ouml;glich</li>
+				<li>Wird die Statistik-Seite nach Hoods gefiltert werden die entsprechenden Gateways angezeigt</li>
+				<li>Wird die Statistik-Seite nach Gateways gefiltert erh&auml;lt man (sofern vom GW-Betreiber bereitgestellt) Information zum Gateway (Admin, Stats-Seite, ...)</li>
+				<li>Router k&ouml;nnen anhand der MAC-Adresse ihrer Gateways gesucht werden:<br />Alle Router mit GW: <span class="consolefont">gw:&lt;mac_of_vpnif&gt;</span> oder <span class="consolefont">bat:&lt;mac_of_batX&gt;</span><br />Alle Router, die GW ausgew&auml;hlt haben: <span class="consolefont">selected:&lt;mac_of_vpn&gt;</span> oder <span class="consolefont">batselected:&lt;mac_of_batX&gt;</span></li>
+				</ul>
+				<h3>&Auml;nderungen bis 30.12.2017</h3>
+				<ul style="font-size:18px">
+				<li>Router-Popup in der Karte ber&uuml;cksichtigt aktivierte Layer</li>
+				<li>Verwaiste (orphaned) Router werden in der Statistik ber&uuml;cksichtigt</li>
+				<li>F&uuml;r die WLAN-Interfaces werden zus&auml;tzliche Informationen (Kanal, SSID, Typ) angezeigt; diese Funktion ben&ouml;tigt ein Firmware-Update des Routers</li>
+				<li>Tx-Power f&uuml;r die WLAN-Interfaces wird angezeigt (Antennen werden so ber&uuml;cksichtigt wie dies in der Firmware eingestellt ist)</li>
+				<li>Bei Klick auf einen freien Bereich der Karte werden die Koordinaten angezeigt</li>
+				<li>Es werden nun 25 Events pro Router angezeigt, die Reihenfolge wurde invertiert</li>
+				</ul>
+				<h3>&Auml;nderungen bis 20.12.2017</h3>
+				<ul style="font-size:18px">
+				<li>Router-Statistiken werden f&uuml;r 30 Tage erfasst</li>
+				<li>Benutzer k&ouml;nnen ihren eigenen Account selbst l&ouml;schen</li>
+				<li>Login mit E-Mail-Adresse statt Username ist ab sofort m&ouml;glich</li>
+				<li>Router k&ouml;nnen per Schaltfl&auml;che an Administratoren gemeldet werden, falls rechtswidrige Texte hinterlegt sind</li>
+				<li>Router k&ouml;nnen permanent aus dem Monitoring verbannt werden</li>
+				</ul>
+				<h3>&Auml;nderungen bis 10.12.2017</h3>
+				<ul style="font-size:18px">
+				<li>Perma-Link auf Router-Detailseite</li>
+				<li>Router k&ouml;nnen anhand der MAC-Adresse ihrer Nachbarn gesucht werden: <span class="consolefont">neighbor:&lt;mac_of_neighbor&gt;</span></li>
+				<li>Globale und Hood-Statistiken werden f&uuml;r 365 Tage erfasst</li>
+				<li>Die Suchfunktion auf der Router-&Uuml;bersichtsseite unterst&uuml;tzt jetzt gr&ouml;&szlig;tenteils Regex</li>
+				<li>Bei Router-Reset wird ein Hinweis auf Detail- und &Uuml;bersichtsseite angezeigt</li>
+				<li>Die Statistik-Seite wurde umgeordnet (Graphen jetzt rechts oben)</li>
+				</ul>
+				<h3>&Auml;nderungen bis 17.11.2017</h3>
+				<ul style="font-size:18px">
+				<li>Router-Status wird schneller aktualisiert: Anzeige nach 90 Sek. (bisher 5 Min.)</li>
+				<li>Statistik wird schneller aktualisiert: Anzeige nach 3 Min. (bisher 6 Min.)</li>
+				<li>Offline-Status wird schneller aktualisiert: Offline nach 18 Min. (bisher 26 Min.)</li>
+				</ul>
+				<h3>&Auml;nderungen bis 16.11.2017</h3>
+				<ul style="font-size:18px">
+				<li>Changelog wird eingef&uuml;hrt</li>
+				<li>MySQL-Unterst&uuml;tzung und reines Python3 (Mapnik, Tilestache)</li>
+				<li>Individuelle Statistik-Seite pro Hood</li>
+				<li><span class="consolefont">/routers/&lt;nr&gt;?fffconfig</span> gibt fffconfig Datei aus</li>
+				<li>Neuer Status "orphaned" wenn Router l&auml;nger als 7 Tage offline sind (Icon wird grau)</li>
+				<li>Separate Layer und Farben f&uuml;r KeyXchange v1 und v2 Router</li>
+				<li>Leerzeichen in Hoodnamen werden unterst&uuml;tzt (aber nicht gew&uuml;nscht)</li>
+				<li>Hoodname wird nicht mehr nach lower-case konvertiert</li>
+				<li><span class="consolefont">/api/wifianalall</span> gibt WifiAnalyzer-Datei f&uuml;r ALLE Router aus</li>
+				<li>Statistiken werden in separatem Cronjob berechnet</li>
+				</ul>
 			</div>
-			<div class="col-xs-8 col-sm-8"><p>
-				Monitoring f&uuml;r das Freifunk Franken Funknetzwerk.<br />
-				Hier werden Daten von Routern ab Firmware 0.5.2 angezeigt.<br />
-				Der Quellcode f&uuml;r diese Webanwendung steht unter der GPL und kann <a href="https://github.com/asdil12/fff-monitoring">hier</a>
-				heruntergeladen werden.
-			</p></div>
 		</div>
 	</div>
 {% endblock %}

ffmap/web/templates/map.html

@@ -33,11 +33,32 @@
 	<script src="{{ url_for('static', filename='js/map.js') }}"></script>
 	<script type="text/javascript">
 		if (window.location.search.match("^\\?mapcenter")) {
-			var maploc = window.location.search.replace("?mapcenter=", "").split(",");
+			var getargs = window.location.search.replace("?mapcenter=", "").split("&");
+			var maploc = getargs[0].split(",");
 			map.setView([maploc[0], maploc[1]], maploc[2]);
+			if (getargs.length > 1 && getargs[1].match("source=")) {
+				setupMap(getargs[1]);
+				if (getargs.length > 2 && getargs[2].match("layers=")) {
+					setupLayers(getargs[2]);
+				} else {
+					initialLayers();
+				}
+			} else if (getargs.length > 1 && getargs[1].match("layers=")) {
+				setupLayers(getargs[1]);
+				if (getargs.length > 2 && getargs[2].match("source=")) {
+					setupMap(getargs[2]);
+				} else {
+					initialMap();
+				}
+			} else {
+				initialMap();
+				initialLayers();
+			}
 		}
 		else {
-			map.setView([49.45, 11.1], 10);
+			map.setView([49.824, 10.786], 9);
+			initialMap();
+			initialLayers();
 		}
 	</script>
 {% endblock %}

ffmap/web/templates/router.html

@@ -14,7 +14,7 @@
 	<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
 	<style type="text/css">
 		#map {
-			height: 405px;
+			height: 467px;
 			width: 100%;
 		}
 		.navbar, .table-condensed {
@@ -26,6 +26,16 @@
 		li.list-group-item:hover {
 			background-color: #f5f5f5;
 		}
+		.wlaninfo {
+			font-weight:bold;
+			font-style:italic;
+		}
+		.clientinfo {
+			font-weight:bold;
+		}
+		.netifdesc {
+			font-style:italic;
+		}
 
 		/* hack to remove flex css on small single-column layout */
 		@media(max-width:991px) {
@@ -38,7 +48,12 @@
 
 {% block content %}
 	<div class="row" style="margin-top: 5px; margin-bottom: 5px;">
-		<div class="col-xs-12 col-sm-10"><h2 style="margin-top: 10px;">Router: {{ router.hostname }}</h2></div>
+		<div class="col-xs-12 col-sm-10">
+		<h2 style="margin-top: 10px;">{%- if router.gateway %}Gateway{%- else %}Router{%- endif %}: {{ router.hostname }}</h2>
+		{%- if mac %}
+		<h4 style="margin-top: 10px;margin-bottom: 20px">Perma-Link: <a href="{{ url_for('router_mac', mac=mac|int2shortmac, _external=True) }}">{{ url_for('router_mac', mac=mac|int2shortmac, _external=True) }}</a></h4>
+		{%- endif %}
+		</div>
 		<div class="col-xs-12 col-sm-2 text-right" style="margin-top: 10px; margin-bottom: 10px;">
 			<form method="post" id="actform">
 				<input type="hidden" name="act" id="act" value="" />
@@ -48,8 +63,14 @@
 						<span class="caret"></span>
 					</button>
 					<ul class="dropdown-menu">
-						{# FIXME: If authorized #}
+						{%- if authuser %}
 						<li><a href="#" onclick="$('#act').val('delete'); $('#actform').submit()">Delete Router</a></li>
+						{%- endif %}
+						{%- if authadmin %}
+						<li><a href="#" onclick="$('#act').val('ban'); $('#actform').submit()">Ban Router</a></li>
+						<li><a href="#" onclick="$('#blockedform').submit()">{{ "Remove blocked status" if router.blocked else "Mark as blocked" }}</a></li>
+						{%- endif %}
+						<li><a href="#" onclick="$('#act').val('report'); $('#actform').submit()">Report abusive/illegal content</a></li>
 					</ul>
 				</div>
 			</form>
@@ -63,16 +84,20 @@
 					<script type="text/javascript">
 						var url_get_nearest_router = "{{ url_for('api.get_nearest_router') }}";
 						var url_router_info = "{{ url_for('router_info', dbid='') }}";
+						var url_load_neigh_stats = "{{ url_for('api.load_neighbor_stats', dbid='%s' % router.id) }}";
+						var url_load_netif_stats = "{{ url_for('api.load_netif_stats', dbid='%s' % router.id) }}";
 						var tileurls = {{ tileurls|tojson|safe }};
 					</script>
 					<script src="{{ url_for('static', filename='js/map.js') }}"></script>
 					<script type="text/javascript">
-						{%- if router.position %}
-						var router_pos = [{{ router.position.coordinates[1] }}, {{ router.position.coordinates[0] }}];
+						initialMap();
+						initialLayers();
+						{%- if router.lng and router.lat %}
+						var router_pos = [{{ router.lat }}, {{ router.lng }}];
 						map.setView(router_pos, 18);
 						var marker = L.marker(router_pos, {
 							icon: L.icon({
-								iconUrl: "{{ url_for('static', filename='img/router_blue.svg') }}",
+								iconUrl: "{{ url_for('static', filename='img/router_blue_white.svg') if router.wan_uplink else url_for('static', filename='img/router_blue.svg') }}",
 								iconSize: [14, 14]
 							}),
 							clickable: false
@@ -97,7 +122,7 @@
 						</td></tr>
 						<tr><th>Status</th><td><span class="{{ router.status|status2css }}">{{ router.status }}</span>
 							{%- if router.status == "online" %}
-							({{ router.system.uptime|format_ts_diff }} up)
+							({{ router.sys_uptime|format_ts_diff }} up)
 							{%- endif -%}
 						</td></tr>
 						<tr><th>Created</th><td>
@@ -105,10 +130,10 @@
 						</td></tr>
 						<tr><th class="text-nowrap">Last contact</th><td>
 							{{ router.last_contact|utc2local|format_dt }}
-							({{ router.last_contact|format_dt_ago }}){{- "" -}}
+							({{ router.last_contact|utc2local|format_dt_ago }}){{- "" -}}
 						</td></tr>
-						{%- if router.system.status_text %}
-						<tr><th>Status Text</th><td>{{ router.system.status_text }}</td></tr>
+						{%- if router.status_text %}
+						<tr><th>Status Text</th><td>{{ router.status_text }}</td></tr>
 						{%- endif %}
 						{%- if router.description %}
 						<tr><th>Description</th><td>{{ router.description }}</td></tr>
@@ -116,60 +141,88 @@
 						{%- if router.position_comment %}
 						<tr><th>Position</th><td>{{ router.position_comment }}</td></tr>
 						{%- endif %}
-						{%- if router.hood %}
-						<tr><th>Hood</th><td><a href="{{ url_for('router_list', q='hood:%s' % router.hood) }}">{{ router.hood }}</a>
-							{%- if router.community %}
-							({{ router.community }})
+						{%- if router.hoodname %}
+						<tr><th>Hood</th><td><span{%- if router.local %} class="hoodlocal"{%- elif router.v2 %} class="hoodv2"{%- endif %}><a href="{{ url_for('router_list', q='hood:^%s$' % router.hoodname) }}">{{ router.hoodname }}</a></span>
+							{%- if router.community and router.community != 'franken' %}
+							({{ router.community }},
+							{%- elif router.local %}
+							(local hood,
+							{%- elif router.v2 %}
+							(V2,
+							{%- else %}
+							(V1,
+							{%- endif -%}
+							&nbsp;<a href="{{ url_for('global_hoodstatistics', selecthood='%s' % router.hoodid) }}">Hood-Stats</a>)
+							{%- if router.reset %}
+							<span style="color:#d90000">- Router has lost its position!</span>
+							{%- elif not router.lat and not router.lng %}
+							<span style="color:#d90000">- Router has no position!</span>
 							{%- endif -%}
 						</td></tr>
 						{%- endif %}
-						{%- if router.user or router.system.contact %}
 						<tr><th>User</th><td>
 						{%- if router.user -%}
-						<a href="{{ url_for('user_info', nickname=router.user.nickname) }}">{{ router.user.nickname }}</a>
-						{%- if router.system.contact %}
-						(<a href="{{ url_for('router_list', q='system.contact:%s' % router.system.contact|anon_email_regex) }}">{{ router.system.contact|anon_email }}</a>)
+						<a href="{{ url_for('user_info', nickname=router.user) }}">{{ router.user }}</a>
+						{%- if router.contact %}
+						(<a href="{{ url_for('router_list', q='contact:%s' % router.contact|anon_email_regex) }}">{{ router.contact|anon_email }}</a>)
 						{%- endif -%}
 						{%- else -%}
-						<a href="{{ url_for('router_list', q='system.contact:%s' % router.system.contact|anon_email_regex) }}">{{ router.system.contact|anon_email }}</a>
+						{%- if router.contact %}
+						<a href="{{ url_for('router_list', q='contact:%s' % router.contact|anon_email_regex) }}">{{ router.contact|anon_email }}</a>
+						{%- else -%}
+						<span style="color:#d90000">FFF routers must have a contact address, but none is set.<br />Please provide a valid e-mail address!</span>
+						{%- endif -%}
 						{%- endif -%}
 						</td></tr>
-						{%- endif %}
-						<tr><th>Hardware</th><td><span title="{{ router.hardware.chipset }}">{{ router.hardware.name }}</span></td></tr>
-						<tr><th>WAN Uplink</th><td><span class="{{ "glyphicon glyphicon-ok" if router.system.has_wan_uplink else "glyphicon glyphicon-remove" }}"></span></td></tr>
-						<tr><th>Clients</th><td>{{ router.system.clients }}</td></tr>
+						<tr><th>Hardware</th><td><span title="{{ router.chipset }}"><a href="{{ url_for('router_list', q='hardware:^%s$' % router.hardware|format_query) }}">{{ router.hardware }}</a></span></td></tr>
+						<tr><th>WAN Uplink</th><td><span class="{{ "glyphicon glyphicon-ok" if router.wan_uplink else "glyphicon glyphicon-remove" }}"></span>
+						{%- if router.blocked and not router.v2 %}
+						<span style="color:#d90000"> &nbsp; - &nbsp; Router BLOCKED by KeyXchange!</span>
+						{%- endif -%}
+						</td></tr>
+						{%- if router.tc_enabled != None %}
+						<tr><th>Traffic control</th><td><span class="{{ "glyphicon glyphicon-ok" if router.tc_enabled else "glyphicon glyphicon-remove" }}"></span>
+						{%- if router.tc_enabled %} &nbsp; (up: {{ router.tc_out }} kBit/s, down: {{ router.tc_in }} kBit/s){%- endif -%}
+						</td></tr>
+						{%- endif -%}
+						<tr><th>Clients</th><td><span class="clientinfo">{{ router.clients }}</span>
+						{%- if router.clients_eth or router.clients_w2 or router.clients_w5 %}
+						 &nbsp; (Ethernet: <span class="clientinfo">{{ router.clients_eth }}</span>{%- if router.clients_w2 != None %}, 2.4 GHz: <span class="clientinfo">{{ router.clients_w2 }}</span>{%- endif -%}{%- if router.clients_w5 != None %}, 5 GHz: <span class="clientinfo">{{ router.clients_w5 }}</span>{%- endif -%})
+						{%- endif -%}
+						</td></tr>
+						{%- if router.w2_airtime != None or router.w5_airtime != None -%}
+						<tr><th>Airtime</th><td>
+						{%- if router.w2_airtime != None -%}2.4 GHz: <span class="clientinfo">{{ router.w2_airtime|format_airtime }}</span>{%- endif -%}{%- if router.w2_airtime != None and router.w5_airtime != None -%},&nbsp;{%- endif -%}{%- if router.w5_airtime != None -%}5 GHz: <span class="clientinfo">{{ router.w5_airtime|format_airtime }}</span>{%- endif -%}
+						</td></tr>
+						{%- endif -%}
 					</table>
 				</div>
 			</div>
 		</div>
 	</div>
 	<div class="row" style="display: flex;">
-		{%- if router.software is defined %}
 		<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
 			<div class="panel panel-default">
 				<div class="panel-heading">Software</div>
 				<div class="panel-body">
 					<table class="table table-condensed">
-						<tr><th>Firmware</th><td><span title="{{ router.software.firmware_rev }}">{{ router.software.firmware }}</span></td></tr>
-						<tr><th>Operating&nbsp;System</th><td>{{ router.software.os }}</td></tr>
-						<tr><th>Kernel</th><td>{{ router.software.kernel }}</td></tr>
-						<tr><th>B.A.T.M.A.N. adv</th><td>{{ router.software.batman_adv }}</td></tr>
-						<tr><th>Nodewatcher</th><td>{{ router.software.nodewatcher }}</td></tr>
+						<tr><th>Firmware</th><td><span title="{{ router.firmware_rev }}"><a href="{{ url_for('router_list', q='firmware:^%s$' % router.firmware|format_query) }}">{{ router.firmware }}</a></span></td></tr>
+						<tr><th>Operating&nbsp;System</th><td><a href="{{ url_for('router_list', q='os:^%s$' % router.os|format_query) }}">{{ router.os }}</a></td></tr>
+						<tr><th>Kernel</th><td><a href="{{ url_for('router_list', q='kernel:^%s$' % router.kernel|format_query) }}">{{ router.kernel }}</a></td></tr>
+						<tr><th>B.A.T.M.A.N. adv</th><td><a href="{{ url_for('router_list', q='batman:^%s$' % router.batman|format_query) }}">{{ router.batman }}</a>{%- if router.routing_protocol -%}&nbsp;&nbsp;&nbsp;-&nbsp;&nbsp;&nbsp;Routing: {{ router.routing_protocol }}{%- endif -%}</td></tr>
+						<tr><th>Nodewatcher</th><td><a href="{{ url_for('router_list', q='nodewatcher:^%s$' % router.nodewatcher|format_query) }}">{{ router.nodewatcher }}</a></td></tr>
 					</table>
 				</div>
 			</div>
 		{%- if not router.neighbours|length > 0 %}
 		</div>
 		<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
-		{%- endif %}
-		{%- else %}
-		<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
 		{%- endif %}
 			<div class="panel panel-default" style="flex: 1 1 auto;">
 				<div class="panel-heading">Events</div>
-				<div class="panel-body">
+				<div class="panel-body" style="max-height:186px;overflow-y:auto">
 					<table class="table table-condensed">
-					{%- for event in router.events[-5:] %}
+					{%- for event in router.events[-250:][::-1] %}
 						<tr>
 							<td style="width: 11em;">{{ event.time|utc2local|format_dt|nbsp|safe }}</td>
 							<td style="width: 1em;"><span class="{{ event.type|status2css }}">{{ event.type }}</span></td>
@@ -177,13 +230,14 @@
 						</tr>
 					{%- endfor %}
 					</table>
+					<div style="height:10px"></div>
 				</div>
 			</div>
 		</div>
 		{%- if router.neighbours|length > 0 %}
 		<div class="col-xs-12 col-md-6" style="display: flex; flex-flow: column;">
 			<div class="panel panel-default" style="flex: 1 1 auto;">
-				<div class="panel-heading">Neighbours</div>
+				<div class="panel-heading">Neighbours <span id="loadneighstats">(<a  href="#" onclick="load_neigh_stats();return false;">Load full stats</a>)</span></div>
 				<div class="panel-body" style="height: 100%;">
 					<div class="table-responsive">
 						<table class="neighbours" style="width: 100%; margin-bottom: 6px;">
@@ -194,11 +248,12 @@
 								<th>Interface</th>
 							</tr>
 							{%- for neighbour in router.neighbours %}
-							<tr style="background-color: {{ neighbour.quality|neighbour_color }};">
-								<td><a href="{{ url_for('router_info', dbid=neighbour._id) }}">{{ neighbour.hostname }}</a></td>
-								<td>{{ neighbour.mac }}</td>
+							<tr style="background-color: {{ neighbour.color }};">
+								
+								<td>{%- if neighbour.hostname -%}<a href="{{ url_for('router_info', dbid=neighbour.id) }}" style="color:#000000">{{ neighbour.hostname }}</a>{%- else -%}---{%- endif -%}</td>
+								<td>{{ neighbour.mac|int2mac }}</td>
 								<td>{{ neighbour.quality }}</td>
-								<td>{{ neighbour.net_if }}</td>
+								<td>{{ neighbour.netif }}</td>
 							</tr>
 							{%- endfor %}
 						</table>
@@ -224,66 +279,120 @@
 				</div>
 				<ul class="list-group" id="netif-list">
 				{# make sure that br-mesh is on top of the list #}
-				{%- for netif in router.netifs if netif.name == 'br-mesh' %}
-					<li class="list-group-item active" data-name="{{ netif.name|replace('.', '')|replace('$', '') }}">
-						<h4 class="list-group-item-heading"><div class="row">
-							<div class="col-xs-6 col-sm-6">{{ netif.name }}</div>
-							<div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div>
-						</div></h4>
-						<p class="list-group-item-text"><div class="row">
+				{%- for netif in router.netifs if netif.netif == 'br-mesh' %}
+					<li class="list-group-item active" data-name="{{ netif.netif|replace('.', '')|replace('$', '') }}">
+						<div class="row">
+							<div class="col-xs-7 col-sm-7"><h4 class="list-group-item-heading">br-mesh: <span class="netifdesc">Bridge</span></h4></div>
+							<div class="col-xs-5 col-sm-5 text-right" style="text-transform: uppercase;"><h4 class="list-group-item-heading">{{ netif.mac|int2mac }}</h4></div>
+						</div>
+						<div class="row">
 							<div class="col-xs-5 col-sm-5">
-								{%- if netif.ipv6_fe80_addr -%}
-								{{ netif.ipv6_fe80_addr }}
+								{%- if netif.fe80_addr -%}
+								{{ netif.fe80_addr|bin2ipv6 }}
 								{%- else -%}
-								<em title="Calculated from MAC Address">{{ netif.mac|mac2fe80 }}</em>
+								<em title="Calculated from MAC Address">{{ netif.mac|macint2fe80 }}</em>
 								{%- endif -%}
 								{%- if netif.ipv4_addr -%}
-								<br />{{ netif.ipv4_addr }}
+								<br />{{ netif.ipv4_addr|int2ipv4 }}
 								{%- endif -%}
 								{%- for ipv6_addr in netif.ipv6_addrs -%}
-								<br />{{ ipv6_addr }}
+								<br />{{ ipv6_addr|bin2ipv6 }}
 								{%- endfor -%}
 							</div>
-							{%- if netif.traffic.rx is defined %}
+							{%- if netif.rx is defined %}
 							<div class="col-xs-7 col-sm-7 text-right">
-								<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.traffic.rx|humanize_bytes }}/s
-								<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.traffic.tx|humanize_bytes }}/s
+								<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.rx|bytes_to_bits }}/s
+								<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.tx|bytes_to_bits }}/s
 							</div>
 							{%- endif %}
-						</div></p>
+						</div>
 					</li>
 				{%- endfor %}
-				{%- for netif in router.netifs if netif.name != 'br-mesh' %}
-					<li class="list-group-item" data-name="{{ netif.name|replace('.', '')|replace('$', '') }}">
-						<h4 class="list-group-item-heading"><div class="row">
-							<div class="col-xs-6 col-sm-6">{{ netif.name }}</div>
-							<div class="col-xs-6 col-sm-6 text-right" style="text-transform: uppercase;">{{ netif.mac }}</div>
-						</div></h4>
-						<p class="list-group-item-text"><div class="row">
+				{%- for netif in router.netifs if netif.netif != 'br-mesh' %}
+					<li class="list-group-item" data-name="{{ netif.netif|replace('.', '')|replace('$', '') }}">
+						<div class="row">
+							<div class="col-xs-7 col-sm-7"><h4 class="list-group-item-heading" style="{%- if netif.color -%}color:{{ netif.color }}{%- endif -%}">{{ netif.netif }}{%- if netif.description -%}: <span class="netifdesc">{{ netif.description }}</span>{%- endif %}</h4></div>
+							<div class="col-xs-5 col-sm-5 text-right" style="text-transform: uppercase;"><h4 class="list-group-item-heading">{{ netif.mac|int2mac }}</h4></div>
+						</div>
+						<div class="row">
+							<div class="col-xs-12 col-sm-12" style="padding-bottom:6px;font-size:14px">
+							{%- if netif.wlan_type -%}
+							<span class="wlaninfo">{{netif.wlan_type}}</span>,&nbsp;
+							{%- endif -%}
+							{%- if netif.wlan_channel -%}
+							Channel: <span class="wlaninfo">{{netif.wlan_channel}}</span>,&nbsp;
+							{%- endif -%}
+							{%- if netif.wlan_ssid -%}
+							SSID: <span class="wlaninfo">{{netif.wlan_ssid}}</span>,&nbsp;
+							{%- endif -%}
+							{%- if netif.wlan_txpower -%}
+							Tx-Power: <span class="wlaninfo">{{netif.wlan_txpower}}</span>
+							{%- endif -%}
+							</div>
+						</div>
+						<div class="row">
 							<div class="col-xs-5 col-sm-5">
-								{%- if netif.ipv6_fe80_addr -%}
-								{{ netif.ipv6_fe80_addr }}
+								{%- if netif.fe80_addr -%}
+								{{ netif.fe80_addr|bin2ipv6 }}
 								{%- else -%}
-								<em title="Calculated from MAC Address">{{ netif.mac|mac2fe80 }}</em>
+								<em title="Calculated from MAC Address">{{ netif.mac|macint2fe80 }}</em>
 								{%- endif -%}
 								{%- if netif.ipv4_addr -%}
-								<br />{{ netif.ipv4_addr }}
+								<br />{{ netif.ipv4_addr|int2ipv4 }}
 								{%- endif -%}
 								{%- for ipv6_addr in netif.ipv6_addrs -%}
-								<br />{{ ipv6_addr }}
+								<br />{{ ipv6_addr|bin2ipv6 }}
 								{%- endfor -%}
 							</div>
-							{%- if netif.traffic.rx is defined %}
+							{%- if netif.rx is defined %}
 							<div class="col-xs-7 col-sm-7 text-right">
-								<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.traffic.rx|humanize_bytes }}/s
-								<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.traffic.tx|humanize_bytes }}/s
+								<span class="glyphicon glyphicon-arrow-down"></span>{{ netif.rx|bytes_to_bits }}/s
+								<span class="glyphicon glyphicon-arrow-up"></span>{{ netif.tx|bytes_to_bits }}/s
 							</div>
 							{%- endif %}
-						</div></p>
+						</div>
 					</li>
 				{%- endfor %}
 				</ul>
 			</div>
+			{%- if router.gws|length > 0 %}
+			<div class="panel panel-default" style="flex: 1 1 auto;">
+				<div class="panel-heading">Gateways</div>
+				<div class="panel-body" style="height: 100%;">
+					<div class="table-responsive">
+						<table class="neighbours" style="width: 100%; margin-bottom: 6px;">
+							<tr>
+								<th>Gateway</th>
+								<th>batX</th>
+								<th>Qual</th>
+								<th>Netif</th>
+								<th>Class</th>
+							</tr>
+							{%- for gw in router.gws %}
+							{%- if gw.selected %}
+							<tr style="background-color:#04ff0a">
+							{%- else %}
+							<tr>
+							{%- endif %}
+								<td><a href="{{ url_for('global_gwstatistics', selectgw='%s' % gw.mac|int2shortmac) }}">{{ gw.label }}</a></td>
+								<td>{{ gw.batX }}</td>
+								<td>{{ gw.quality }}</td>
+								<td>{{ gw.netif }}</td>
+								<td>{{ gw.gw_class }}</td>
+							</tr>
+							{%- endfor %}
+						</table>
+					</div>
+					{# hack for graph vertical align #}
+					{%- if  router.gws|length < 3 %}
+					{%- for n in range(3- router.gws|length) %}
+					<br />
+					{%- endfor %}
+					{%- endif %}
+					<div id="gwstat" class="graph" style="height: 150px;"></div>
+				</div>
+			</div>
+			{%- endif %}
 		</div>
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
@@ -298,6 +407,12 @@
 					<div id="loadstat" class="graph"></div>
 				</div>
 			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">Airtime (requires Firmware-Update)</div>
+				<div class="panel-body">
+					<div id="airstat" class="graph"></div>
+				</div>
+			</div>
 			<div class="panel panel-default">
 				<div class="panel-heading">Memory</div>
 				<div class="panel-body">
@@ -314,27 +429,66 @@
 	</div>
 	<script type="text/javascript">
 		var router_stats = {{ router.stats|statbson2json|safe }};
+		var netif_stats = null;
+		var neigh_stats = {{ neighstats|safe }};
+		var neigh_label = {{ neighlabel|bson2json|safe }};
+		var gw_stats = {{ gwstats|statbson2json|safe }};
 		var neighbours = [
 		{%- for neighbour in router.neighbours %}
-			{"name": "{{ neighbour.hostname or neighbour.mac }}", "mac": "{{ neighbour.mac }}", "net_if": "{{ neighbour.net_if }}"},
+			{"name": "{{ neighbour.hostname or neighbour.mac|int2mac }}", "mac": "{{ neighbour.mac }}", "netif": "{{ neighbour.netif }}"},
+		{%- endfor %}
+		];
+		var gws = [
+		{%- for gw in router.gws %}
+			{"name": "{{ gw.label }}", "mac": "{{ gw.mac }}", "netif": "{{ gw.netif }}"},
 		{%- endfor %}
 		];
 		$(document).ready(function() {
 			{%- if router.neighbours|length > 0 %}
-			neighbour_graph(neighbours);
+			neighbour_graph(neigh_label);
+			{%- endif %}
+			{%- if router.gws|length > 0 %}
+			gw_graph(gws);
 			{%- endif %}
 			memory_graph();
 			process_graph();
 			client_graph();
 			loadavg_graph();
+			airtime_graph();
 
 			$("#netif-list li").on("click", function() {
 				$("#netif-list li").removeClass("active");
 				var netif = this.getAttribute("data-name");
-				network_graph(netif);
+				load_netif_stats(netif);
 				$(this).addClass("active");
 			});
-			network_graph("br-mesh");
+			load_netif_stats("br-mesh");
 		});
+		{%- if router.neighbours|length > 0 %}
+		function load_neigh_stats() {
+			$("#loadneighstats").css('font-style', 'italic');
+			$("#loadneighstats").html("(Loading ...)");
+			var starttimeneigh = performance.now();
+			ajax_get_request(url_load_neigh_stats, function(neighstats) {
+				neigh_stats = neighstats;
+				neighbour_graph(neigh_label) && $("#loadneighstats").hide();
+				console.debug("Loaded full neighbor stats in "+((performance.now() - starttimeneigh)/1000).toFixed(3)+" seconds.");
+				});
+		}
+		{%- endif %}
+		function load_netif_stats(netif) {
+			var starttimenetif = performance.now();
+			ajax_get_request(url_load_netif_stats + "?netif=" + netif, function(netifstats) {
+				network_graph(netifstats,"netstat","tx","rx");
+				console.debug("Loaded netif stats for "+netif+" in "+((performance.now() - starttimenetif)/1000).toFixed(3)+" seconds.");
+				});
+		}
 	</script>
+
+{%- if session.admin %}
+<form method="post" id="blockedform">
+	<input type="hidden" name="act" value="changeblocked" />
+	<input type="hidden" name="blocked" value="{{ "false" if router.blocked else "true" }}" />
+</form>
+{%- endif %}
 {% endblock %}

ffmap/web/templates/router_list.html

@@ -29,48 +29,45 @@
 	<table id="routerlist" class="table table-condensed table-striped table-bordered" style="margin-bottom: 8px;">
 		<thead>
 			<tr>
-				<th style="width: 1%; padding-right: 5px; min-width: 90px;">Hostname</th>
+				<th style="width: 1%; padding-right: 5px; min-width: 240px;">Hostname</th>
 				<th style="width: 45px; padding-right: 5px;">Status</th>
 				<th style="padding-right: 5px;">Hood</th>
 				<th style="padding-right: 5px;">User</th>
 				<th style="padding-right: 5px;">Hardware</th>
 				<th style="padding-right: 5px;">Created</th>
 				<th style="padding-right: 5px;">Uptime</th>
-				<th>Clients</th>
+				<th style="padding-right: 5px;">Last contact</th>
+				<th>Users</th>
 			</tr>
 		</thead>
 		<tbody>
 			{%- for router in routers %}
 			<tr>
-				<td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router._id) }}">{{ router.hostname }}</a></td>
-				<td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
-				<td>{{ router.hood }}</td>
-				<td>{{ router.user.nickname if router.user }}</td>
-				<td class="text-nowrap">{{ router.get("hardware", {}).get("name", "") }}</td>
+				<td class="text-responsive"><a href="{{ url_for("router_info", dbid=router.id) }}">{{ router.hostname }}</a>
+				{%- if not router.lat and not router.lng %} - <span style="color:#d90000">Reset!</span>{%- endif %}{%- if router.blocked and not router.v2 %} - <span style="color:#d90000">Blocked!</span>{%- endif %}
+				</td>
+				<td class="text-center" data-order="{{ router.status }}"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
+				<td{%- if router.local %} class="hoodlocal"{%- elif router.v2 %} class="hoodv2"{%- endif %}>{{ router.hood }}</td>
+				<td>{%- if router.nickname %}{{ router.nickname }}{%- elif not router.contact %}<span style="color:#d90000">missing</span>{%- endif %}</td>
+				<td class="text-nowrap">{{ router.hardware }}</td>
 				<td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td>
-				<td class="text-nowrap">{{ router.system.uptime|format_ts_diff }}</td>
-				<td>{{ router.system.clients }}</td>
+				<td class="text-nowrap" data-order="{{ router.sys_uptime if router.status == "online" else 0 }}">{{ router.sys_uptime|format_ts_diff }}</td>
+				<td class="text-nowrap">{{ router.last_contact|utc2local|format_dt_date }}</td>
+				<td>{{ router.clients }}</td>
 			</tr>
 			{%- endfor %}
 		</tbody>
 	</table>
 </div>
 <div style="margin-bottom: 20px;">
-	{{ routers.count() }} Router{{ "s" if (routers.count() == 1) else "" }} found.
+	{{ numrouters }} Router{{ "s" if (numrouters == 1) else "" }} found.
 </div>
 <script type="text/javascript">
 	$(document).ready(function() {
 		$("#routerlist").DataTable({
 			"paging": false,
 			"info": false,
-			"searching": false,
-			/*"responsive": {
-				"details": false
-			},*/
-			"columnDefs": [
-				{"orderable": false, "targets": 1},
-				{"orderable": false, "targets": -2},
-			]
+			"searching": false
 		});
 	});
 </script>

ffmap/web/templates/statistics.html

@@ -1,5 +1,5 @@
 {% extends "bootstrap.html" %}
-{% block title %}{{super()}} :: Statistics{% endblock %}
+{% block title %}{{super()}} :: Statistics{%- if selecthood %} for {{ selecthoodname }}{%- endif -%}{%- if selectgw %} for GW {{ selectgw }}{%- endif -%}{% endblock %}
 {% block head %}{{super()}}
 	<script src="{{ url_for('static', filename='js/graph/date.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/graph/jquery.flot.js') }}"></script>
@@ -12,6 +12,9 @@
 	<script src="{{ url_for('static', filename='js/graph/jquery.flot.pie.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/graph/jquery.flot.tooltip.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/jquery.dataTables.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/dataTables.bootstrap.min.js') }}"></script>
+	<link rel="stylesheet" href="{{ url_for('static', filename='css/datatables/dataTables.bootstrap.min.css') }}">
 	<style type="text/css">
 		.table-condensed {
 			margin-bottom: 0;
@@ -33,6 +36,15 @@
 				padding-right: 3px !important;
 			}
 		}
+		.table-hoods th {
+			text-align: center;
+		}
+		.table-hoods td {
+			text-align: center;
+		}
+		.table-hoods .firstrow {
+			text-align: left;
+		}
 	</style>
 {% endblock %}
 
@@ -40,43 +52,75 @@
 	<div class="row">
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Hoods</div>
+				<div class="panel-heading">Hoods - V1: <a href="#" id="enablev1">On</a>, V2: <a href="#" id="enablev2">On</a>, Local: <a href="#" id="enablelocal">On</a></div>
 				<div class="panel-body">
-					<table class="table table-condensed">
+					<table id="hoodlist" class="table table-condensed table-hoods">
+					<thead>
 						<tr>
-							<th>Hood</th>
-							<th class="success" title="Online Routers">Online</th>
-							<th class="danger" title="Offline Routers">Offline</th>
-							<th class="warning" title="Unknown Routers">Unknown</th>
-							<th class="active" title="Total Routers">Total</th>
-							<th class="info">Clients</th>
+							<th class="firstrow">Hood</th>
+							<th class="stats" title="Gateways">G</th>
+							<th class="success" title="Online Routers">On</th>
+							<th class="danger" title="Offline Routers">Off</th>
+							<th class="warning" title="Unknown Routers">Unk.</th>
+							<th class="active" title="Total Routers">Sum</th>
+							<th class="info">User</th>
+							<th class="stats">List</th>
 						</tr>
-						{%- for hood, value in hoods|dictsort %}
-						<tr>
-							<td><a href="{{ url_for('router_list', q='hood:%s' % hood) }}">{{ hood }}</a></td>
+					</thead>
+					<tbody>
+						{%- for hoodid, value in hoods|dictsort %}
+						<tr{%- if hoods_sum[hoodid]["local"] %} class="rowlocal"{%- elif hoods_sum[hoodid]["v2"] %} class="rowv2"{%- else %} class="rowv1"{%- endif %}>
+							<td class="firstrow{%- if hoods_sum[hoodid]["local"] %} hoodlocal{%- elif hoods_sum[hoodid]["v2"] %} hoodv2{%- endif %}"><a href="{{ url_for('global_hoodstatistics', selecthood='%s' % hoodid) }}">{{ value['name'] }}</a></td>
+							<td class="stats">{{ hoods_gws[hoodid] or "-" }}</td>
 							<td class="success">{{ value["online"] or 0 }}</td>
-							<td class="danger">{{ value["offline"] or 0 }}</td>
+							<td class="danger" data-order="{{ value["offline"] or 0 }}">{{ value["offline"] or 0 }}{%- if value["orphaned"] %} ({{ value["orphaned"] or 0 }}){%- endif %}</td>
 							<td class="warning">{{ value["unknown"] or 0 }}</td>
-							<td class="active">{{ hoods_sum[hood]["routers"] or 0 }}</td>
-							<td class="info">{{ hoods_sum[hood]["clients"] or 0 }}</td>
+							<td class="active">{{ hoods_sum[hoodid]["routers"] }}</td>
+							<td class="info">{{ hoods_sum[hoodid]["clients"] }}</td>
+							<td class="stats"><a href="{{ url_for('router_list', q='hood:^%s$' % value['name'].replace(' ','_')) }}">List</a></td>
 						</tr>
 						{%- endfor %}
+					</tbody>
+					<tfoot>
 						<tr>
-							<th>Sum</th>
+							<th class="firstrow"><a href="{{ url_for('global_statistics') }}">All Hoods</a></th>
+							<td class="stats">&nbsp;</td>
 							<td class="success">{{ router_status.online or 0 }}</td>
 							<td class="danger">{{ router_status.offline or 0 }}</td>
 							<td class="warning">{{ router_status.unknown or 0 }}</td>
-							<td class="active">{{ (router_status.online or 0) + (router_status.offline or 0) + (router_status.unknown or 0) }}</td>
+							<td class="active">{{ router_status.sum or 0 }}</td>
 							<td class="info">{{ clients }}</td>
+							<td class="stats">&nbsp;</td>
 						</tr>
+					</tfoot>
 					</table>
 				</div>
 			</div>
 		</div>
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Newest Routers</div>
+				<div class="panel-heading">Routers{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }} (selected only){%- endif -%}</div>
 				<div class="panel-body">
+					<div id="globrouterstat" class="graph"></div>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">Clients{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }} (selected only){%- endif -%}</div>
+				<div class="panel-body">
+					<div id="globclientstat" class="graph"></div>
+				</div>
+			</div>
+			{%- if not selectgw %}
+			<div class="panel panel-default">
+				<div class="panel-heading">Traffic{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}</div>
+				<div class="panel-body">
+					<div id="netstat" class="graph"></div>
+				</div>
+			</div>
+			{%- endif -%}
+			<div class="panel panel-default">
+				<div class="panel-heading">Newest Routers{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
+				<div class="panel-body" style="padding-bottom:34px">
 					<div class="table-responsive">
 						<table class="table table-condensed">
 							<tr>
@@ -86,7 +130,7 @@
 							</tr>
 							{%- for router in newest_routers|reverse %}
 							<tr>
-								<td><a href="{{ url_for('router_info', dbid=router._id) }}">{{ router.hostname }}</a></td>
+								<td><a href="{{ url_for('router_info', dbid=router.id) }}">{{ router.hostname }}</a></td>
 								<td>{{ router.hood }}</td>
 								<td class="text-nowrap">{{ router.created|utc2local|format_dt }}</td>
 							</tr>
@@ -100,35 +144,89 @@
 	<div class="row">
 		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Routers</div>
+				<div class="panel-heading">Gateways (selected / others){%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}</div>
 				<div class="panel-body">
-					<div id="globrouterstat" class="graph"></div>
+					<table id="gwlist" class="table table-condensed table-hoods">
+					<thead>
+						<tr>
+							<th class="firstrow">Gateway</th>
+							<th class="success" title="Online Routers">On</th>
+							<th class="danger" title="Offline Routers">Off</th>
+							<th class="warning" title="Unknown Routers">Unk.</th>
+							<th class="active" title="Total Routers">Sum</th>
+							<th class="stats">List</th>
+						</tr>
+					</thead>
+					<tbody>
+						{%- for mac, value in gws.items() %}
+						<tr>
+							<td class="firstrow" data-order="{{ value["sort"] }}"><p style="margin:0"><a href="{{ url_for('global_gwstatistics', selectgw='%s' % mac|int2shortmac) }}">{{ gws_info[mac]["label"] }}</a></p>
+							{%- if gws_info[mac]["gw"] %}
+							<p style="margin:0;font-size:12px">{{ mac|int2mac }}
+							{%- if gws_info[mac]["batmac"] %}
+							 / {{ gws_info[mac]["batmac"]|int2mac }}
+							{%- endif %}
+							</p>
+							{%- endif %}
+							</td>
+							<td class="success" data-order="{{ (value["selected"]["online"] or 0) + (value["others"]["online"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["online"] or 0 }}</span> / {{ value["others"]["online"] or 0 }}</td>
+							<td class="danger" data-order="{{ (value["selected"]["offline"] or 0) + (value["others"]["offline"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["offline"] or 0 }}</span> / {{ value["others"]["offline"] or 0 }}</td>
+							<td class="warning" data-order="{{ (value["selected"]["unknown"] or 0) + (value["others"]["unknown"] or 0) }}"><span style="font-weight:bold">{{ value["selected"]["unknown"] or 0 }}</span> / {{ value["others"]["unknown"] or 0 }}</td>
+							<td class="active" data-order="{{ (value["selected"]|sumdict if value["selected"] else 0) + (value["others"]|sumdict if value["others"] else 0) }}"><span style="font-weight:bold">{{ gws_sum[mac]["routers"] if gws_sum[mac] else 0 }}</span> / {{ value["others"]|sumdict if value["others"] else 0 }}</td>
+							<td class="stats"><a href="{{ url_for('router_list', q='selected:^%s$' % mac|int2shortmac) }}">Sel</a>/<a href="{{ url_for('router_list', q='gw:^%s$' % mac|int2shortmac) }}">All</a></td>
+						</tr>
+						{%- endfor %}
+					</tbody>
+					</table>
 				</div>
 			</div>
 		</div>
 		<div class="col-xs-12 col-md-6">
+			{%- if selectgw %}
 			<div class="panel panel-default">
-				<div class="panel-heading">Clients</div>
+				<div class="panel-heading">Gateway-Details</div>
 				<div class="panel-body">
-					<div id="globclientstat" class="graph"></div>
+					<table class="table table-condensed">
+						<tr><th>Gateway</th><td>{{ gws_info[selectgwint]["gw"] }}</td></tr>
+						<tr><th>Interface</th><td>{{ gws_info[selectgwint]["gwif"] }}</td></tr>
+						<tr><th>MAC address</th><td>{{ selectgw }}</td></tr>
+						<tr><th>BatX interface</th><td>{{ gws_info[selectgwint]["batX"] }}</td></tr>
+						{%- if gws_info[selectgwint]["ipv4"] %}
+						<tr><th>Internal IPv4</th><td>{{ gws_info[selectgwint]["ipv4"] }}</td></tr>
+						{%- endif %}
+						{%- if gws_info[selectgwint]["ipv6"] %}
+						<tr><th>Internal IPv6</th><td>{{ gws_info[selectgwint]["ipv6"] }}</td></tr>
+						{%- endif %}
+						{%- if gws_info[selectgwint]["dhcpstart"] %}
+						<tr><th>DHCP range</th><td>{{ gws_info[selectgwint]["dhcpstart"] }} - {{ gws_info[selectgwint]["dhcpend"] }}</td></tr>
+						{%- endif %}
+						{%- if gws_info[selectgwint]["stats_page"] %}
+						<tr><th>Stats page</th><td>{{ gws_info[selectgwint]["stats_page"] }}</td></tr>
+						{%- endif %}
+						{%- for a in gws_admin %}
+						<tr><th>Admin</th><td>{{ a }}</td></tr>
+						{%- endfor %}
+						<tr><th>gwinfo version</th><td>{{ gws_info[selectgwint]["version"] if gws_info[selectgwint]["version"] else "< 1.4 or custom" }}</td></tr>
+					</table>
 				</div>
 			</div>
-		</div>
-	</div>
-	<div class="row">
-		<div class="col-xs-12 col-md-6">
+			{%- endif %}
 			<div class="panel panel-default">
-				<div class="panel-heading">Routers Firmwares</div>
+				<div class="panel-heading">Router Firmwares{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
 				<div class="panel-body">
-					<div id="globrouterfwstat" class="graph"></div>
+					<div id="globrouterfwstat" class="graph-pie"></div>
 				</div>
 			</div>
-		</div>
-		<div class="col-xs-12 col-md-6">
 			<div class="panel panel-default">
-				<div class="panel-heading">Routers Models</div>
+				<div class="panel-heading">Router Models{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
 				<div class="panel-body">
-					<div id="globroutermodelsstat" class="graph"></div>
+					<div id="globroutermodelsstat" class="graph-pie"></div>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">Router Models per Client{%- if selecthood %} @ {{ selecthoodname }}{%- endif -%}{%- if selectgw %} @ {{ selectgw }}{%- endif -%}</div>
+				<div class="panel-body">
+					<div id="globroutermodelsperclient" class="graph-pie"></div>
 				</div>
 			</div>
 		</div>
@@ -138,11 +236,77 @@
 		var router_firmwares = {{ router_firmwares|tojson }};
 		var router_models = {{ router_models|tojson }};
 		var routers_page_url = "{{ url_for('router_list') }}";
+		{%- if selecthood %}
+		var hood = "{{selecthoodname}}";
+		var hoodstr = "hood:^" + hood.replace(/ /g, '_') + "$";
+		{%- else %}
+		var hoodstr = "";
+		{%- endif -%}
 		$(document).ready(function() {
-			global_client_graph();
-			global_router_graph();
+			global_client_graph(global_stats,"globclientstat");
+			global_router_graph(global_stats,"globrouterstat");
 			global_router_firmwares_graph();
-			global_router_models_graph();
+			global_router_models_graph("globroutermodelsstat","count");
+			global_router_models_graph("globroutermodelsperclient","clients");
+			{%- if not selectgw %}
+			network_graph(global_stats,"netstat","sent to clients","received from clients");
+			{%- endif -%}
+			
+			$("#hoodlist").DataTable({
+				"order": [[0,'asc']],
+				"paging": false,
+				"info": false,
+				"searching": false,
+				/*"responsive": {
+					"details": false
+				},*/
+				"columnDefs": [
+					{"orderable": false, "targets": -1},
+				]
+			});
+			
+			$("#gwlist").DataTable({
+				"order": [],
+				"paging": false,
+				"info": false,
+				"searching": false,
+				/*"responsive": {
+					"details": false
+				},*/
+				"columnDefs": [
+					{"orderable": false, "targets": -1},
+				]
+			});
+			
+			function enableHood(aid,classname) {
+				var avx = document.getElementById(aid)
+				var rows = document.getElementsByClassName(classname)
+				if(avx.text=="On") {
+					avx.text = "Off";
+					for (var i = 0; i < rows.length; i++) {
+						rows[i].style.display = 'none';
+					}
+				} else {
+					avx.text = "On";
+					for (var i = 0; i < rows.length; i++) {
+						rows[i].style.display = '';
+					}
+				}
+				return true
+			}
+			
+			document.getElementById("enablev1").onclick = function() {
+				enableHood("enablev1","rowv1")
+				return false;
+			}
+			document.getElementById("enablev2").onclick = function() {
+				enableHood("enablev2","rowv2")
+				return false;
+			}
+			document.getElementById("enablelocal").onclick = function() {
+				enableHood("enablelocal","rowlocal")
+				return false;
+			}
 		});
 	</script>
 {% endblock %}

ffmap/web/templates/user.html

@@ -30,12 +30,14 @@
 				<span class="caret"></span>
 			</button>
 			<ul class="dropdown-menu">
-				{# FIXME: If authorized #}
+				{%- if authuser %}
 				<li><a href="#" data-toggle="modal" data-target="#changepw">Change Password</a></li>
 				<li><a href="#" data-toggle="modal" data-target="#changemail">Change E-Mail Address</a></li>
-				{%- if session.admin %}
 				<li><a href="#" onclick="$('#delaccform').submit()">Delete Account</a></li>
+				{%- endif %}
+				{%- if authadmin %}
 				<li><a href="#" onclick="$('#adminform').submit()">Toggle admin</a></li>
+				<li><a href="#" onclick="$('#abuseform').submit()">Toggle abuse</a></li>
 				{%- endif %}
 			</ul>
 		</div>
@@ -46,7 +48,7 @@
 	<div class="hidden-xs col-sm-2">
 		<div class="panel panel-default">
 			<div class="panel-body" style="padding: 5px;">
-				<a href="https://de.gravatar.com/" rel="nofollow" title="&Auml;ndere dein Avatar auf gravatar.com"><img id="avatar" class="img-responsive center-block" src="{{ user.get('email', '')|gravatar_url }}&s=150" style="width: 150px; height: 150px;" /></a>
+				<a href="https://de.gravatar.com/" rel="nofollow" title="&Auml;ndere dein Avatar auf gravatar.com"><img id="avatar" alt="Avatar" class="img-responsive center-block" src="{{ user.get('email', '')|gravatar_url }}&s=150" style="width: 150px; height: 150px;" /></a>
 			</div>
 		</div>
 	</div>
@@ -57,12 +59,11 @@
 				<table class="table table-condensed">
 					<tr><th>Nickname</th><td>
 						{{ user.nickname }}
-						(<a href="{{ url_for('router_list', q='user.nickname:%s' % user.nickname) }}">Filter</a>)
+						(<a href="{{ url_for('router_list', q='nickname:^%s$' % user.nickname.replace(" ","_")) }}">Filter</a>)
 					</td></tr>
 					{%- if user.email %}
 					<tr><th>E-Mail</th><td>
 						{{ user.email|anon_email }}
-						(<a href="{{ url_for('router_list', q='system.contact:%s' % user.email|anon_email_regex) }}">Filter</a>)
 					</td></tr>
 					{%- endif %}
 					<tr><th>Created</th><td>
@@ -76,6 +77,9 @@
 					<tr><th>Admin</th><td>
 						<span class="glyphicon glyphicon-{%- if user.admin -%}ok{%- else -%}remove{%- endif -%}"></span>
 					</td></tr>
+					<tr><th>Receive abuse reports</th><td>
+						<span class="glyphicon glyphicon-{%- if user.abuse -%}ok{%- else -%}remove{%- endif -%}"></span>
+					</td></tr>
 				</table>
 			</div>
 		</div>
@@ -99,36 +103,31 @@
 			{%- set total_clients = 0 %}
 			{%- for router in routers %}
 			<tr>
-				<td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router._id) }}">{{ router.hostname }}</a></td>
-				<td class="text-center"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
-				<td>{{ router.hood }}</td>
-				<td>{{ router.software.firmware }}</td>
-				<td class="text-nowrap">{{ router.get("hardware", {}).get("name", "") }}</td>
+				<td class="text-nowrap-responsive"><a href="{{ url_for("router_info", dbid=router.id) }}">{{ router.hostname }}</a>
+				{%- if not router.lat and not router.lng %} - <span style="color:#d90000">Reset!</span>{%- endif %}{%- if router.blocked and not router.v2 %} - <span style="color:#d90000">Blocked!</span>{%- endif %}
+				</td>
+				<td class="text-center" data-order="{{ router.status }}"><span class="{{ router.status|status2css }}">{{ router.status }}</span></td>
+				<td{%- if router.local %} class="hoodlocal"{%- elif router.v2 %} class="hoodv2"{%- endif %}>{{ router.hood }}</td>
+				<td>{{ router.firmware }}</td>
+				<td class="text-nowrap">{{ router.get("hardware", "") }}</td>
 				<td class="text-nowrap">{{ router.created|utc2local|format_dt_date }}</td>
-				<td class="text-nowrap">{{ router.system.uptime|format_ts_diff }}</td>
-				<td>{{ router.system.clients }}</td>
-				{%- set total_clients = total_clients + router.system.clients %}
+				<td class="text-nowrap" data-order="{{ router.sys_uptime if router.status == "online" else 0 }}">{{ router.sys_uptime|format_ts_diff }}</td>
+				<td>{{ router.clients }}</td>
+				{%- set total_clients = total_clients + router.clients %}
 			</tr>
 			{%- endfor %}
 		</tbody>
 	</table>
 </div>
 <div style="margin-bottom: 20px;">
-	{{ routers.count() }} Router{{ "s" if (routers.count() == 1) else "" }} found.
+	{{ routers_count }} Router{{ "s" if (routers_count == 1) else "" }} found.
 </div>
 <script type="text/javascript">
 	$(document).ready(function() {
 		$("#routerlist").DataTable({
 			"paging": false,
 			"info": false,
-			"searching": false,
-			/*"responsive": {
-				"details": false
-			},*/
-			"columnDefs": [
-				{"orderable": false, "targets": 1},
-				{"orderable": false, "targets": -2},
-			]
+			"searching": false
 		});
 	});
 </script>
@@ -202,11 +201,17 @@
 	</div>
 </div>
 
-{%- if session.admin %}
+{%- if authadmin %}
 <form method="post" id="adminform">
 	<input type="hidden" name="action" value="changeadmin" />
 	<input type="hidden" name="admin" value="{{ "false" if user.admin else "true" }}" />
 </form>
+<form method="post" id="abuseform">
+	<input type="hidden" name="action" value="changeabuse" />
+	<input type="hidden" name="abuse" value="{{ "false" if user.abuse else "true" }}" />
+</form>
+{%- endif %}
+{%- if authuser %}
 <form method="post" id="delaccform">
 	<input type="hidden" name="action" value="deleteaccount" />
 </form>

ffmap/web/templates/user_list.html

@@ -24,6 +24,7 @@
 				<th style="padding-right: 5px;">E-Mail</th>
 				<th style="padding-right: 5px;">Admin</th>
 				<th style="padding-right: 5px;">Created</th>
+				<th style="padding-right: 5px;">V2</th>
 				<th style="padding-right: 5px;">Routers</th>
 				<th style="padding-right: 5px;">Clients</th>
 			</tr>
@@ -41,15 +42,16 @@
 						<span class="glyphicon glyphicon-remove" title="Automatically imported stub from netmon"></span>
 					{%- endif -%}
 				</td>
-				<td>{{ user_routers.get(user.nickname, {}).get('routers', 0) }}</td>
-				<td>{{ user_routers.get(user.nickname, {}).get('clients', 0) }}</td>
+				<td style="{{users_v2.get(user.email.lower(), {})|v2colorpercent }}" data-order="{{ users_v2.get(user.email.lower(), {})|v2userpercent }}">{{ users_v2.get(user.email.lower(), {})|v2userpercent }} %</td>
+				<td>{{ user_routers.get(user.email.lower(), {}).get('routers', 0) }}</td>
+				<td>{{ user_routers.get(user.email.lower(), {}).get('clients', 0) }}</td>
 			</tr>
 			{%- endfor %}
 		</tbody>
 	</table>
 </div>
 <div style="margin-bottom: 20px;">
-	{{ users.count() }} User{{ "s" if (users.count() > 1) else "" }} found.
+	{{ users_count }} User{{ "s" if (users_count > 1) else "" }} found.
 </div>
 <script type="text/javascript">
 	$(document).ready(function() {

ffmap/web/templates/v2routers.html

@@ -0,0 +1,108 @@
+{% extends "bootstrap.html" %}
+{% block title %}{{super()}} :: Statistics{% endblock %}
+{% block head %}{{super()}}
+	<script src="{{ url_for('static', filename='js/graph/date.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.time.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.byte.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.selection.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.downsample.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.resize.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.hiddengraphs.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.pie.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph/jquery.flot.tooltip.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/jquery.dataTables.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/datatables/dataTables.bootstrap.min.js') }}"></script>
+	<link rel="stylesheet" href="{{ url_for('static', filename='css/datatables/dataTables.bootstrap.min.css') }}">
+	<style type="text/css">
+		.table-condensed {
+			margin-bottom: 0;
+		}
+		.table-condensed tr:last-child td, th {
+			border-bottom: 1px solid #ddd;
+		}
+		@media(max-width:500px) {
+			th {
+				padding-left: 2px !important;
+				padding-right: 2px !important;
+			}
+			td {
+				padding-left: 2px !important;
+				padding-right: 2px !important;
+			}
+			.panel-body {
+				padding-left: 3px !important;
+				padding-right: 3px !important;
+			}
+		}
+		.table-hoods th {
+			text-align: center;
+		}
+		.table-hoods td {
+			text-align: center;
+		}
+		.table-hoods .firstrow {
+			text-align: left;
+		}
+	</style>
+{% endblock %}
+
+{% block content %}
+	<div class="row">
+		<div class="col-xs-12 col-md-6">
+		<div class="panel panel-default">
+			<div class="panel-heading">Routers V1: {{ statsv1[-1]["online"] }} on, {{ statsv1[-1]["offline"] }} off, {{ statsv1[-1]["unknown"] }} unknown, {{ statsv1[-1]["orphaned"] }} orphaned; {{ statsv1[-1]["online"]+statsv1[-1]["offline"]+statsv1[-1]["unknown"]+statsv1[-1]["orphaned"] }} total</div>
+			<div class="panel-body">
+				<div id="globrouterstat1" class="graph"></div>
+			</div>
+		</div>
+		<div class="panel panel-default">
+			<div class="panel-heading">Clients V1: {{ statsv1[-1]["clients"] }}</div>
+			<div class="panel-body">
+				<div id="globclientstat1" class="graph"></div>
+			</div>
+		</div>
+		<div class="panel panel-default">
+			<div class="panel-heading">Traffic V1</div>
+			<div class="panel-body">
+				<div id="netstat1" class="graph"></div>
+			</div>
+		</div>
+		</div>
+		<div class="col-xs-12 col-md-6">
+			<div class="panel panel-default">
+				<div class="panel-heading">Routers V2: {{ statsv2[-1]["online"] }} on, {{ statsv2[-1]["offline"] }} off, {{ statsv2[-1]["unknown"] }} unknown, {{ statsv2[-1]["orphaned"] }} orphaned; {{ statsv2[-1]["online"]+statsv2[-1]["offline"]+statsv2[-1]["unknown"]+statsv2[-1]["orphaned"] }} total</div>
+				<div class="panel-body">
+					<div id="globrouterstat" class="graph"></div>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">Clients V2: {{ statsv2[-1]["clients"] }}</div>
+				<div class="panel-body">
+					<div id="globclientstat" class="graph"></div>
+				</div>
+			</div>
+			<div class="panel panel-default">
+				<div class="panel-heading">Traffic V2</div>
+				<div class="panel-body">
+					<div id="netstat" class="graph"></div>
+				</div>
+			</div>
+		</div>
+	</div>
+	<script type="text/javascript">
+		var global_stats_v2 = {{ statsv2|statbson2json|safe }};
+		var global_stats_v1 = {{ statsv1|statbson2json|safe }};
+		var routers_page_url = "{{ url_for('router_list') }}";
+		$(document).ready(function() {
+			global_client_graph(global_stats_v2,"globclientstat");
+			global_router_graph(global_stats_v2,"globrouterstat");
+			network_graph(global_stats_v2,"netstat","sent to clients","received from clients");
+			global_client_graph(global_stats_v1,"globclientstat1");
+			global_router_graph(global_stats_v1,"globrouterstat1");
+			network_graph(global_stats_v1,"netstat1","sent to clients","received from clients");
+		});
+	</script>
+{% endblock %}
+

gwinfo/gwinfofirmware.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Gateway data script for FFF Monitoring
+# Copyright Adrian Schmutzler, 2018.
+# License GPLv3
+#
+# designed for GATEWAY FIRMWARE
+#
+# v1.4.6 - 2018-10-17
+# - Fix IPv4/IPv6 sed (leading space in match pattern)
+#
+# v1.4.3 - 2018-08-28
+# - Added version to json
+# - GW-Firmware: Only append IPv4/IPv6/DHCP to bat0
+#
+# v1.4.2 - 2018-08-28
+# - Fixed IPv4 sed to ignore subnet mask
+# - Check for multiple IPv6 addresses
+# - GW-Firmware: Ignore wireless devices
+# - GW-Firmware: Use eth device from batctl if
+# - GW-Firmware: Use only br-mesh for batctl if
+# - GW-Firmware: Select fd43 address with ::
+# - GW-Firmware: Adjust DHCP to uci
+#
+# v1.4.1 - 2018-08-25
+# - Fixed greps for IPv4/IPv6/dnsmasq
+#
+# v1.4 - 2018-08-23
+# - Transmit internal IPv4/IPv6
+# - Transmit DHCP range for dnsmasq
+#
+# v1.3 - 2018-08-23
+# - Support multiple Monitoring URLs
+# - Use https by default
+# - Changed batctl default path
+#
+# v1.2.1 - 2018-01-12
+# - Added "grep fff" to support L2TP
+#
+# v1.2 - 2018-01-12
+# - Added batctl command and vpnif
+#
+# v1.1 - 2018-01-12
+# - Initial Version
+#
+
+# Config
+api_urls="https://monitoring.freifunk-franken.de/api/gwinfo" # space-separated list of addresses (api_urls="url1 url2")
+batctlpath=/usr/sbin/batctl
+hostname="$(uci -q get system.@system[0].hostname)"
+statslink="$(uci -q get gateway.@gateway[0].statslink)"
+
+# Code
+tmp=$(/bin/mktemp)
+echo "{\"version\":\"1.4.6\",\"hostname\":\"$hostname\",\"stats_page\":\"$statslink\",\"netifs\":[" > $tmp
+
+comma=""
+for netif in $(ls /sys/class/net); do
+	if [ "$netif" = "lo" ] || echo "$netif" | grep -q "w" ; then # remove wXap, wXmesh, etc.
+		continue
+	fi
+	mac="$(cat "/sys/class/net/$netif/address")"
+	batctl="$("$batctlpath" -m "$netif" if | grep "eth" | sed -n 's/:.*//p')"
+
+	ipv4=""
+	ipv6=""
+	dhcpstart=""
+	dhcpend=""
+	if [ "$netif" = "bat0" ]; then
+		ipv4="$(ip -4 addr show dev br-mesh | grep " 10\." | sed 's/.* \(10\.[^ ]*\/[^ ]*\) .*/\1/')"
+		ipv6="$(ip -6 addr show dev br-mesh | grep " fd43" | grep '::' | sed 's/.* \(fd43[^ ]*\) .*/\1/')"
+		[ "$(echo "$ipv6" | wc -l)" = "1" ] || ipv6=""
+		dhcpstart="$(uci -q get dhcp.mesh.start)"
+	fi
+
+	echo "$comma{\"mac\":\"$mac\",\"netif\":\"$netif\",\"vpnif\":\"$batctl\",\"ipv4\":\"$ipv4\",\"ipv6\":\"$ipv6\",\"dhcpstart\":\"$dhcpstart\",\"dhcpend\":\"$dhcpend\"}" >> $tmp
+	comma=","
+done
+
+echo "],\"admins\":[" >> $tmp
+
+comma=""
+for admin in $(uci -q get gateway.@gateway[0].admin); do
+	echo "$comma\"$admin\"" >> $tmp && comma=","
+done
+
+echo "]}" >> $tmp
+
+for api_url in $api_urls; do
+	/usr/bin/curl -k -v -H "Content-type: application/json; charset=UTF-8" -X POST --data-binary @$tmp $api_url
+done
+/bin/rm "$tmp"

gwinfo/sendgwinfo.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+#
+# Gateway data script for FFF Monitoring
+# Copyright Adrian Schmutzler, 2018.
+# License GPLv3
+#
+# designed for GATEWAY SERVER
+#
+# v1.4.6 - 2018-10-17
+# - Fix IPv4/IPv6 sed (leading space in match pattern)
+#
+# v1.4.5 - 2018-08-29
+# - Fix one bug regarding DHCP range processing
+#
+# v1.4.4 - 2018-08-29
+# - Fix two bugs regarding DHCP range processing
+#
+# v1.4.3 - 2018-08-28
+# - Added version to json
+#
+# v1.4.2 - 2018-08-28
+# - Fixed IPv4 sed to ignore subnet mask
+# - Check for multiple IPv6 addresses
+# - Provide experimental support for isc-dhpc-server
+#
+# v1.4.1 - 2018-08-25
+# - Fixed greps for IPv4/IPv6/dnsmasq
+#
+# v1.4 - 2018-08-23
+# - Transmit internal IPv4/IPv6
+# - Transmit DHCP range for dnsmasq
+#
+# v1.3 - 2018-08-23
+# - Support multiple Monitoring URLs
+# - Use https by default
+# - Changed batctl default path
+#
+# v1.2.1 - 2018-01-12
+# - Added "grep fff" to support L2TP
+#
+# v1.2 - 2018-01-12
+# - Added batctl command and vpnif
+#
+# v1.1 - 2018-01-12
+# - Initial Version
+#
+
+# Config
+api_urls="https://monitoring.freifunk-franken.de/api/gwinfo" # space-separated list of addresses (api_urls="url1 url2")
+batctlpath=/usr/sbin/batctl # Adjust to YOUR path!
+hostname="MyHost"
+admin1="Admin"
+admin2=
+admin3=
+statslink="" # Provide link to stats page (MRTG or similar)
+dhcp=1 # 0=disabled, 1=dnsmasq, 2=isc-dhcp-server
+
+# Code
+tmp=$(/bin/mktemp)
+echo "{\"version\":\"1.4.6\",\"hostname\":\"$hostname\",\"stats_page\":\"$statslink\",\"netifs\":[" > $tmp
+
+comma=""
+for netif in $(ls /sys/class/net); do
+	if [ "$netif" = "lo" ] ; then
+		continue
+	fi
+	mac="$(cat "/sys/class/net/$netif/address")"
+	batctl="$("$batctlpath" -m "$netif" if | grep "fff" | sed -n 's/:.*//p')"
+
+	ipv4="$(ip -4 addr show dev "$netif" | grep " 10\." | sed 's/.* \(10\.[^ ]*\/[^ ]*\) .*/\1/')"
+	ipv6="$(ip -6 addr show dev "$netif" | grep " fd43" | sed 's/.* \(fd43[^ ]*\) .*/\1/')"
+	[ "$(echo "$ipv6" | wc -l)" = "1" ] || ipv6=""
+
+	dhcpstart=""
+	dhcpend=""
+	if [ "$dhcp" = "1" ]; then
+		dhcpdata="$(ps ax | grep "dnsmasq" | grep "$netif " | sed 's/.*dhcp-range=\([^ ]*\) .*/\1/')"
+		dhcpstart="$(echo "$dhcpdata" | cut -d',' -f1)"
+		dhcpend="$(echo "$dhcpdata" | cut -d',' -f2)"
+	elif [ "$dhcp" = "2" ]; then
+		ipv4cut="${ipv4%/*}"
+		if [ -n "$ipv4cut" ] && grep -q "routers $ipv4cut" /etc/dhcp/dhcpd.conf; then
+			dhcpdata="$(sed -z 's/.*range \([^;]*\);[^}]*option routers '$ipv4cut'.*/\1/' /etc/dhcp/dhcpd.conf)"
+			dhcpstart="$(echo "$dhcpdata" | cut -d' ' -f1)"
+			dhcpend="$(echo "$dhcpdata" | cut -d' ' -f2)"
+		fi
+	fi
+
+	echo "$comma{\"mac\":\"$mac\",\"netif\":\"$netif\",\"vpnif\":\"$batctl\",\"ipv4\":\"$ipv4\",\"ipv6\":\"$ipv6\",\"dhcpstart\":\"$dhcpstart\",\"dhcpend\":\"$dhcpend\"}" >> $tmp
+	comma=","
+done
+
+echo "],\"admins\":[" >> $tmp
+
+comma=""
+[ -n "$admin1" ] && echo "\"$admin1\"" >> $tmp && comma=","
+[ -n "$admin2" ] && echo "$comma\"$admin2\"" >> $tmp && comma=","
+[ -n "$admin3" ] && echo "$comma\"$admin3\"" >> $tmp
+
+echo "]}" >> $tmp
+
+for api_url in $api_urls; do
+	/usr/bin/curl -k -v -H "Content-type: application/json; charset=UTF-8" -X POST --data-binary @$tmp $api_url
+done
+/bin/rm "$tmp"

install.sh

@@ -5,9 +5,9 @@ mkdir -vp /var/lib/ffmap/csv
 chown -R www-data:www-data /var/lib/ffmap
 
 mkdir -vp /usr/share/ffmap
-cp -v ffmap/mapnik/{hoods,hoodsv2,links_and_routers}.xml /usr/share/ffmap
-sed -i -e 's#>csv/#>/var/lib/ffmap/csv/#' /usr/share/ffmap/{hoods,hoodsv2,links_and_routers}.xml
-chown www-data:www-data /usr/share/ffmap/{hoods,hoodsv2,links_and_routers}.xml
+cp -v ffmap/mapnik/{hoods_v2,hoods_poly,routers,routers_v2,routers_local}.xml /usr/share/ffmap
+sed -i -e 's#>csv/#>/var/lib/ffmap/csv/#' /usr/share/ffmap/{hoods_v2,hoods_poly,routers,routers_v2,routers_local}.xml
+chown www-data:www-data /usr/share/ffmap/{hoods_v2,hoods_poly,routers,routers_v2,routers_local}.xml
 
 cp -v ffmap/mapnik/tilestache.cfg /usr/share/ffmap
 cp -rv ffmap/web/static /usr/share/ffmap
@@ -21,4 +21,4 @@ systemctl daemon-reload
 
 python3 setup.py install --force
 
-(cd ffmap/mapnik; python2 setup.py install)
+(cd ffmap/mapnik; python3 setup.py install)

restart.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+printf "\nStopping ...\n\n"
+systemctl stop uwsgi-tiles
+systemctl stop uwsgi-ffmap
+
+./install.sh
+
+printf "\nStarting ...\n\n"
+systemctl start uwsgi-ffmap
+systemctl start uwsgi-tiles
+
+printf "Done.\n\n"

scripts/calcglobalstats.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+
+# Execute every 5 min, 2 mins after alfred comes in (sleep 120 in cron)
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.routertools import *
+from ffmap.maptools import *
+from ffmap.mysqltools import FreifunkMySQL
+from ffmap.stattools import record_global_stats, record_hood_stats, record_gw_stats
+from ffmap.hoodtools import update_hoods_v2
+
+import time
+start_time = time.time()
+
+mysql = FreifunkMySQL()
+detect_offline_routers(mysql)
+detect_orphaned_routers(mysql)
+delete_orphaned_routers(mysql)
+#delete_old_stats(mysql) # Only execute once daily, takes 2 minutes
+update_hoods_v2(mysql)
+record_global_stats(mysql)
+record_hood_stats(mysql)
+record_gw_stats(mysql)
+update_mapnik_csv(mysql)
+mysql.close()
+
+print("--- %.3f seconds ---" % (time.time() - start_time))

scripts/copyusers.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+
+import pymongo
+from bson.json_util import dumps as bson2json
+from bson.objectid import ObjectId
+import base64
+import datetime
+
+client = MongoClient(tz_aware=True, connect=False)
+db = client.freifunk
+
+users = db.users.find({}, {"nickname": 1, "password":1, "email": 1, "token": 1, "created": 1, "admin": 1})
+
+mysql = FreifunkMySQL()
+cur = mysql.cursor()
+for u in users:
+	#print(u)
+	cur.execute("""
+		INSERT INTO users (nickname, password, token, email, created, admin)
+		VALUES (%s, %s, %s, %s, %s, %s)
+	""",(u.get("nickname"),u.get("password"),u.get("token"),u.get("email",""),u.get("created"),u.get("admin",0),))
+mysql.commit()
+mysql.close()

scripts/crontiles.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+command="systemctl restart uwsgi-tiles"
+append="2>&1 | /usr/bin/logger -t uwsgi-tiles"
+
+if crontab -l | grep -q "$command" ; then
+	echo "Cron already set."
+	exit 1
+fi
+
+# Runs at X:14
+(crontab -l 2>/dev/null; echo "14 * * * * $command $append") | crontab -
+
+echo "Cron set successfully."
+exit 0
+

scripts/csv2users.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.mysqltools import FreifunkMySQL
+
+import pymongo
+from bson.json_util import dumps as bson2json
+from bson.objectid import ObjectId
+import base64
+import datetime
+
+import csv
+
+targetfile = "/data/fff/users.txt"
+
+mysql = FreifunkMySQL()
+data = []
+with open(targetfile, newline='') as csvfile:
+	spamreader = csv.reader(csvfile, delimiter=';')
+	for row in spamreader:
+		if row[5]=="None":
+			row[5]=None
+		if row[1]=="None":
+			row[1]=None
+		if row[1]=="None":
+			row[1]=None
+		if row[2]=="None":
+			row[2]=None
+		if row[3]=="None":
+			row[3]=None
+		if row[4]=="True":
+			row[4]=1
+		else:
+			row[4]=0
+		row[3] = datetime.datetime.strptime(''.join(row[3].rsplit(':', 1)),"%Y-%m-%d %H:%M:%S.%f%z").strftime('%Y-%m-%d %H:%M:%S')
+
+		data.append((row[0],row[5],row[1],row[2],row[3],row[4],))
+
+mysql.executemany("""
+	INSERT INTO users (nickname, password, token, email, created, admin)
+	VALUES (%s, %s, %s, %s, %s, %s)
+""",data)
+mysql.commit()
+mysql.close()

scripts/defragtable.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+# Execute manually
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.misc import defrag_table, writelog
+from ffmap.config import CONFIG
+from ffmap.mysqltools import FreifunkMySQL
+
+import time
+start_time = time.time()
+
+mysql = FreifunkMySQL()
+i = 1
+while i < len(sys.argv):
+	defrag_table(mysql,sys.argv[i],1)
+	i = i + 1
+mysql.close()
+
+writelog(CONFIG["debug_dir"] + "/deletetime.txt", "-------")
+print("--- Total defrag duration: %.3f seconds ---" % (time.time() - start_time))

scripts/defragtables.py

@@ -0,0 +1,22 @@
+#!/usr/bin/python3
+
+# Execute manually
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.misc import defrag_all
+from ffmap.mysqltools import FreifunkMySQL
+
+import time
+start_time = time.time()
+
+mysql = FreifunkMySQL()
+if(len(sys.argv)>1):
+	defrag_all(mysql,sys.argv[1])
+else:
+	defrag_all(mysql,False)
+mysql.close()
+
+print("--- Total defrag duration: %.3f seconds ---" % (time.time() - start_time))

scripts/deletestats.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+# Execute once daily, also 2 min after full 5 mins (so it does not coincide with alfred)
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.routertools import delete_old_stats
+from ffmap.mysqltools import FreifunkMySQL
+
+import time
+start_time = time.time()
+
+mysql = FreifunkMySQL()
+delete_old_stats(mysql)
+mysql.close()
+
+print("--- Total duration: %.3f seconds ---" % (time.time() - start_time))

scripts/deleteunlinked.py

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+# Deletes unlinked rows from gw_* and router_* tables
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.routertools import delete_unlinked_routers
+from ffmap.gwtools import delete_unlinked_gws
+from ffmap.mysqltools import FreifunkMySQL
+
+import time
+start_time = time.time()
+
+mysql = FreifunkMySQL()
+delete_unlinked_routers(mysql)
+delete_unlinked_gws(mysql)
+mysql.close()
+
+print("\n--- Total duration: %.3f seconds ---\n" % (time.time() - start_time))

scripts/readpolyhoods.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+
+# Execute manually
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from ffmap.hoodtools import update_hoods_poly
+from ffmap.mysqltools import FreifunkMySQL
+
+mysql = FreifunkMySQL()
+update_hoods_poly(mysql)
+mysql.commit()
+mysql.close()

scripts/setupcron.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+monpath=/data/fff/fff-monitoring
+
+if crontab -l | grep -q "$monpath" ; then
+	echo "Cron already set."
+	exit 1
+fi
+
+# Runs every 5 min and waits 3 min
+(crontab -l 2>/dev/null; echo "3-59/5 * * * * $monpath/scripts/calcglobalstats.py 2>&1 | /usr/bin/logger -t calcglobalstats") | crontab -
+
+# Runs at 4:02
+(crontab -l 2>/dev/null; echo "2 4 * * * $monpath/scripts/deletestats.py 2>&1 | /usr/bin/logger -t deletestats") | crontab -
+
+echo "Cron set successfully."
+exit 0
+

scripts/users2csv.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from pymongo import MongoClient
+from bson.json_util import dumps as bson2json
+from bson.objectid import ObjectId
+import base64
+import datetime
+
+targetfile = "/data/fff/users.txt"
+
+client = MongoClient(tz_aware=True, connect=False)
+db = client.freifunk
+
+users = db.users.find({}, {"nickname": 1, "password":1, "email": 1, "token": 1, "created": 1, "admin": 1})
+
+with open(targetfile, "wb") as csv:
+	for u in users:
+		str = "%s;%s;%s;%s;%s;%s\n" % (u.get("nickname"),u.get("token"),u.get("email",""),u.get("created"),u.get("admin",0),u.get("password"))
+		csv.write(str.encode("UTF-8"))

setup.py

@@ -9,7 +9,7 @@ setup(
 	description='FF-MAP',
 	author='Dominik Heidler',
 	author_email='dominik@heidler.eu',
-	url='http://github.com/asdil12/ff-map',
+	url='https://github.com/FreifunkFranken/fff-monitoring',
 	#requires=['flask', 'flup'],
 	packages=['ffmap', 'ffmap.web'],
 	#scripts=['bin/aurbs'],

start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+printf "\nStarting ...\n\n"
+systemctl start uwsgi-ffmap
+systemctl start uwsgi-tiles

stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+printf "\nStopping ...\n\n"
+systemctl stop uwsgi-tiles
+systemctl stop uwsgi-ffmap