diff --git a/.gitignore b/.gitignore index d22b40f..ab0e100 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +build __pycache__ .*.swp diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4702bc --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +## Installation +``` +./install.sh +systemctl enable uwsgi-ffmap +systemctl enable uwsgi-tiles-links_and_routers +systemctl enable uwsgi-tiles-hoods +systemctl start uwsgi-ffmap +systemctl start uwsgi-tiles-links_and_routers +systemctl start uwsgi-tiles-hoods +# Then apply NGINX Config +``` + +## Debian Dependencies +``` +apt-get install python python3 mongodb python3-requests python3-lxml python3-pip python3-flask python3-dateutil python3-numpy python3-scipy python-mapnik python3-pip uwsgi-plugin-python uwsgi-plugin-python3 nginx +pip3 install pymongo +git clone https://github.com/asdil12/tilelite +cd tilelite +python setup.py install +``` + +## NGINX Config +``` +... + location / { + include uwsgi_params; + uwsgi_pass 127.0.0.1:3031; + } + + location /tiles/links_and_routers { + include uwsgi_params; + uwsgi_pass 127.0.0.1:3032; + } + + location /tiles/hoods { + include uwsgi_params; + uwsgi_pass 127.0.0.1:3033; + } +... +``` diff --git a/contrib/debug_webapp.sh b/contrib/debug_webapp.sh new file mode 100644 index 0000000..44279f6 --- /dev/null +++ b/contrib/debug_webapp.sh @@ -0,0 +1 @@ +uwsgi_python3 -w ffmap.web.application:app --http-socket :9090 --catch-exceptions diff --git a/db/alfred.py b/db/alfred.py index ffb87cf..e50c045 100755 --- a/db/alfred.py +++ b/db/alfred.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import socket import gzip diff --git a/db/hoods.py b/db/hoods.py index 378a8ae..fe6e34d 100644 --- a/db/hoods.py +++ b/db/hoods.py @@ -8,7 +8,7 @@ db = client.freifunk # create db indexes db.hoods.create_index([("position", "2dsphere")]) -db.hoods.insert_many([ +hoods = [ { "keyxchange_id": 1, "name": "default", @@ -61,4 +61,7 @@ db.hoods.insert_many([ "name": "HassbergeSued", "net": "10.50.60.0/22", "position": {"type": "Point", "coordinates": [10.568013390003, 50.08]} -}]) +}] + +for hood in hoods: + db.hoods.insert_one(hood) diff --git a/db/init_db.py b/db/init_db.py new file mode 100755 index 0000000..1c58e73 --- /dev/null +++ b/db/init_db.py @@ -0,0 +1,5 @@ +#!/usr/bin/python3 + +import routers +import chipsets +import hoods diff --git a/db/routers.py b/db/routers.py new file mode 100644 index 0000000..52e0baf --- /dev/null +++ b/db/routers.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 + +from pymongo import MongoClient +client = MongoClient() + +db = client.freifunk + +# create db indexes +db.routers.create_index("status") +db.routers.create_index("last_contact") +db.routers.create_index("netifs.mac") +db.routers.create_index([("position", "2dsphere")]) diff --git a/map/__init__.py b/ffmap/__init__.py similarity index 100% rename from map/__init__.py rename to ffmap/__init__.py diff --git a/ffmap/dbtools.py b/ffmap/dbtools.py new file mode 100644 index 0000000..bbff1ca --- /dev/null +++ b/ffmap/dbtools.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +from pymongo import MongoClient + +class FreifunkDB(object): + client = None + db = None + + @classmethod + def handle(cls): + if not cls.client: + cls.client = MongoClient() + if not cls.db: + cls.db = cls.client.freifunk + return cls.db diff --git a/ffmap/maptools.py b/ffmap/maptools.py new file mode 100644 index 0000000..1b613bd --- /dev/null +++ b/ffmap/maptools.py @@ -0,0 +1,114 @@ +#!/usr/bin/python3 + +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) + +from ffmap.dbtools import FreifunkDB + +import math +import numpy as np +from scipy.spatial import Voronoi + +db = FreifunkDB().handle() + +CONFIG = { + "csv_dir": "/var/lib/ffmap/csv" +} + +def touch(fname, times=None): + with open(fname, 'a'): + os.utime(fname, times) + +def update_mapnik_csv(): + with open(os.path.join(CONFIG["csv_dir"], "routers.csv"), "w") as csv: + csv.write("lng,lat,status\n") + for router in db.routers.find({"position.coordinates": {"$exists": True}}): + csv.write("%f,%f,%s\n" % ( + router["position"]["coordinates"][0], + router["position"]["coordinates"][1], + router["status"] + )) + + with open(os.path.join(CONFIG["csv_dir"], "links.csv"), "w") as csv: + csv.write("WKT,quality\n") + for router in db.routers.find({"position.coordinates": {"$exists": True}, "neighbours": {"$exists": True}}): + for neighbour in router["neighbours"]: + if "position" in neighbour: + csv.write("\"LINESTRING (%f %f,%f %f)\",%i\n" % ( + router["position"]["coordinates"][0], + router["position"]["coordinates"][1], + neighbour["position"]["coordinates"][0], + neighbour["position"]["coordinates"][1], + neighbour["quality"] + )) + + with open(os.path.join(CONFIG["csv_dir"], "hood-points.csv"), "w", encoding="UTF-8") as csv: + csv.write("lng,lat,name\n") + for hood in db.hoods.find({"position": {"$exists": True}}): + csv.write("%f,%f,\"%s\"\n" % ( + hood["position"]["coordinates"][0], + hood["position"]["coordinates"][1], + hood["name"] + )) + + with open(os.path.join(CONFIG["csv_dir"], "hoods.csv"), "w") as csv: + EARTH_RADIUS = 6378137.0 + + def merc_sphere(lng, lat): + x = math.radians(lng) * EARTH_RADIUS + y = math.log(math.tan(math.pi/4 + math.radians(lat)/2)) * EARTH_RADIUS + return (x,y) + + def merc_sphere_inv(x, y): + lng = math.degrees(x / EARTH_RADIUS) + lat = math.degrees(2*math.atan(math.exp(y / 6378137.0)) - math.pi/2) + return (lng,lat) + + csv.write("WKT\n") + hoods = [] + for hood in db.hoods.find({"position": {"$exists": True}}): + # convert coordinates info marcator sphere as voronoi doesn't work with lng/lat + x, y = merc_sphere(hood["position"]["coordinates"][0], hood["position"]["coordinates"][1]) + hoods.append([x, y]) + + points = np.array(hoods) + vor = Voronoi(points) + #mp = voronoi_plot_2d(vor) + #mp.show() + + lines = [vor.vertices[line] for line in vor.ridge_vertices if -1 not in line] + + for line in lines: + x = [line[0][0], line[1][0]] + y = [line[0][1], line[1][1]] + for i in range(len(x)-1): + # convert mercator coordinates back into lng/lat + lng1, lat1 = merc_sphere_inv(x[i], y[i]) + lng2, lat2 = merc_sphere_inv(x[i+1], y[i+1]) + csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2)) + + ptp_bound = np.array(merc_sphere(180, 360)) + center = vor.points.mean(axis=0) + + for pointidx, simplex in zip(vor.ridge_points, vor.ridge_vertices): + simplex = np.asarray(simplex) + if np.any(simplex < 0): + i = simplex[simplex >= 0][0] # finite end Voronoi vertex + + t = vor.points[pointidx[1]] - vor.points[pointidx[0]] # tangent + t /= np.linalg.norm(t) + n = np.array([-t[1], t[0]]) # normal + + midpoint = vor.points[pointidx].mean(axis=0) + direction = np.sign(np.dot(midpoint - center, n)) * n + far_point = vor.vertices[i] + direction * ptp_bound.max() + + # convert mercator coordinates back into lng/lat + lng1, lat1 = merc_sphere_inv(vor.vertices[i,0], vor.vertices[i,1]) + lng2, lat2 = merc_sphere_inv(far_point[0], far_point[1]) + csv.write("\"LINESTRING (%f %f,%f %f)\"\n" % (lng1, lat1, lng2, lat2)) + + # touch mapnik XML files to trigger tilelite watcher + touch("/usr/share/ffmap/hoods.xml") + touch("/usr/share/ffmap/links_and_routers.xml") diff --git a/map/nodewatcher.py b/ffmap/routertools.py similarity index 73% rename from map/nodewatcher.py rename to ffmap/routertools.py index 6a9424d..0dba24b 100644 --- a/map/nodewatcher.py +++ b/ffmap/routertools.py @@ -1,19 +1,22 @@ -#!/usr/bin/python +#!/usr/bin/python3 -import netmon +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '..')) + +from ffmap.dbtools import FreifunkDB import lxml.etree import datetime -from pymongo import MongoClient +import requests -client = MongoClient() -db = client.freifunk +db = FreifunkDB().handle() CONFIG = { "vpn_netif": "fffVPN", } -def process_router_xml(mac, xml): +def load_nodewatcher_xml(mac, xml): try: router = db.routers.find_one({"netifs.mac": mac.lower()}) if router: @@ -25,6 +28,7 @@ def process_router_xml(mac, xml): router_update = { "status": tree.xpath("/data/system_data/status/text()")[0], "hostname": tree.xpath("/data/system_data/hostname/text()")[0], + "last_contact": datetime.datetime.utcnow(), "neighbours": [], "netifs": [], "system": { @@ -118,14 +122,15 @@ def process_router_xml(mac, xml): if router: # keep hood up to date router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router["position"]}}})["name"] - db.routers.update_one({"netifs.mac": mac.lower()}, {"$set": router_update, "$currentDate": {"last_contact": True}}) + db.routers.update_one({"netifs.mac": mac.lower()}, {"$set": router_update}) else: # new router # fetch additional information from netmon as it is not yet contained in xml - router_info = netmon.fetch_router_info(mac) + router_info = netmon_fetch_router_info(mac) if router_info: # keep hood up to date router_update["hood"] = db.hoods.find_one({"position": {"$near": {"$geometry": router_info["position"]}}})["name"] + router_update["events"] = [] router_update.update(router_info) router_id = db.routers.insert_one(router_update).inserted_id status = router_update["status"] @@ -133,7 +138,8 @@ def process_router_xml(mac, xml): if router: db.routers.update_one({"_id": router_id}, {"$set": {"status": "unknown"}}) status = "unknown" - finally: + + if router_id: # fire events events = [] try: @@ -169,3 +175,66 @@ def process_router_xml(mac, xml): # calculate RRD statistics (rrdcache?) #FIXME: implementation pass + +def detect_offline_routers(): + db.routers.update_many({ + "last_contact": {"$lt": datetime.datetime.utcnow() - datetime.timedelta(minutes=10)}, + "status": {"$ne": "offline"} + }, { + "$set": {"status": "offline"}, + "$push": {"events": { + "$each": [{ + "time": datetime.datetime.utcnow(), + "type": "offline" + }], + "$slice": -10 + }} + }) + +def netmon_fetch_router_info(mac): + mac = mac.replace(":", "").lower() + tree = lxml.etree.fromstring(requests.get("https://netmon.freifunk-franken.de/api/rest/router/%s" % mac, params={"limit": 5000}).content) + + for r in tree.xpath("/netmon_response/router"): + user_netmon_id = int(r.xpath("user_id/text()")[0]) + user = db.users.find_one({"netmon_id": user_netmon_id}) + if user: + user_id = user["_id"] + else: + user_id = db.users.insert({ + "netmon_id": user_netmon_id, + "nickname": r.xpath("user/nickname/text()")[0] + }) + user = db.users.find_one({"_id": user_id}) + + router = { + "netmon_id": int(r.xpath("router_id/text()")[0]), + "user": {"nickname": user["nickname"], "_id": user["_id"]} + } + + try: + lng = float(r.xpath("longitude/text()")[0]) + lat = float(r.xpath("latitude/text()")[0]) + assert lng != 0 + assert lat != 0 + + router["position"] = { + "type": "Point", + "coordinates": [lng, lat] + } + + # try to get comment + position_comment = r.xpath("location/text()")[0] + if position_comment != "undefined": + router["position"]["comment"] = position_comment + except (IndexError, AssertionError): + pass + + try: + router["description"] = r.xpath("description/text()")[0] + except IndexError: + pass + + router["created"] = datetime.datetime.utcnow() + + return router diff --git a/ffmap/web/__init__.py b/ffmap/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/map/api.py b/ffmap/web/api.py similarity index 67% rename from map/api.py rename to ffmap/web/api.py index c722513..f591bb5 100644 --- a/map/api.py +++ b/ffmap/web/api.py @@ -1,6 +1,8 @@ -#!/usr/bin/python +#!/usr/bin/python3 -import nodewatcher +from ffmap.routertools import * +from ffmap.maptools import * +from ffmap.dbtools import FreifunkDB from flask import Blueprint, request, make_response from pymongo import MongoClient @@ -9,8 +11,7 @@ import json api = Blueprint("api", __name__) -client = MongoClient() -db = client.freifunk +db = FreifunkDB().handle() @api.route('/get_nearest_router') def get_nearest_router(): @@ -31,9 +32,12 @@ def alfred(): r = make_response(json.dumps(set_alfred_data)) if request.method == 'POST': alfred_data = request.get_json() - # load router status xml data - for mac, xml in alfred_data.get("64", {}).items(): - nodewatcher.process_router_xml(mac, xml) - r.headers['X-API-STATUS'] = "ALFRED data imported" + if alfred_data: + # load router status xml data + for mac, xml in alfred_data.get("64", {}).items(): + load_nodewatcher_xml(mac, xml) + r.headers['X-API-STATUS'] = "ALFRED data imported" + detect_offline_routers() + update_mapnik_csv() r.mimetype = 'application/json' return r diff --git a/map/ffmap.py b/ffmap/web/application.py similarity index 66% rename from map/ffmap.py rename to ffmap/web/application.py index 7947e0a..c57124f 100755 --- a/map/ffmap.py +++ b/ffmap/web/application.py @@ -1,10 +1,14 @@ -#!/usr/bin/python +#!/usr/bin/python3 -from api import api -from filters import filters +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/' + '../..')) + +from ffmap.web.api import api +from ffmap.web.filters import filters +from ffmap.dbtools import FreifunkDB from flask import Flask, render_template, request, make_response -from pymongo import MongoClient from bson.json_util import dumps as bson2json from bson.objectid import ObjectId import json @@ -13,12 +17,11 @@ app = Flask(__name__) app.register_blueprint(api, url_prefix='/api') app.register_blueprint(filters) -client = MongoClient() -db = client.freifunk +db = FreifunkDB().handle() tileurls = { - "links_and_routers": "http://localhost:8000", - "hoods": "http://localhost:8001", + "links_and_routers": "/tiles/links_and_routers", + "hoods": "/tiles/hoods", } @app.route('/') @@ -43,5 +46,10 @@ def router_list(): def router_info(dbid): return render_template("router.html", router=db.routers.find_one({"_id": ObjectId(dbid)}), tileurls=tileurls) + if __name__ == '__main__': app.run(host='0.0.0.0', debug=True) +else: + app.template_folder = "/usr/share/ffmap/templates" + app.static_folder = "/usr/share/ffmap/static" + #app.debug = True diff --git a/map/filters.py b/ffmap/web/filters.py similarity index 96% rename from map/filters.py rename to ffmap/web/filters.py index 5525636..1b34fcf 100644 --- a/map/filters.py +++ b/ffmap/web/filters.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from flask import Blueprint from dateutil import tz diff --git a/map/static/css/style.css b/ffmap/web/static/css/style.css similarity index 100% rename from map/static/css/style.css rename to ffmap/web/static/css/style.css diff --git a/map/static/img/offline.png b/ffmap/web/static/img/offline.png similarity index 100% rename from map/static/img/offline.png rename to ffmap/web/static/img/offline.png diff --git a/map/static/img/online.png b/ffmap/web/static/img/online.png similarity index 100% rename from map/static/img/online.png rename to ffmap/web/static/img/online.png diff --git a/map/static/img/router.svg b/ffmap/web/static/img/router.svg similarity index 100% rename from map/static/img/router.svg rename to ffmap/web/static/img/router.svg diff --git a/map/static/img/router_blue.svg b/ffmap/web/static/img/router_blue.svg similarity index 100% rename from map/static/img/router_blue.svg rename to ffmap/web/static/img/router_blue.svg diff --git a/map/static/img/router_direct_green.svg b/ffmap/web/static/img/router_direct_green.svg similarity index 100% rename from map/static/img/router_direct_green.svg rename to ffmap/web/static/img/router_direct_green.svg diff --git a/map/static/img/router_direct_red.svg b/ffmap/web/static/img/router_direct_red.svg similarity index 100% rename from map/static/img/router_direct_red.svg rename to ffmap/web/static/img/router_direct_red.svg diff --git a/map/static/img/router_direct_yellow.svg b/ffmap/web/static/img/router_direct_yellow.svg similarity index 100% rename from map/static/img/router_direct_yellow.svg rename to ffmap/web/static/img/router_direct_yellow.svg diff --git a/map/static/img/router_green.svg b/ffmap/web/static/img/router_green.svg similarity index 100% rename from map/static/img/router_green.svg rename to ffmap/web/static/img/router_green.svg diff --git a/map/static/img/router_red.svg b/ffmap/web/static/img/router_red.svg similarity index 100% rename from map/static/img/router_red.svg rename to ffmap/web/static/img/router_red.svg diff --git a/map/static/img/router_yellow.svg b/ffmap/web/static/img/router_yellow.svg similarity index 100% rename from map/static/img/router_yellow.svg rename to ffmap/web/static/img/router_yellow.svg diff --git a/map/static/img/unknown.png b/ffmap/web/static/img/unknown.png similarity index 100% rename from map/static/img/unknown.png rename to ffmap/web/static/img/unknown.png diff --git a/map/static/js/map.js b/ffmap/web/static/js/map.js similarity index 94% rename from map/static/js/map.js rename to ffmap/web/static/js/map.js index da30593..ac7fbda 100644 --- a/map/static/js/map.js +++ b/ffmap/web/static/js/map.js @@ -52,8 +52,19 @@ map.on('click', function(pos) { if (px_distance <= router_pointer_radius) { console.log("Click on '"+router.hostname+"' detected."); console.log(router); - var has_neighbours = 'neighbours' in router && router.neighbours.length > 0; var popup_html = ""; + var has_neighbours = 'neighbours' in router && router.neighbours.length > 0; + + // avoid empty tables + if (has_neighbours) { + has_neighbours = false; + for (neighbour of router.neighbours) { + if ('_id' in neighbour) { + has_neighbours = true; + } + } + } + if (has_neighbours) { popup_html += "
"; } @@ -71,7 +82,7 @@ map.on('click', function(pos) { popup_html += "Outgoing Interface"; popup_html += ""; for (neighbour of router.neighbours) { - // skip unknown neighbours (FIXME: avoid empty table) + // skip unknown neighbours if ('_id' in neighbour) { var tr_color = "#04ff0a"; if (neighbour.quality < 105) { tr_color = "#ff1e1e"; } diff --git a/map/templates/bootstrap.html b/ffmap/web/templates/bootstrap.html similarity index 100% rename from map/templates/bootstrap.html rename to ffmap/web/templates/bootstrap.html diff --git a/map/templates/index.html b/ffmap/web/templates/index.html similarity index 100% rename from map/templates/index.html rename to ffmap/web/templates/index.html diff --git a/map/templates/map.html b/ffmap/web/templates/map.html similarity index 100% rename from map/templates/map.html rename to ffmap/web/templates/map.html diff --git a/map/templates/router.html b/ffmap/web/templates/router.html similarity index 100% rename from map/templates/router.html rename to ffmap/web/templates/router.html diff --git a/map/templates/router_list.html b/ffmap/web/templates/router_list.html similarity index 100% rename from map/templates/router_list.html rename to ffmap/web/templates/router_list.html diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..2fd5a2f --- /dev/null +++ b/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +mkdir -vp /var/lib/ffmap/csv +#FIXME: create dummy csv files +chown -R www-data:www-data /var/lib/ffmap + +mkdir -vp /usr/share/ffmap +cp -v mapnik/{hoods,links_and_routers}.xml /usr/share/ffmap +sed -i -e 's#>csv/#>/var/lib/ffmap/csv/#' /usr/share/ffmap/{hoods,links_and_routers}.xml +chown www-data:www-data /usr/share/ffmap/{hoods,links_and_routers}.xml +cp -v wsgi/{hoods,links_and_routers,web}.wsgi /usr/share/ffmap +cp -rv ffmap/web/static /usr/share/ffmap +cp -rv ffmap/web/templates /usr/share/ffmap + +mkdir -vp /var/cache/tiles/{hoods,links_and_routers} +chown -R www-data:www-data /var/cache/tiles/ + +cp -v systemd/*.service /etc/systemd/system/ +systemctl daemon-reload + +python3 setup.py install diff --git a/map/netmon.py b/map/netmon.py deleted file mode 100644 index b63b357..0000000 --- a/map/netmon.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/python - -import lxml.etree -import requests -import datetime -from pymongo import MongoClient - -client = MongoClient() -db = client.freifunk - -def fetch_router_info(mac): - mac = mac.replace(":", "").lower() - tree = lxml.etree.fromstring(requests.get("https://netmon.freifunk-franken.de/api/rest/router/%s" % mac, params={"limit": 5000}).content) - - for r in tree.xpath("/netmon_response/router"): - user_netmon_id = int(r.xpath("user_id/text()")[0]) - user = db.users.find_one({"netmon_id": user_netmon_id}) - if user: - user_id = user["_id"] - else: - user_id = db.users.insert({ - "netmon_id": user_netmon_id, - "nickname": r.xpath("user/nickname/text()")[0] - }) - user = db.users.find_one({"_id": user_id}) - - router = { - "netmon_id": int(r.xpath("router_id/text()")[0]), - "user": {"nickname": user["nickname"], "_id": user["_id"]} - } - - try: - lng = float(r.xpath("longitude/text()")[0]) - lat = float(r.xpath("latitude/text()")[0]) - assert lng != 0 - assert lat != 0 - - router["position"] = { - "type": "Point", - "coordinates": [lng, lat] - } - - # try to get comment - position_comment = r.xpath("location/text()")[0] - if position_comment != "undefined": - router["position"]["comment"] = position_comment - except (IndexError, AssertionError): - pass - - try: - router["description"] = r.xpath("description/text()")[0] - except IndexError: - pass - - router["last_contact"] = datetime.datetime.utcnow() - router["created"] = datetime.datetime.utcnow() - - return router diff --git a/map/mapnik/csv/.gitignore b/mapnik/csv/.gitignore similarity index 100% rename from map/mapnik/csv/.gitignore rename to mapnik/csv/.gitignore diff --git a/map/mapnik/hoods.xml b/mapnik/hoods.xml similarity index 90% rename from map/mapnik/hoods.xml rename to mapnik/hoods.xml index ac3a69d..180e754 100644 --- a/map/mapnik/hoods.xml +++ b/mapnik/hoods.xml @@ -1,6 +1,6 @@ - +