add crawler

router/crawl.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+
+import lxml.etree
+import requests
+import time
+import subprocess
+import gzip
+from pymongo import MongoClient
+client = MongoClient()
+
+db = client.freifunk
+routers = db.routers
+
+routervpnif = "fffVPN"
+
+for router in routers.find({"bootstrap_ip": {"$exists": True}, "status": {"$ne": "offline"}}):
+#for router in routers.find({"bootstrap_ip": {"$exists": True}, "status": "offline"}):
+	print(router["name"])
+	try:
+		node_data = subprocess.check_output(["curl", "-s", "--max-time", "5", "http://[%s%%wlan0]/node.data" % router["bootstrap_ip"]])
+		try:
+			node_data = gzip.decompress(node_data)
+		except:
+			pass
+
+		assert "<TITLE>404" not in str(node_data).upper()
+
+		tree = lxml.etree.fromstring(node_data)
+		print(" --> " + tree.xpath("/data/system_data/hostname/text()")[0])
+
+		router_update = {
+			"status": "online",
+			"has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0,
+			"hostname": tree.xpath("/data/system_data/hostname/text()")[0],
+			"neighbours": [], # list of mongoDB ids (or mac if no corresponding id found)
+			"netifs": [],
+		}
+
+		for netif in tree.xpath("/data/interface_data/*"):
+			interface = {
+				"name": netif.xpath("name/text()")[0],
+				"mtu": int(netif.xpath("mtu/text()")[0]),
+				"mac": netif.xpath("mac_addr/text()")[0].lower(),
+			}
+			if len(netif.xpath("ipv6_link_local_addr/text()")) > 0:
+				interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0]
+			if len(netif.xpath("ipv4_addr/text()")) > 0:
+				interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0]
+			router_update["netifs"].append(interface)
+
+
+		for originator in tree.xpath("/data/batman_adv_originators/*"):
+			o_mac = originator.xpath("originator/text()")[0]
+			o_nexthop = originator.xpath("nexthop/text()")[0]
+			# mac is the mac of the neighbour w2/5mesh if
+			# (which might also be called wlan0-1)
+			o_link_quality = originator.xpath("link_quality/text()")[0]
+			o_out_if = originator.xpath("outgoing_interface/text()")[0]
+			if o_mac.upper() == o_nexthop.upper():
+				# skip vpn server
+				if o_out_if == routervpnif:
+					continue
+				neighbour = {
+					"mac": o_mac.lower(),
+					"quality": int(o_link_quality),
+					"net_if": o_out_if,
+				}
+				try:
+					neighbour["_id"] = routers.find_one({"netifs.mac": neighbour["mac"]})["_id"]
+				except:
+					pass
+				router_update["neighbours"].append(neighbour)
+
+		routers.update_one({"_id": router["_id"]}, {"$set": router_update})
+
+		#from pprint import pprint
+		#pprint(router)
+	except subprocess.CalledProcessError:
+		#routers.update_one({"_id": router["_id"]}, {"$set": {"status": "offline"}})
+		print(" --> OFFLINE (ignored)")
+	except AssertionError:
+		routers.update_one({"_id": router["_id"]}, {"$set": {"status": "error"}})
+		print(" --> ERROR")
+	#import sys; sys.exit(1)

router/mkcsv.py

@@ -16,6 +16,6 @@ csv.write("x,y,status\n")
 
 #for router in routers.find({"bootstrap_ip": {"$exists": True}}):
 for router in routers.find({}):
-	csv.write("%f,%f,%s\n" % (router["position"]["lng"], router["position"]["lat"], "online"))
+	csv.write("%f,%f,%s\n" % (router["position"]["lng"], router["position"]["lat"], router.get("status", "unknown")))
 
 csv.close()

router/parse_crawl_data.py

@@ -2,6 +2,11 @@
 
 import lxml.etree
 import sys
+from pymongo import MongoClient
+client = MongoClient()
+
+db = client.freifunk
+routers = db.routers
 
 routervpnif = "fffVPN"
 
@@ -16,18 +21,20 @@ router = {
 	"has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0,
 	"hostname": tree.xpath("/data/system_data/hostname/text()")[0],
 	"neighbours": [], # list of mongoDB ids (or mac if no corresponding id found)
-	"netifs": {},
+	"netifs": [],
 }
 
 for netif in tree.xpath("/data/interface_data/*"):
-	router["netifs"][netif.xpath("name/text()")[0]] = {
+	interface = {
+		"name": netif.xpath("name/text()")[0],
 		"mtu": int(netif.xpath("mtu/text()")[0]),
 		"mac": netif.xpath("mac_addr/text()")[0].lower(),
 	}
 	if len(netif.xpath("ipv6_link_local_addr/text()")) > 0:
-		router["netifs"][netif.xpath("name/text()")[0]]["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0]
+		interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0]
 	if len(netif.xpath("ipv4_addr/text()")) > 0:
-		router["netifs"][netif.xpath("name/text()")[0]]["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0]
+		interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0]
+	router["netifs"].append(interface)
 
 
 for originator in tree.xpath("/data/batman_adv_originators/*"):
@@ -45,10 +52,14 @@ for originator in tree.xpath("/data/batman_adv_originators/*"):
 			"mac": o_mac.lower(),
 			"quality": int(o_link_quality),
 			"net_if": o_out_if,
-			#FIXME "_id": mongoDB router id
 		}
+		try:
+			neighbour["_id"] = routers.find_one({"netifs.mac": neighbour["mac"]})["_id"]
+		except:
+			pass
 		router["neighbours"].append(neighbour)
 
-from pprint import pprint
+routers.update_one({"name": router["hostname"]}, {"$set": router})
 
+from pprint import pprint
 pprint(router)

router/routers.xml

@@ -4,9 +4,17 @@
   <!--<Style name="routerpoint" filter-mode="first">-->
   <Style name="routerpoint">
 	<Rule>
-	  <!--<Filter>([color] = '20ff0000')</Filter>-->
+	  <Filter>([status] = 'online')</Filter>
 	  <PointSymbolizer file="green-dot.png" allow-overlap="true" />
 	</Rule>
+	<Rule>
+	  <Filter>([status] = 'offline')</Filter>
+	  <PointSymbolizer file="red-dot.png" allow-overlap="true" />
+	</Rule>
+	<Rule>
+	  <Filter>([status] = 'unknown')</Filter>
+	  <PointSymbolizer file="yellow-dot.png" allow-overlap="true" />
+	</Rule>
   </Style>
   
   <Layer name="layer">