diff --git a/db/crawl.py b/db/crawl.py index e0c7e80..cc88b77 100755 --- a/db/crawl.py +++ b/db/crawl.py @@ -43,12 +43,25 @@ def crawl(router): router_update = { "status": tree.xpath("/data/system_data/status/text()")[0], - "has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0, "hostname": tree.xpath("/data/system_data/hostname/text()")[0], - "systemtime": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])), - "uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])), "neighbours": [], "netifs": [], + "system": { + "time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])), + "uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])), + "memory": { + "free": int(tree.xpath("/data/system_data/memory_free/text()")[0]), + "buffering": int(tree.xpath("/data/system_data/memory_buffering/text()")[0]), + "caching": int(tree.xpath("/data/system_data/memory_caching/text()")[0]), + }, + "loadavg": float(tree.xpath("/data/system_data/loadavg/text()")[0]), + "processes": { + "runnable": int(tree.xpath("/data/system_data/processes/text()")[0].split("/")[0]), + "total": int(tree.xpath("/data/system_data/processes/text()")[0].split("/")[1]), + }, + "clients": int(tree.xpath("/data/client_count/text()")[0]), + "has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0, + }, "hardware": { "chipset": tree.xpath("/data/system_data/chipset/text()")[0], "cpu": tree.xpath("/data/system_data/cpu/text()")[0] @@ -77,6 +90,10 @@ def crawl(router): "name": netif.xpath("name/text()")[0], "mtu": int(netif.xpath("mtu/text()")[0]), "mac": netif.xpath("mac_addr/text()")[0].lower(), + "traffic": { + "rx": int(netif.xpath("traffic_rx/text()")[0]), + "tx": int(netif.xpath("traffic_tx/text()")[0]), + }, } if len(netif.xpath("ipv6_link_local_addr/text()")) > 0: interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0] @@ -84,7 +101,10 @@ def crawl(router): interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0] router_update["netifs"].append(interface) + visible_neighbours = 0 + for originator in tree.xpath("/data/batman_adv_originators/*"): + visible_neighbours += 1 o_mac = originator.xpath("originator/text()")[0] o_nexthop = originator.xpath("nexthop/text()")[0] # mac is the mac of the neighbour w2/5mesh if @@ -114,16 +134,47 @@ def crawl(router): pass router_update["neighbours"].append(neighbour) - db.routers.update_one({"_id": router["_id"]}, {"$set": router_update, "$currentDate": {"last_contact": True}}) + router_update["system"]["visible_neighbours"] = visible_neighbours - #from pprint import pprint - #pprint(router) + db.routers.update_one({"_id": router["_id"]}, {"$set": router_update, "$currentDate": {"last_contact": True}}) + status = router_update["status"] except subprocess.CalledProcessError: + # in a non-crawling setup the system would need to + # mark routers as offline when the last_contact is too far in the past + # eg by a cronjob db.routers.update_one({"_id": router["_id"]}, {"$set": {"status": "offline"}}) + status = "offline" print(" --> OFFLINE") except (AssertionError, lxml.etree.XMLSyntaxError): db.routers.update_one({"_id": router["_id"]}, {"$set": {"status": "unknown"}}) + status = "unknown" print(" --> UNKNOWN") + finally: + # fire events + events = [] + try: + if router["system"]["uptime"] > router_update["system"]["uptime"]: + events.append({ + "time": datetime.datetime.utcnow(), + "type": "reboot", + }) + except: + pass + if router["status"] != status: + events.append({ + "time": datetime.datetime.utcnow(), + "type": status, + }) + db.routers.update_one({"_id": router["_id"]}, {"$push": {"events": { + "$each": events, + "$slice": -10, + }}}) + + if status == "online": + # calculate RRD statistics + #FIXME: implementation + pass + q = Queue() keep_working = True diff --git a/db/import_from_netmon.py b/db/import_from_netmon.py index 858d72b..398b503 100755 --- a/db/import_from_netmon.py +++ b/db/import_from_netmon.py @@ -31,11 +31,12 @@ for r in tree.xpath("/netmon_response/routerlist/router"): "netmon_id": user_netmon_id, "nickname": r.xpath("user/nickname/text()")[0] }) + user = db.users.find_one({"_id": user_id}) router = { "netmon_id": int(r.xpath("router_id/text()")[0]), "hostname": r.xpath("hostname/text()")[0], - "user": user_id + "user": {"nickname": user["nickname"], "_id": user["_id"]} } try: @@ -79,5 +80,6 @@ for r in tree.xpath("/netmon_response/routerlist/router"): }] router["created"] = datetime.datetime.utcnow() + router["status"] = "unknown" db.routers.insert_one(router)