freifunk-franken
/
monitoring
mirror of https://github.com/FreifunkFranken/fff-monitoring.git
3
0
Fork 0
Code Issues Releases Wiki Activity

crawl more data

This commit is contained in:
Dominik Heidler 2015-10-02 23:52:10 +02:00
parent f0cc23ae35
commit f8cd3a38cc
2 changed files with 60 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
db/crawl.py
View File

@ -43,12 +43,25 @@ def crawl(router):
router_update = {
"status": tree.xpath("/data/system_data/status/text()")[0],
"has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0,
"hostname": tree.xpath("/data/system_data/hostname/text()")[0],
"systemtime": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])),
"uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])),
"neighbours": [],
"netifs": [],
"system": {
"time": datetime.datetime.fromtimestamp(int(tree.xpath("/data/system_data/local_time/text()")[0])),
"uptime": int(float(tree.xpath("/data/system_data/uptime/text()")[0])),
"memory": {
"free": int(tree.xpath("/data/system_data/memory_free/text()")[0]),
"buffering": int(tree.xpath("/data/system_data/memory_buffering/text()")[0]),
"caching": int(tree.xpath("/data/system_data/memory_caching/text()")[0]),
},
"loadavg": float(tree.xpath("/data/system_data/loadavg/text()")[0]),
"processes": {
"runnable": int(tree.xpath("/data/system_data/processes/text()")[0].split("/")[0]),
"total": int(tree.xpath("/data/system_data/processes/text()")[0].split("/")[1]),
},
"clients": int(tree.xpath("/data/client_count/text()")[0]),
"has_wan_uplink": len(tree.xpath("/data/interface_data/fffVPN")) > 0,
},
"hardware": {
"chipset": tree.xpath("/data/system_data/chipset/text()")[0],
"cpu": tree.xpath("/data/system_data/cpu/text()")[0]
@ -77,6 +90,10 @@ def crawl(router):
"name": netif.xpath("name/text()")[0],
"mtu": int(netif.xpath("mtu/text()")[0]),
"mac": netif.xpath("mac_addr/text()")[0].lower(),
"traffic": {
"rx": int(netif.xpath("traffic_rx/text()")[0]),
"tx": int(netif.xpath("traffic_tx/text()")[0]),
},
}
if len(netif.xpath("ipv6_link_local_addr/text()")) > 0:
interface["ipv6_fe80_addr"] = netif.xpath("ipv6_link_local_addr/text()")[0].lower().split("/")[0]
@ -84,7 +101,10 @@ def crawl(router):
interface["ipv4_addr"] = netif.xpath("ipv4_addr/text()")[0]
router_update["netifs"].append(interface)
visible_neighbours = 0
for originator in tree.xpath("/data/batman_adv_originators/*"):
visible_neighbours += 1
o_mac = originator.xpath("originator/text()")[0]
o_nexthop = originator.xpath("nexthop/text()")[0]
# mac is the mac of the neighbour w2/5mesh if
@ -114,16 +134,47 @@ def crawl(router):
pass
router_update["neighbours"].append(neighbour)
db.routers.update_one({"_id": router["_id"]}, {"$set": router_update, "$currentDate": {"last_contact": True}})
router_update["system"]["visible_neighbours"] = visible_neighbours
#from pprint import pprint
#pprint(router)
db.routers.update_one({"_id": router["_id"]}, {"$set": router_update, "$currentDate": {"last_contact": True}})
status = router_update["status"]
except subprocess.CalledProcessError:
# in a non-crawling setup the system would need to
# mark routers as offline when the last_contact is too far in the past
# eg by a cronjob
db.routers.update_one({"_id": router["_id"]}, {"$set": {"status": "offline"}})
status = "offline"
print(" --> OFFLINE")
except (AssertionError, lxml.etree.XMLSyntaxError):
db.routers.update_one({"_id": router["_id"]}, {"$set": {"status": "unknown"}})
status = "unknown"
print(" --> UNKNOWN")
finally:
# fire events
events = []
try:
if router["system"]["uptime"] > router_update["system"]["uptime"]:
events.append({
"time": datetime.datetime.utcnow(),
"type": "reboot",
})
except:
pass
if router["status"] != status:
events.append({
"time": datetime.datetime.utcnow(),
"type": status,
})
db.routers.update_one({"_id": router["_id"]}, {"$push": {"events": {
"$each": events,
"$slice": -10,
}}})
if status == "online":
# calculate RRD statistics
#FIXME: implementation
pass
q = Queue()
keep_working = True

4
db/import_from_netmon.py
View File

@ -31,11 +31,12 @@ for r in tree.xpath("/netmon_response/routerlist/router"):
"netmon_id": user_netmon_id,
"nickname": r.xpath("user/nickname/text()")[0]
})
user = db.users.find_one({"_id": user_id})
router = {
"netmon_id": int(r.xpath("router_id/text()")[0]),
"hostname": r.xpath("hostname/text()")[0],
"user": user_id
"user": {"nickname": user["nickname"], "_id": user["_id"]}
}
try:
@ -79,5 +80,6 @@ for r in tree.xpath("/netmon_response/routerlist/router"):
}]
router["created"] = datetime.datetime.utcnow()
router["status"] = "unknown"
db.routers.insert_one(router)