From 777c211ccd7dead0c87b945530d953b4cd172c5a Mon Sep 17 00:00:00 2001 From: Pat Thoyts Date: Mon, 9 May 2016 11:52:50 +0100 Subject: [PATCH] Added update to sensordata script to add new values to the database. Fix the official mongodb collection name as well. --- sensor-hub.wsgi | 10 +++++----- sensordata.py | 33 +++++++++++++++++++++++++++++++-- static/sensor-hub.html | 2 +- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/sensor-hub.wsgi b/sensor-hub.wsgi index 67981e4..9912063 100755 --- a/sensor-hub.wsgi +++ b/sensor-hub.wsgi @@ -50,7 +50,7 @@ class SensorHubService(): try: uri = cherrypy.request.app.config['database']['uri'] client = MongoClient(uri) - db = client.test.sensorlog + db = client.sensorhub.sensorlog cursor = db.find().sort([("timestamp", pymongo.DESCENDING)]).limit(1) item = dict(next(cursor)) del item['_id'] @@ -86,7 +86,7 @@ class SensorHubService(): def get_since(self, when, until = None): uri = cherrypy.request.app.config['database']['uri'] client = MongoClient(uri) - db = client.test.sensorlog + db = client.sensorhub.sensorlog selector = [{"timestamp": {"$gt": str(when)}}] if not until is None: selector.append({"timestamp": {"$lt": str(until)}}) @@ -98,7 +98,7 @@ class SensorHubService(): def hosts(self): uri = cherrypy.request.app.config['database']['uri'] client = MongoClient(uri) - db = client.test.sensorlog + db = client.sensorhub.sensorlog # groupby("name") result = [x['_id'] for x in db.aggregate([{"$group": {"_id": "$name"}} ])] return dict(response=result, version=self.version) @@ -108,7 +108,7 @@ class SensorHubService(): def sensors(self, hostname): uri = cherrypy.request.app.config['database']['uri'] client = MongoClient(uri) - db = client.test.sensorlog + db = client.sensorhub.sensorlog groups = [x['_id'] for x in db.aggregate([ {"$match": {"name": hostname}}, {"$group": {"_id": "$sensors.id"}} @@ -134,7 +134,7 @@ class SensorHubService(): return json.dumps(res).encode(encoding="utf-8") except Exception as e: cherrypy.response.headers['content-type'] = 'application/json' - self.log("error in \"since\": {0}\n {1}".format(str(e), repr(param))) + self.log("error in \"since\": {0}\n {1}".format(str(e), repr(cherrypy.request.params))) res = dict(response='error', message=str(e)) res = json.dumps(res).encode(encoding='utf-8') return res diff --git a/sensordata.py b/sensordata.py index 7b81cc9..813bf5a 100644 --- a/sensordata.py +++ b/sensordata.py @@ -15,6 +15,13 @@ __author__ = 'Pat Thoyts ' __copyright__ = 'Copyright (c) 2016 Pat Thoyts' class SensorDataIterator(): + """Parse a sensor hub ASCII data file into discrete records. + + The sensor hub emits ASCII data with one record per line but with some initial + lines prefixed with a # character as a comment and the initial line may get + truncated due to serial buffering. + The results may have from 1 to 3 temperature values depending on how many sensors + have been picked up.""" def __init__(self, filename): self.filename = filename self.re = re.compile(r': {(?P.*?)} {(?P.*?)} {(?P

.*?)}$') @@ -96,13 +103,33 @@ def import_logfile(filename, granularity = None): granularity = 60 # 1 per minute granularity = int(granularity) mongo = MongoClient() - db = mongo.test.sensorlog + db = mongo.sensorhub.sensorlog db.drop() r = db.insert_many([item for item in GranularData(iter(SensorData(filename)), granularity)]) print("imported {0} records".format(len(r.inserted_ids))) return 0 +def update_from_logfile(filename, granularity = None): + """Import new records from the sensor-hub logfile into the mongo database.""" + if granularity is None: + granularity = 60 # 1 per minute + granularity = int(granularity) + mongo = MongoClient() + db = mongo.sensorhub.sensorlog + # find the last entry in the database + cursor = db.find().sort([("timestamp", pymongo.DESCENDING)]).limit(1) + last = dict(next(cursor)) + last_timestamp = int(last['timestamp']) + granularity + # skip logdata up to the last database entry + # add the remaining entries. + r = db.insert_many([x for x in GranularData(iter(SensorData(filename)), granularity) + if int(x['timestamp']) > last_timestamp]) + print("imported {0} records".format(len(r.inserted_ids))) + return 0 + def tojson(filename, granularity = None): + if granularity is None: + granularity = 60 # 1 per minute for item in GranularData(iter(SensorData(filename)), granularity): print(item) return 0 @@ -117,7 +144,7 @@ def testing(*args, **kwargs): ''' uri = 'mongodb://localhost:27017/test' client = MongoClient(uri) - db = client.test.sensorlog + db = client.sensorhub.sensorlog #db.insert_one(json) / insert_many (iterable) #cursor = db.find({"name": "spd-office"}) #cursor = db.find({"sensors.id": "office1", "sensors.value": "22.25"}) @@ -140,6 +167,8 @@ if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == 'import': import_logfile(*sys.argv[2:]) + elif sys.argv[1] == 'update': + update_from_logfile(*sys.argv[2:]) elif sys.argv[1] == 'json': tojson(*sys.argv[2:]) elif sys.argv[1] == 'test': diff --git a/static/sensor-hub.html b/static/sensor-hub.html index 21bb52f..6174906 100644 --- a/static/sensor-hub.html +++ b/static/sensor-hub.html @@ -4,7 +4,7 @@ -Lab Temperature Monitor +Office Temperature Monitor