From: Pat Thoyts Date: Thu, 21 Apr 2016 09:54:36 +0000 (+0100) Subject: Initial version working from a reduced text log file. X-Git-Url: http://privyetmir.co.uk/gitweb?a=commitdiff_plain;h=87c12dbcdc53dad502aa083cc53d6d0d85533321;p=spd%2Fsensor-hub.git Initial version working from a reduced text log file. The default log has values for every second, reduced to every 10 minutes. The lab-monitor data is 1 each minute. Signed-off-by: Pat Thoyts --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..17a6511 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/__pycache__/ +/data/ +/sensor-hub.config + diff --git a/importlog.py b/importlog.py new file mode 100755 index 0000000..7d837c6 --- /dev/null +++ b/importlog.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 + +"""Import sensorlog data to JSON for import into mongodb + +mongoimport --db test --collection sensorlog --drop --file JSONDATAFILE +""" + +from __future__ import print_function, absolute_import, division +import sys, json, pymongo +from sensordata import SensorData +from pymongo import MongoClient + +def usage(): + print("usage: importlog import filename\n demo", file=sys.stderr) + +class GranularData(): + def __init__(self, iterable, granularity): + self.iterable = iterable + self.granularity = granularity + def __iter__(self): + self.last = 0 + return self; + def __next__(self): + while True: + item = next(self.iterable) + t = int(int(item['timestamp']) / self.granularity) * self.granularity + if t != self.last: + self.last = t + return item + +def import_logfile(filename, granularity = None): + """Import a sensor-hub logfile into mongodb""" + if granularity is None: + granularity = 600 # 10 minutes + granularity = int(granularity) + last = 0 + mongo = MongoClient() + db = mongo.test.sensorlog + db.drop() + db.insert_many([item for item in GranularData(iter(SensorData(filename)), granularity)]) + return 0 + +def tojson(filename): + last = 0 + for item in SensorData(filename): + t = int(int(item['timestamp']) / 600) * 600 + if t != last: + print(json.dumps(item)) + last = t + return 0 + +def demo(): + ''' + {"timestamp": "1460971800", + "name": "spd-office", + "sensors": [{"id": "office1", "value": "22.25"}, + {"id": "office2", "value": "22.69"}, + {"id": "office3", "value": "22.37"}]} + ''' + uri = 'mongodb://localhost:27017/test' + client = MongoClient(uri) + db = client.test.sensorlog + #db.insert_one(json) / insert_many (iterable) + #cursor = db.find({"name": "spd-office"}) + #cursor = db.find({"sensors.id": "office1", "sensors.value": "22.25"}) + cursor = db.find({ + "$and": [ + { "timestamp": {"$gt": "1460968800"} }, + { "timestamp": {"$lt": "1460970100"} } + ] + }).sort([("timestamp", pymongo.ASCENDING)]) + for item in cursor: + print(item) + return 0 + +def main(args = None): + if args is None: + args = sys.argv + if len(args) < 2: + usage() + return 1 + if args[1] == "convert": + return tojson(args[2]) + if args[1] == "import": + return import_logfile(args[2], args[3]) + if args[1] == "demo": + return demo() + usage() + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/reduce.sh b/reduce.sh new file mode 100644 index 0000000..18e9186 --- /dev/null +++ b/reduce.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail +awk ' +BEGIN { last=0; } +/^14/ { + t = (int($1 / 600) * 600); + if (t != last) { + last = t; + print $0 + } +}' $1 > $2 + diff --git a/sensor-hub.config.sample b/sensor-hub.config.sample new file mode 100644 index 0000000..4e4b370 --- /dev/null +++ b/sensor-hub.config.sample @@ -0,0 +1,6 @@ +[database] +url = 'mysql+pymysql://USERNAME:PASSWORD@localhost/DATABASENAME' + +[logging] +error_log = 'error.log' + diff --git a/sensor-hub.wsgi b/sensor-hub.wsgi new file mode 100755 index 0000000..9d3dca5 --- /dev/null +++ b/sensor-hub.wsgi @@ -0,0 +1,104 @@ +#!/usr/bin/python3 +# +# Present data from the Arduino sensor-hub over a web API. + +from __future__ import print_function, division, absolute_import +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__))) + +import cherrypy, json +from cherrypy import config, tools +from time import localtime, time +from datetime import datetime +from dateutil import parser as dateparser +from statistics import median +from urllib3.util import parse_url +from sensordata import SensorData + +class SensorHubService(): + + def __init__(self): + self.version = '1.0' + self.errlog = None + + def log(self, msg): + if self.errlog is None and 'logging' in cherrypy.request.app.config: + self.errlog = os.path.join(os.path.dirname(__file__), + 'data', + cherrypy.request.app.config['logging']['error_log']) + if not self.errlog is None: + with open(self.errlog, 'a') as f: + print(msg, file=f) + + @cherrypy.expose + def recent(self, *args, **kwargs): + return json.dumps(dict(response='error', message='not implemented')) + + @cherrypy.expose + def since(self, *args, **kwargs): + """Get data since any timepoint. + eg: /lab-monitor/since?when=2016-03-01T00:00:00 + /lab-monitor/since?when=14 days ago + """ + try: + cherrypy.response.headers['content-type'] = 'application/json' + param = cherrypy.request.params.get('when') + if not param is None: + when = dateparser.parse(param) + when = when.timestamp() + else: + when = int(time() - (86400 * 7)) # 7 days ago + res = json.dumps(self.get_since(int(when))).encode(encoding='utf-8') + except Exception as e: + cherrypy.response.headers['content-type'] = 'application/json' + self.log("error in \"since\": {0}".format(str(e))) + res = dict(response='error', message=str(e)) + res = json.dumps(res).encode(encoding='utf-8') + return res + + def get_since(self, when): + url = cherrypy.request.app.config['database']['url'] + path = parse_url(url).path + return dict(result=[x for x in SensorData(path) if int(x['timestamp']) >= when]) + +def application(environ, start_response): + staticdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static')) + logdir = os.path.join(os.path.dirname(__file__), 'data') + conf = { + '/': { + 'tools.staticdir.root': staticdir + }, + '/sensor-hub.html': { + 'tools.staticfile.on': True, + 'tools.gzip.on': True, + 'tools.staticfile.filename': os.path.join(staticdir, 'sensor-hub.html') + }, + '/loading.gif': { + 'tools.staticfile.on': True, + 'tools.gzip.on': False, + 'tools.staticfile.filename': os.path.join(staticdir, 'ajax-loader.gif') + } + } + app = cherrypy.tree.mount(SensorHubService(), script_name='/sensor-hub', config=conf) + app.merge(os.path.join(os.path.dirname(__file__), 'sensor-hub.config')) + return cherrypy.tree(environ, start_response) + +if __name__ == '__main__': + from cherrypy import wsgiserver, config + config.update({'environment': 'embedded', + 'request.throw_errors': True, + 'tools.gzip.on': True, + 'tools.gzip.mime_types':['text/*','text/html', 'application/json']}) + try: + srv = (r'0.0.0.0', 9876) + server = wsgiserver.CherryPyWSGIServer(srv, application) + print("starting server on port %d" % srv[1]) + server.start() + except Exception as e: + print(repr(e)) + except KeyboardInterrupt: + server.stop() + +# Local variables: +# mode: python +# End: diff --git a/sensordata.py b/sensordata.py new file mode 100644 index 0000000..41db2ee --- /dev/null +++ b/sensordata.py @@ -0,0 +1,91 @@ +#!/usr/bin/python3 + +"""Data manager for sensor hub. + +Reads data from the ASCII log file and presents a collection +""" + +__all__ = ['SensorData'] +__version__ = '1.0.0' +__author__ = 'Pat Thoyts ' +__copyright__ = 'Copyright (c) 2016 Pat Thoyts' + +import re + +class SensorDataIterator(): + def __init__(self, filename): + self.filename = filename + self.re = re.compile(r': {(?P.*?)} {(?P.*?)} {(?P

.*?)}$') + def __iter__(self): + self.fp = open(self.filename, 'rt') + return self + def __next__(self): + while True: + line = self.fp.readline() + if not line: + self.fp.close() + raise StopIteration() + if line.startswith('#') or line.startswith('!'): + continue + timestamp = line.split()[0] + m = self.re.search(line) + if m: + sensors = [] + for sensor,value in zip(['office1','office2','office3'], m.group('T').split()): + sensors.append({'id': sensor, 'value': value }) + return dict(timestamp=timestamp, name='spd-office', sensors=sensors) + +class SensorData(): + def __init__(self, filename): + self.filename = filename + + def __iter__(self): + return iter(SensorDataIterator(self.filename)) + +import unittest + +class TestSensorData(unittest.TestCase): + datafile = '_test_data.log' + data = '''# comment +! junk +1460641524 23.19 32.25 1003.39 : {23.0 23.37 23.19} {32.25} {1003.39} +1460641594 23.19 32.50 1003.43 : {23.06 23.37 23.19} {32.5} {1003.43} +1460641608 23.19 32.50 1003.42 : {23.06 23.37 23.19} {32.5} {1003.42} +''' + @classmethod + def setUpClass(clazz): + with open(clazz.datafile,'wt') as f: + f.write(clazz.data) + + @classmethod + def tearDownClass(clazz): + import os + os.remove(clazz.datafile) + + def test_parse(self): + data = SensorData(self.datafile) + count = 0 + for item in data: + check = [] + for sensor in item['sensors']: + check.append(sensor['id']) + self.assertListEqual(['office1','office2','office3'], check) + count = count + 1 + self.assertEqual(3, count) + +if __name__ == '__main__': + import sys + if len(sys.argv) > 2: + if sys.argv[1] == 'test': + # tester().test(argv[2]) etc + print("not implemented yet", file = sys.stderr) + 1 + else: + print("usage: SensorData test") + 1 + else: + unittest.main() + +# Local variables: +# mode: python +# End: diff --git a/static/ajax-loader.gif b/static/ajax-loader.gif new file mode 100644 index 0000000..d2378cd Binary files /dev/null and b/static/ajax-loader.gif differ diff --git a/static/sensor-hub.html b/static/sensor-hub.html new file mode 100644 index 0000000..580d108 --- /dev/null +++ b/static/sensor-hub.html @@ -0,0 +1,203 @@ + + + + + + +Lab Temperature Monitor + + + + +

+
+
+
+

Download view data as ASCII or JSON

+

+

+
+ + + + + + + + +