From 87c12dbcdc53dad502aa083cc53d6d0d85533321 Mon Sep 17 00:00:00 2001 From: Pat Thoyts Date: Thu, 21 Apr 2016 10:54:36 +0100 Subject: [PATCH] Initial version working from a reduced text log file. The default log has values for every second, reduced to every 10 minutes. The lab-monitor data is 1 each minute. Signed-off-by: Pat Thoyts --- .gitignore | 4 + importlog.py | 92 ++++++++++++++++++ reduce.sh | 12 +++ sensor-hub.config.sample | 6 ++ sensor-hub.wsgi | 104 ++++++++++++++++++++ sensordata.py | 91 ++++++++++++++++++ static/ajax-loader.gif | Bin 0 -> 9427 bytes static/sensor-hub.html | 203 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 512 insertions(+) create mode 100644 .gitignore create mode 100755 importlog.py create mode 100644 reduce.sh create mode 100644 sensor-hub.config.sample create mode 100755 sensor-hub.wsgi create mode 100644 sensordata.py create mode 100644 static/ajax-loader.gif create mode 100644 static/sensor-hub.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..17a6511 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/__pycache__/ +/data/ +/sensor-hub.config + diff --git a/importlog.py b/importlog.py new file mode 100755 index 0000000..7d837c6 --- /dev/null +++ b/importlog.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 + +"""Import sensorlog data to JSON for import into mongodb + +mongoimport --db test --collection sensorlog --drop --file JSONDATAFILE +""" + +from __future__ import print_function, absolute_import, division +import sys, json, pymongo +from sensordata import SensorData +from pymongo import MongoClient + +def usage(): + print("usage: importlog import filename\n demo", file=sys.stderr) + +class GranularData(): + def __init__(self, iterable, granularity): + self.iterable = iterable + self.granularity = granularity + def __iter__(self): + self.last = 0 + return self; + def __next__(self): + while True: + item = next(self.iterable) + t = int(int(item['timestamp']) / self.granularity) * self.granularity + if t != self.last: + self.last = t + return item + +def import_logfile(filename, granularity = None): + """Import a sensor-hub logfile into mongodb""" + if granularity is None: + granularity = 600 # 10 minutes + granularity = int(granularity) + last = 0 + mongo = MongoClient() + db = mongo.test.sensorlog + db.drop() + db.insert_many([item for item in GranularData(iter(SensorData(filename)), granularity)]) + return 0 + +def tojson(filename): + last = 0 + for item in SensorData(filename): + t = int(int(item['timestamp']) / 600) * 600 + if t != last: + print(json.dumps(item)) + last = t + return 0 + +def demo(): + ''' + {"timestamp": "1460971800", + "name": "spd-office", + "sensors": [{"id": "office1", "value": "22.25"}, + {"id": "office2", "value": "22.69"}, + {"id": "office3", "value": "22.37"}]} + ''' + uri = 'mongodb://localhost:27017/test' + client = MongoClient(uri) + db = client.test.sensorlog + #db.insert_one(json) / insert_many (iterable) + #cursor = db.find({"name": "spd-office"}) + #cursor = db.find({"sensors.id": "office1", "sensors.value": "22.25"}) + cursor = db.find({ + "$and": [ + { "timestamp": {"$gt": "1460968800"} }, + { "timestamp": {"$lt": "1460970100"} } + ] + }).sort([("timestamp", pymongo.ASCENDING)]) + for item in cursor: + print(item) + return 0 + +def main(args = None): + if args is None: + args = sys.argv + if len(args) < 2: + usage() + return 1 + if args[1] == "convert": + return tojson(args[2]) + if args[1] == "import": + return import_logfile(args[2], args[3]) + if args[1] == "demo": + return demo() + usage() + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/reduce.sh b/reduce.sh new file mode 100644 index 0000000..18e9186 --- /dev/null +++ b/reduce.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail +awk ' +BEGIN { last=0; } +/^14/ { + t = (int($1 / 600) * 600); + if (t != last) { + last = t; + print $0 + } +}' $1 > $2 + diff --git a/sensor-hub.config.sample b/sensor-hub.config.sample new file mode 100644 index 0000000..4e4b370 --- /dev/null +++ b/sensor-hub.config.sample @@ -0,0 +1,6 @@ +[database] +url = 'mysql+pymysql://USERNAME:PASSWORD@localhost/DATABASENAME' + +[logging] +error_log = 'error.log' + diff --git a/sensor-hub.wsgi b/sensor-hub.wsgi new file mode 100755 index 0000000..9d3dca5 --- /dev/null +++ b/sensor-hub.wsgi @@ -0,0 +1,104 @@ +#!/usr/bin/python3 +# +# Present data from the Arduino sensor-hub over a web API. + +from __future__ import print_function, division, absolute_import +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__))) + +import cherrypy, json +from cherrypy import config, tools +from time import localtime, time +from datetime import datetime +from dateutil import parser as dateparser +from statistics import median +from urllib3.util import parse_url +from sensordata import SensorData + +class SensorHubService(): + + def __init__(self): + self.version = '1.0' + self.errlog = None + + def log(self, msg): + if self.errlog is None and 'logging' in cherrypy.request.app.config: + self.errlog = os.path.join(os.path.dirname(__file__), + 'data', + cherrypy.request.app.config['logging']['error_log']) + if not self.errlog is None: + with open(self.errlog, 'a') as f: + print(msg, file=f) + + @cherrypy.expose + def recent(self, *args, **kwargs): + return json.dumps(dict(response='error', message='not implemented')) + + @cherrypy.expose + def since(self, *args, **kwargs): + """Get data since any timepoint. + eg: /lab-monitor/since?when=2016-03-01T00:00:00 + /lab-monitor/since?when=14 days ago + """ + try: + cherrypy.response.headers['content-type'] = 'application/json' + param = cherrypy.request.params.get('when') + if not param is None: + when = dateparser.parse(param) + when = when.timestamp() + else: + when = int(time() - (86400 * 7)) # 7 days ago + res = json.dumps(self.get_since(int(when))).encode(encoding='utf-8') + except Exception as e: + cherrypy.response.headers['content-type'] = 'application/json' + self.log("error in \"since\": {0}".format(str(e))) + res = dict(response='error', message=str(e)) + res = json.dumps(res).encode(encoding='utf-8') + return res + + def get_since(self, when): + url = cherrypy.request.app.config['database']['url'] + path = parse_url(url).path + return dict(result=[x for x in SensorData(path) if int(x['timestamp']) >= when]) + +def application(environ, start_response): + staticdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static')) + logdir = os.path.join(os.path.dirname(__file__), 'data') + conf = { + '/': { + 'tools.staticdir.root': staticdir + }, + '/sensor-hub.html': { + 'tools.staticfile.on': True, + 'tools.gzip.on': True, + 'tools.staticfile.filename': os.path.join(staticdir, 'sensor-hub.html') + }, + '/loading.gif': { + 'tools.staticfile.on': True, + 'tools.gzip.on': False, + 'tools.staticfile.filename': os.path.join(staticdir, 'ajax-loader.gif') + } + } + app = cherrypy.tree.mount(SensorHubService(), script_name='/sensor-hub', config=conf) + app.merge(os.path.join(os.path.dirname(__file__), 'sensor-hub.config')) + return cherrypy.tree(environ, start_response) + +if __name__ == '__main__': + from cherrypy import wsgiserver, config + config.update({'environment': 'embedded', + 'request.throw_errors': True, + 'tools.gzip.on': True, + 'tools.gzip.mime_types':['text/*','text/html', 'application/json']}) + try: + srv = (r'0.0.0.0', 9876) + server = wsgiserver.CherryPyWSGIServer(srv, application) + print("starting server on port %d" % srv[1]) + server.start() + except Exception as e: + print(repr(e)) + except KeyboardInterrupt: + server.stop() + +# Local variables: +# mode: python +# End: diff --git a/sensordata.py b/sensordata.py new file mode 100644 index 0000000..41db2ee --- /dev/null +++ b/sensordata.py @@ -0,0 +1,91 @@ +#!/usr/bin/python3 + +"""Data manager for sensor hub. + +Reads data from the ASCII log file and presents a collection +""" + +__all__ = ['SensorData'] +__version__ = '1.0.0' +__author__ = 'Pat Thoyts ' +__copyright__ = 'Copyright (c) 2016 Pat Thoyts' + +import re + +class SensorDataIterator(): + def __init__(self, filename): + self.filename = filename + self.re = re.compile(r': {(?P.*?)} {(?P.*?)} {(?P

.*?)}$') + def __iter__(self): + self.fp = open(self.filename, 'rt') + return self + def __next__(self): + while True: + line = self.fp.readline() + if not line: + self.fp.close() + raise StopIteration() + if line.startswith('#') or line.startswith('!'): + continue + timestamp = line.split()[0] + m = self.re.search(line) + if m: + sensors = [] + for sensor,value in zip(['office1','office2','office3'], m.group('T').split()): + sensors.append({'id': sensor, 'value': value }) + return dict(timestamp=timestamp, name='spd-office', sensors=sensors) + +class SensorData(): + def __init__(self, filename): + self.filename = filename + + def __iter__(self): + return iter(SensorDataIterator(self.filename)) + +import unittest + +class TestSensorData(unittest.TestCase): + datafile = '_test_data.log' + data = '''# comment +! junk +1460641524 23.19 32.25 1003.39 : {23.0 23.37 23.19} {32.25} {1003.39} +1460641594 23.19 32.50 1003.43 : {23.06 23.37 23.19} {32.5} {1003.43} +1460641608 23.19 32.50 1003.42 : {23.06 23.37 23.19} {32.5} {1003.42} +''' + @classmethod + def setUpClass(clazz): + with open(clazz.datafile,'wt') as f: + f.write(clazz.data) + + @classmethod + def tearDownClass(clazz): + import os + os.remove(clazz.datafile) + + def test_parse(self): + data = SensorData(self.datafile) + count = 0 + for item in data: + check = [] + for sensor in item['sensors']: + check.append(sensor['id']) + self.assertListEqual(['office1','office2','office3'], check) + count = count + 1 + self.assertEqual(3, count) + +if __name__ == '__main__': + import sys + if len(sys.argv) > 2: + if sys.argv[1] == 'test': + # tester().test(argv[2]) etc + print("not implemented yet", file = sys.stderr) + 1 + else: + print("usage: SensorData test") + 1 + else: + unittest.main() + +# Local variables: +# mode: python +# End: diff --git a/static/ajax-loader.gif b/static/ajax-loader.gif new file mode 100644 index 0000000000000000000000000000000000000000..d2378cda8afd27be50f817d13262cb10a3b9535f GIT binary patch literal 9427 zcmb{2c~}$o+BfjYkbx{fLNXyN2~I-9fRL~W0YO6u5Ct&-w1~*Af<;6{#U9&+2?<$% z1aM&yWm9qA6{!m#Dpgdl)>6faXe%ylwe_^Nr_Y>s0NcKO-|M*^d#)${>g69FzxQ{3 z_irX69Ew5Cjj_2{;LbXe)|>s?knv3huDAphK*jquGL{To3XF^v0s12{`msC zxf8p#4I4U!y?%yWD#y+gVg2>kk592*USn6cVqc!XzB-Q`O2dw1V#k(Y7t64hqu8%M zVV`cs?w!W|^$Rw33p?+{F&2Vu#bQ`)9EygV?h>*w3%9 zZ-=o1i?ER%?9r##y))RGAFy5{_Sqiny9d}0Pp}(1u%-xX=WJ}xd~C-oZ2x=MokQ3U z-(asEV?TY5eRTo*;xKl5A9kSx8*IZq+l!r8ft_B3b*#Zo|rm~U5f2nhz)mQEz#J8kFmBn*cCH&r3!n{ zjkT+=>)WxTIoLo8)))r-^~d`+H9jgSF)~6KD9oQ3K4XW4>fr3;Y!ARL2WO+yU@2G$GF+TJo&15IuvtPt zCL9!*nXWEgzO4X}gE9w&u!ZC9!Vl2N#6n56V7gzoj$3P{z+9-}R}*M$%Mfg)Z)*oE z1R{y=QrHv$(L9oce8Qo{e30anD<6o7*7|UAfwX$UPAm6J~)_`!3Z7sCI*&r!(Yyk(UV{~;SGiVy#oYk z3glGKv>Vwiq<41u2r|&&GsQnXyMJv%0`e|MWi+O_lI@kexW+_rShy$^s?$*Yd^Fs+ zMir$QKSuH zl$*3)*#dC4h|)82GT29Q_NPJ=Mt~HmgydP-OO|*8vuEW_0fMr8gd(%RHY?RZ0f0On zx*$;~+-zo&Zu#k?c)D6e(-Lpku4_6FM;vC^vP#LkmFTnxUX{ zSJ!+1iQXR#QS1sFBxbf-`ui(=0DEVrRe&NTyfD!$n(aoTAtZ0yJYLU(a;2Ph2Bxg7KSAX_^( zArPQcMFI=fEDD`%=Cb2M+@=Fi!hF>It5Eh9(W4M=T~Oi7Ot^t76qg~kXB|fOsR$Zo z!Eo2niLv|1b4^JLpW#9<5|ndPe_7O+ueBnCpr7s!I`&weeXeL7uw{w1MvTg$li60fmYl zwzwTIwC9r^U2Tj#{aw5Cof0IPo=-7XOm#>K*kVFF9Ok)$4F>MoommKqh-q8q=L=Cv zk%RL67HWEdDxs2@Hd~PgfOZaZiU3Ob(&cO7&rf%ZiYEf=G92=O+3RmCTvv%CuCfnW z3_#+eVDz#zh0(AXUa@NGIc?0#Xh8!KfpS`2UPdjd3aL^!peGVQJRP}%sI)8!R_4kh zA{`7B9N&V+?Fy12npQBZZ*K?eL8-_;OMgT~VEfytYESQ#(qVC`6aT1v-)^(ZB9smR zSE6@Ry5(*ftQxB@dCZ`ENJP7Uj{#C{ASI*Q4vCvSNG!~L(R6YKOo$sO_C3@L3Qpaf z?4%FzC*Zq`8f0ZA%^8*sjdBf$9yE4Lj0tC;v#;ED+N6Hl`MVfWXbOAf}SC z!jD}8#;i|oOWyeLBh+1`QOvHPPct&AP`1$jma_D5KWoZLWkpZB z%7G9>P9DxUF><-0%t#qQ0C`DwgPziXmp2$Tcf7n+N9g2oY2dl5BxY{_$657T{%hos zsUl|8OPFyoHw+Ah8GW^|G#0t3w(pEsuM}=a9XxVikj)?;UN`=PJAL@w;#7u!=E{hu z+0vTwDXjD8>*F;@QsdQuh3hqIZV3Aak!&fIcP;oYAc;L4;`}Z#n3)dyYVMs$4*d>Dys;OWyQq-&n;0nao}rGh#ghAbPtk zAh)S+Mg;fT`^T;wAewAw_Z+>XS^I}S89xV@g6Z#)B(tTAlEb7VcI6WGg~~KtL4ZFmI}vDYlXU&^pD24kSKF8Am@>4h0K^Vi*w5CW(Q_CG+B3 z0?g7ILK-RS?##bO_44wuGsCR1riPFxQo{XHkM1{#x5yHX~Gjvwfk&HN5br(e#_H?WOADS1P;g@UE6vjZYecUOZ_!Dw-y7A3LTbFp?{nNg=x`kv?&tj-B)PIS$jWqfA@; zvcU+_B4n2hrF6tM)4}56k|g-0`ldxqxa;E5nZ1W^p6<_JscYl1c*JG0`lH#>Dk>OE z@%rEhkHYkbCyoUqHEFKL{1A#{`N@}`MXutFR5cRo+I;fFULubv4f3>CN9b zMU=s<&mBmc9{_ZPwR`||l>nohBFzirE%h1Nfi2yX+gZ{p~dD2McW6cCsu)4AqnTF&c37kTtZ-B4)~ zqW5F#IJtxV)?@CBJH&X#W^1Dx*AEF7F0%&=DM4DNuGTV=fbKqn(5FXXHRSKLxu|Sk z?V>ahY%@=|etH4k+~seZy8(Qs=EegBRaL;jM9O9(4!|izYAjF-Kx`DF6tbLMR~Z#ZQ+H0jiB-QPolSu$%H>`gpSus#?NaF^FlQU$~Qu6JoEaX+dq5I>VvaX@F!j+65y#4#eCzV`Y}z!l~6- z*?TQhayp9)bY2-BwRT|T$*M-V{~#8OCB7c1tTl(^o@;IETGG1qpSM?jS$#`Yg52NA zx_a@^Rh_;p$+{`z0la1~B%zv03!F73EN=5&EMa?FwElFG!k34Cyj4l6j?!4 z2Z7w;u9lpxX-n9!UOig)&qM-)8vW7m#L8xQ*CE3b1Tv}61QfMSQ#&3Y4J@kS67l%7 z;_+ePyVf5}@Y(G?VheS9Bc7aNLZ1~<3xpeWX=Of%!e+dD?&xIOcAEBZhxHRcvguLr zK~oMI4hsW~el~V2mfK?Z{rrDRfA>)7Z}NwVE!)|Xhip;XI~}qoZbu-2pvH)Y zh+3;BHgGgtZWV!G5h7t3vbF2Z86)RR3T$S>u^`FBA=dl<)8(C zz^!M5OZCyV&J;e2G#Z!DJY^mq8M>nHJ-h}?=>U1rR^B3EPB-;Xoh%A!f zHQlWQDSX6$SfL|Mo`6htQ{X|7R#Vi(^0v&j&X)+^2+dzvjXBA?f#~_n%HJ$s9+@1k z=)W;u6EMs_U=ZX_7@F8B(1OE^7fvvIrU4Jogbaw*H%!u;)5;pU@uInQ4_+6^W=}JH|Ees(5p%S5RE-oq+TLPsz3i-E^{0z@gwgslT}$;k z+ZnqM774tr<%zTJCJo-fVb+2>00J-?OBAi%CoE2)-t{Ji$w&SroeI`U{NBQZExej9 z1o&pg&Jcd6QAmuu6*QVV+XyIJ$dEu* zW2F5l1C7gU1I8mm69;oG^+NISaTq|>Af){+!@RJs65^aXtDU$xn>YmGp@DWyz?dMZ zm6f#5h|3sg^L_I2?tFLQl|I5B^FIXRG8eb==|1t>EnrUruh4Sj`rXT;b1v#5lnQd@ z7)Yc|@d#aW^l2X_o2PV|Ik4Nhk;Zn-`#^Jv3QU{z(h2Yt#5XvsWrTsRWVTiWxO%80 zvnP3r@9dAI45h7&hu-sbBcit=jGBWHx4fAo^_34O*S#2Mi zFmfil{~CwJ7?Sc|N$F${Px&U#&%05>%mm-?ij+qUYS%-)XL76g8ZxR4lYgk$I)E}( zk$g{$pQM}qM(N1@f&*3m(K$3L4)~`{M}u&$nR*_7j<*5K^Jj6Klh;ar$I*`CT=_je zoxP_0I8IN)sb}ZE)yvMmGEU-1h!Ws=G+jY>gXk0`!5-7eTcPF-X6u_+`6R2gehOoR~`uOaOW|A^?fi zmLl0!dGi`ItoNC1tL}nl7^^7{3O2SxTz<|)wciyV)NG|2XzNI){`YXG!4dxu|DD4DTn00BZ zr#p9}CHwMqI*`_JyBL%jwXFQx>f+Z#$__pVA9$M8M(p!UTeyAb7X`@=5cL!p)|PHo zv&s81*EsBntqpVjWRU*o^XP6_%vl$S>$fOjcfK`#CqzZwaXrPK4fKxkV}HbEFD`S>AUGfVmELLv*{{(c!#mk0bDqa( z#`x_Z>60jK>5E6vB%3N8Vt=Ls_EfL~3OG(Jtmh5}-=&A%Sb!#gumVt)TF}vMcN^aQ z_GH3{%#LgY5JQeuce#_-54ZKXOG^whJaTtG_PkoPnWQofx2Ur1xD``N02T1*B}qge za-c45H$gfiFV#UBME+r$?{7BdalKSS3JT7e_-*OAfvP5WV^speX zkC{k^j1|H@Npsh;Mr)GJS3u;`hJ3Qg&XN(q@{}5mP5ZgV*lh)WqmuuUo*e)s;_X34 zK$Jp%md@!0;U{M5IlOd^0?cz4@bsMcOM0qg?G=~Hr#tz+-)MlOXYLtgPBXt>WfYvl zd7Kw5DhzmWuEi8tudme{R1!Eu6b@_af;v<>w|-w>l&;m93Qsr?zD9YBTN&XDkU(R4 zEl61c0`j3&FOMySet`74nRl`DH8@I=yL!y)>=zM`qClVz8yl~)RXYr4B{^Rug=R%@!p-8@y z`?!!5aY!dCURYSj#*JExGS#wWOIN-5sXVJ_&S(jDDlX&{*rv9|g&b2A$OtZE^ypXh zXe)3bwRDZDgi$S%J zS3mzRkH63FkMxQ9389~uj1XQxQf2!5{>cWC*k3B+2I8xPKn7Oy83pMs&V1MQ38vGA zwDs%5!j{@UIp1Ok7$(7EQC0Nx1B|Ms&xlv6t+##kuy|L7YAVAahR5&rAcjQ3Z6PUz zNPGoRr%18_4c4f{b?ml78&HZM=Z9x}kezdpL!J*1H~Np<>7IAbiEKIYFjmV1C`Xzu zF}kcavbfg^uj;k~GfrRxiS?Y4&G?Sr*1Yd1-7-yVCqK2za`NE#w(j+J4$Xgzn1+k* z>J6M7K;FFBlZzHUWzFWb;hxr$=bld*gcr<=nc>ki6g|rw5TCOtL<@A+j(rGFJ z=a|$RelYj@)$!dvs#qe8zo_6V#xprh&f`e7^;=1N3Pfb zzBXei9eLgEd>bcZHMwa^KlIPypoE$|`g$c)kncijlFo@{RvYfN*$H@QL_4^{CujY{ zeTLhACIrhSg?#3DMop}u6Fua%D0HFq#9}caWLuarlYt8X6GTac%O_u^Rx_tA?w=%* z1&$b>>hsp-#5%Y&PXvi;KxU{^E-uA-A>ZKzi%%%37bZy}t1KB(c#hxT?&~DiAL*kC=tRa+jFzaCtu&^Ot)M0a?3T5W!fWpG;n(m{X&Sa!F^E0qoYcDJu8DLG`_$>K5 z`^BFj3jCc&>Ac`iBAaCkm-_!n6ce_5VaV@9%$%_G>n2mgO!J>4f<+TV2{u`ot3w`5 z5K#;uYsnIa*jN!xWUD_vGn?VTX;CO5qO6>1eFwX`!{kt5A9z#&KrY-k&u1@tMtGC~ zHC4c`0vM!NJ51Gxmh2yvezlIRo@$gV3tjFlB(pErTvJ< zs1hQ8)7YI{FLgFs9|eXgot)-YfgBlKMSn_?>xuJ4EWRrKQ7b!fN7E1y=$!*dP7M}0 z%f?I_!fc9^r{*nh1+A240Z1*4j-VEdv|mwlgZqPPA2Q_Y8T4gTT8az9_hJ3pR~Vk} zenzB51yE{G8JkyHG&3}&4z!v0+M{@8xJO-Jsff8vb^M-K1{MT3Zt2SwX0twY$m>%0IJa8&dFj8{&WTDOYE7kY# N6VS}@Phci~{|7LHP_F<0 literal 0 HcmV?d00001 diff --git a/static/sensor-hub.html b/static/sensor-hub.html new file mode 100644 index 0000000..580d108 --- /dev/null +++ b/static/sensor-hub.html @@ -0,0 +1,203 @@ + + + + + + +Lab Temperature Monitor + + + + +

+
+
+
+

Download view data as ASCII or JSON

+

+

+
+ + + + + + + + + -- 2.23.0