The default log has values for every second, reduced to every 10 minutes.
The lab-monitor data is 1 each minute.
Signed-off-by: Pat Thoyts <patthoyts@users.sourceforge.net>
--- /dev/null
+/__pycache__/
+/data/
+/sensor-hub.config
+
--- /dev/null
+#!/usr/bin/python3
+
+"""Import sensorlog data to JSON for import into mongodb
+
+mongoimport --db test --collection sensorlog --drop --file JSONDATAFILE
+"""
+
+from __future__ import print_function, absolute_import, division
+import sys, json, pymongo
+from sensordata import SensorData
+from pymongo import MongoClient
+
+def usage():
+ print("usage: importlog import filename\n demo", file=sys.stderr)
+
+class GranularData():
+ def __init__(self, iterable, granularity):
+ self.iterable = iterable
+ self.granularity = granularity
+ def __iter__(self):
+ self.last = 0
+ return self;
+ def __next__(self):
+ while True:
+ item = next(self.iterable)
+ t = int(int(item['timestamp']) / self.granularity) * self.granularity
+ if t != self.last:
+ self.last = t
+ return item
+
+def import_logfile(filename, granularity = None):
+ """Import a sensor-hub logfile into mongodb"""
+ if granularity is None:
+ granularity = 600 # 10 minutes
+ granularity = int(granularity)
+ last = 0
+ mongo = MongoClient()
+ db = mongo.test.sensorlog
+ db.drop()
+ db.insert_many([item for item in GranularData(iter(SensorData(filename)), granularity)])
+ return 0
+
+def tojson(filename):
+ last = 0
+ for item in SensorData(filename):
+ t = int(int(item['timestamp']) / 600) * 600
+ if t != last:
+ print(json.dumps(item))
+ last = t
+ return 0
+
+def demo():
+ '''
+ {"timestamp": "1460971800",
+ "name": "spd-office",
+ "sensors": [{"id": "office1", "value": "22.25"},
+ {"id": "office2", "value": "22.69"},
+ {"id": "office3", "value": "22.37"}]}
+ '''
+ uri = 'mongodb://localhost:27017/test'
+ client = MongoClient(uri)
+ db = client.test.sensorlog
+ #db.insert_one(json) / insert_many (iterable)
+ #cursor = db.find({"name": "spd-office"})
+ #cursor = db.find({"sensors.id": "office1", "sensors.value": "22.25"})
+ cursor = db.find({
+ "$and": [
+ { "timestamp": {"$gt": "1460968800"} },
+ { "timestamp": {"$lt": "1460970100"} }
+ ]
+ }).sort([("timestamp", pymongo.ASCENDING)])
+ for item in cursor:
+ print(item)
+ return 0
+
+def main(args = None):
+ if args is None:
+ args = sys.argv
+ if len(args) < 2:
+ usage()
+ return 1
+ if args[1] == "convert":
+ return tojson(args[2])
+ if args[1] == "import":
+ return import_logfile(args[2], args[3])
+ if args[1] == "demo":
+ return demo()
+ usage()
+ return 1
+
+if __name__ == '__main__':
+ sys.exit(main())
--- /dev/null
+#!/bin/bash
+set -euo pipefail
+awk '
+BEGIN { last=0; }
+/^14/ {
+ t = (int($1 / 600) * 600);
+ if (t != last) {
+ last = t;
+ print $0
+ }
+}' $1 > $2
+
--- /dev/null
+[database]
+url = 'mysql+pymysql://USERNAME:PASSWORD@localhost/DATABASENAME'
+
+[logging]
+error_log = 'error.log'
+
--- /dev/null
+#!/usr/bin/python3
+#
+# Present data from the Arduino sensor-hub over a web API.
+
+from __future__ import print_function, division, absolute_import
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__)))
+
+import cherrypy, json
+from cherrypy import config, tools
+from time import localtime, time
+from datetime import datetime
+from dateutil import parser as dateparser
+from statistics import median
+from urllib3.util import parse_url
+from sensordata import SensorData
+
+class SensorHubService():
+
+ def __init__(self):
+ self.version = '1.0'
+ self.errlog = None
+
+ def log(self, msg):
+ if self.errlog is None and 'logging' in cherrypy.request.app.config:
+ self.errlog = os.path.join(os.path.dirname(__file__),
+ 'data',
+ cherrypy.request.app.config['logging']['error_log'])
+ if not self.errlog is None:
+ with open(self.errlog, 'a') as f:
+ print(msg, file=f)
+
+ @cherrypy.expose
+ def recent(self, *args, **kwargs):
+ return json.dumps(dict(response='error', message='not implemented'))
+
+ @cherrypy.expose
+ def since(self, *args, **kwargs):
+ """Get data since any timepoint.
+ eg: /lab-monitor/since?when=2016-03-01T00:00:00
+ /lab-monitor/since?when=14 days ago
+ """
+ try:
+ cherrypy.response.headers['content-type'] = 'application/json'
+ param = cherrypy.request.params.get('when')
+ if not param is None:
+ when = dateparser.parse(param)
+ when = when.timestamp()
+ else:
+ when = int(time() - (86400 * 7)) # 7 days ago
+ res = json.dumps(self.get_since(int(when))).encode(encoding='utf-8')
+ except Exception as e:
+ cherrypy.response.headers['content-type'] = 'application/json'
+ self.log("error in \"since\": {0}".format(str(e)))
+ res = dict(response='error', message=str(e))
+ res = json.dumps(res).encode(encoding='utf-8')
+ return res
+
+ def get_since(self, when):
+ url = cherrypy.request.app.config['database']['url']
+ path = parse_url(url).path
+ return dict(result=[x for x in SensorData(path) if int(x['timestamp']) >= when])
+
+def application(environ, start_response):
+ staticdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static'))
+ logdir = os.path.join(os.path.dirname(__file__), 'data')
+ conf = {
+ '/': {
+ 'tools.staticdir.root': staticdir
+ },
+ '/sensor-hub.html': {
+ 'tools.staticfile.on': True,
+ 'tools.gzip.on': True,
+ 'tools.staticfile.filename': os.path.join(staticdir, 'sensor-hub.html')
+ },
+ '/loading.gif': {
+ 'tools.staticfile.on': True,
+ 'tools.gzip.on': False,
+ 'tools.staticfile.filename': os.path.join(staticdir, 'ajax-loader.gif')
+ }
+ }
+ app = cherrypy.tree.mount(SensorHubService(), script_name='/sensor-hub', config=conf)
+ app.merge(os.path.join(os.path.dirname(__file__), 'sensor-hub.config'))
+ return cherrypy.tree(environ, start_response)
+
+if __name__ == '__main__':
+ from cherrypy import wsgiserver, config
+ config.update({'environment': 'embedded',
+ 'request.throw_errors': True,
+ 'tools.gzip.on': True,
+ 'tools.gzip.mime_types':['text/*','text/html', 'application/json']})
+ try:
+ srv = (r'0.0.0.0', 9876)
+ server = wsgiserver.CherryPyWSGIServer(srv, application)
+ print("starting server on port %d" % srv[1])
+ server.start()
+ except Exception as e:
+ print(repr(e))
+ except KeyboardInterrupt:
+ server.stop()
+
+# Local variables:
+# mode: python
+# End:
--- /dev/null
+#!/usr/bin/python3
+
+"""Data manager for sensor hub.
+
+Reads data from the ASCII log file and presents a collection
+"""
+
+__all__ = ['SensorData']
+__version__ = '1.0.0'
+__author__ = 'Pat Thoyts <patthoyts@users.sourceforge.net>'
+__copyright__ = 'Copyright (c) 2016 Pat Thoyts'
+
+import re
+
+class SensorDataIterator():
+ def __init__(self, filename):
+ self.filename = filename
+ self.re = re.compile(r': {(?P<T>.*?)} {(?P<H>.*?)} {(?P<P>.*?)}$')
+ def __iter__(self):
+ self.fp = open(self.filename, 'rt')
+ return self
+ def __next__(self):
+ while True:
+ line = self.fp.readline()
+ if not line:
+ self.fp.close()
+ raise StopIteration()
+ if line.startswith('#') or line.startswith('!'):
+ continue
+ timestamp = line.split()[0]
+ m = self.re.search(line)
+ if m:
+ sensors = []
+ for sensor,value in zip(['office1','office2','office3'], m.group('T').split()):
+ sensors.append({'id': sensor, 'value': value })
+ return dict(timestamp=timestamp, name='spd-office', sensors=sensors)
+
+class SensorData():
+ def __init__(self, filename):
+ self.filename = filename
+
+ def __iter__(self):
+ return iter(SensorDataIterator(self.filename))
+
+import unittest
+
+class TestSensorData(unittest.TestCase):
+ datafile = '_test_data.log'
+ data = '''# comment
+! junk
+1460641524 23.19 32.25 1003.39 : {23.0 23.37 23.19} {32.25} {1003.39}
+1460641594 23.19 32.50 1003.43 : {23.06 23.37 23.19} {32.5} {1003.43}
+1460641608 23.19 32.50 1003.42 : {23.06 23.37 23.19} {32.5} {1003.42}
+'''
+ @classmethod
+ def setUpClass(clazz):
+ with open(clazz.datafile,'wt') as f:
+ f.write(clazz.data)
+
+ @classmethod
+ def tearDownClass(clazz):
+ import os
+ os.remove(clazz.datafile)
+
+ def test_parse(self):
+ data = SensorData(self.datafile)
+ count = 0
+ for item in data:
+ check = []
+ for sensor in item['sensors']:
+ check.append(sensor['id'])
+ self.assertListEqual(['office1','office2','office3'], check)
+ count = count + 1
+ self.assertEqual(3, count)
+
+if __name__ == '__main__':
+ import sys
+ if len(sys.argv) > 2:
+ if sys.argv[1] == 'test':
+ # tester().test(argv[2]) etc
+ print("not implemented yet", file = sys.stderr)
+ 1
+ else:
+ print("usage: SensorData test")
+ 1
+ else:
+ unittest.main()
+
+# Local variables:
+# mode: python
+# End:
--- /dev/null
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+<meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1.5"/>
+<title>Lab Temperature Monitor</title>
+<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.8.3/modernizr.min.js"></script>
+<style type="text/css">
+{margin:0;padding:0}
+html {background:#eee;}
+body {margin:0 auto;max-width:980px;padding:1em;color:#000;background-color:#fff;min-height:640px;}
+#header {border-bottom:thick solid #ff9933;}
+#footer {border-top: thin solid #ccc;}
+#footer #status {float:left;}
+#footer #timestamp {float:right;font-size:small;}
+#result table {margin-left:auto;margin-right:auto;background-color:#888;}
+#result th {text-align:left;background-color:#eee;}
+#result tbody td {vertical-align:top;font-family:fixed;font-size:11pt;background-color:#fff;padding:6px 4px 6px 4px;}
+#tooltip {position:absolute;display:none;border:1px solid #fdd;padding:2px;background-color:#fee;opacity:0.80;}
+div.loading { background: rgba(255,255,255,.8) url('loading.gif') 50% 50% no-repeat; }
+</style>
+</head>
+<body>
+<div id="header"><h1>Office Temperature Monitor</h1></div>
+<div id="content">
+<div id="overview" style="height:80px;"></div>
+<div id="graph" style="height:400px;"></div>
+<p>Download view data as <a href="#" onclick="on_download('text')">ASCII</a> or <a href="#" onclick="on_download('json')">JSON</a></p>
+<p id="result"/>
+<div id="tooltip"></div>
+</div><!-- /content -->
+<div id="footer">
+ <span id="status">Fetching data...</span>
+ <span id="timestamp"></span>
+</div>
+<!-- <script type="text/javascript" src="/static/jquery.min.js"></script> -->
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.time.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.selection.min.js"></script>
+<script type="text/javascript">
+function displayTime(ts) {
+ var d = new Date(ts);
+ return d.toLocaleTimeString() + " " + d.toLocaleDateString();
+}
+function on_ajax_error(jqxhr, err, evt) {
+ $('#status').text(err);
+ alert("argh: " + err);
+}
+function on_ajax_complete(jqqxhr, msg) {
+ $('#status').text("");
+ $('#graph').removeClass("loading");
+ $('#overview').removeClass("loading");
+}
+function weekend_markings(axes) {
+ var markings = [], d = new Date(axes.xaxis.min);
+ d.setUTCDate(d.getUTCDate() - ((d.getUTCDay() + 1) % 7));
+ d.setUTCSeconds(0);
+ d.setUTCMinutes(0);
+ d.setUTCHours(0);
+ var i = d.getTime();
+ do {
+ markings.push({ xaxis: { from: i, to: i + 2 * 24 * 60 * 60 * 1000 } });
+ i += 7 * 24 * 60 * 60 * 1000;
+ } while (i < axes.xaxis.max);
+ return markings;
+}
+function on_download(mimetype)
+{
+ if (document.custom.graph) {
+ var axes = document.custom.graph.getAxes()
+ var upper = parseInt(axes.xaxis.max / 1000);
+ var lower = parseInt(axes.xaxis.min / 1000);
+ var url = 'download?type=' + mimetype + '&from=' + lower + '&to=' + upper;
+ var win = window.open(url, '_blank');
+ if (win)
+ win.focus();
+ }
+}
+function on_received_recent(recent) {
+ document.custom.recent = recent;
+ var t = new Date(recent.response.timestamp * 1000);
+ var ts = t.toLocaleTimeString() + " " + t.toLocaleDateString();
+ var r = $('<tr>');
+ $('<td>').text(recent.response.name).appendTo(r);
+ $('<td>').text(ts).appendTo(r);
+ $.each(recent.response.sensors, function(i, sensor) {
+ $('<td>').text(sensor.id).appendTo(r);
+ $('<td>').text(sensor.value).appendTo(r);
+ });
+ var t = $('<table><tr><th colspan="6">last received data</th></tr><tr><th>name</th><th>time</th><th colspan="4">sensors</th></tr></table>');
+ r.appendTo(t);
+ t.appendTo('#result');
+}
+function on_draw_graph(data) {
+ document.custom.data = data;
+ if (data.response == "error") {
+ $("#status").text(data.message);
+ return;
+ }
+ var now = new Date();
+ var plotA = [];
+ var plotB = [];
+ var plotC = [];
+
+ var colors = ["#a040b0","#00a040","#0040a0"];
+ var plotdata = [];
+ $.each(data.result[10].sensors, function(n, sensor) {
+ plotdata.push({data: [], label: sensor.id, color: colors[n]});
+ });
+ $.each(data.result, function(i,item) {
+ var t = item.timestamp*1000;
+ $.each(item.sensors, function(n, sensor) {
+ plotdata[n].data.push([t,sensor.value]);
+ });
+ });
+ $('#status').html('');
+ if (typeof(document.custom.recent) != typeof(undefined)) {
+ $('#timestamp').html("Last collection at " + displayTime(document.custom.recent.response.timestamp * 1000));
+ }
+ var options = {
+ series: {
+ shadowSize: 0,
+ bars: { show: false },
+ points: { show: false, radius: 1, },
+ lines: { show: true, lineWidth: 2, fill: false,}
+ },
+ selection: { mode: "x" },
+ grid: { hoverable: true, clickable: false, markings: weekend_markings, show: true },
+ xaxis: { mode: "time", timezone: "browser", timeformat: "%d-%b<br/>%H:%M" },
+ yaxis: { }
+ };
+ var plot = $.plot($('#graph'), plotdata, options);
+ document.custom.graph = plot;
+ var ovopt = {
+ series: { lines: { show: true, lineWidth: 1 }, shadowSize: 0 },
+ legend: { show: false },
+ grid: { markings: weekend_markings },
+ xaxis: { mode: "time", timezone: "browser", timeformat: "%d-%b" },
+ yaxis: { ticks: [], autoscaleMargin: 0.1 },
+ selection: { mode: "x" }
+ };
+ var ovplot = $.plot("#overview", plotdata, ovopt);
+ $("#graph").bind("plotselected", function (event, ranges) {
+ $.each(plot.getXAxes(), function (_, axis) {
+ var opts = axis.options;
+ opts.min = ranges.xaxis.from;
+ opts.max = ranges.xaxis.to;
+ });
+ plot.setupGrid();
+ plot.draw();
+ plot.clearSelection();
+ ovplot.setSelection(ranges, true); // avoid event loop
+ });
+ $("#overview").bind("plotselected", function (event, ranges) {
+ plot.setSelection(ranges);
+ });
+}
+function main() {
+ //$.ajax({
+ // url: 'recent',
+ // dataType: 'json',
+ // complete: function(jqqxhr, msg) { $('#result').removeClass("loading"); },
+ // beforeSend: function(jqxhr, opt) { $('#result').addClass("loading"); },
+ // success: on_received_recent,
+ // error: function(jqqxhr, err, evt) { alert("recent: "+err); }
+ //});
+ var now = ((new Date() - 0) / 1000);
+ var when = now - (60 * 60 * 24 * 360); // 28 days
+ when = (new Date(when * 1000)).toISOString();
+ $.ajax({
+ url: 'since',
+ dataType: 'json',
+ data: { when: when },
+ beforeSend: function (jqxhr, opts) {
+ $("#graph").addClass("loading");
+ $("#overview").addClass("loading");
+ },
+ complete: on_ajax_complete,
+ error: on_ajax_error,
+ success: on_draw_graph,
+ });
+}
+$("#graph").bind("plothover", function(event, pos, item) {
+ if (item) {
+ var x = item.datapoint[0].toFixed(2),
+ y = item.datapoint[1].toFixed(2);
+ var t = displayTime(new Date(x - 0));
+ $("#tooltip").html(item.series.label + "<br/>" + y + "°C @ " + t)
+ .css({top: item.pageY + 5, left: item.pageX + 5})
+ .fadeIn(200);
+ } else {
+ $("#tooltip").hide();
+ }
+});
+$(document).ready(function () {
+ document.custom = {};
+ main();
+});
+</script>
+</body>
+</html>
projects / spd / sensor-hub.git / commitdiff
