Switched to a python serial port monitor which updates mongodb on the fly.

The database is now directly updated by the serial monitor which is using
twisted to read the serial lines. Timestamps and values are now numerical
and not stored in the db as strings so minor change in the wsgi to handle
that.

diff --git a/monitor.py b/monitor.py
new file mode 100755 (executable)
index 0000000..6ed71ef
--- /dev/null
+++ b/monitor.py
@@ -0,0 +1,129 @@
+#!/usr/bin/python3
+#
+# curl --data "key=$KEY&field1=$Temp0&field2=$Pressure0&field3=$Lux" $URL
+
+from __future__ import print_function, division, absolute_import
+import sys, pymongo
+from twisted.internet import reactor, stdio
+from twisted.internet.serialport import SerialPort, PARITY_ODD, PARITY_NONE
+from twisted.protocols import basic
+from pymongo import MongoClient
+from os import path
+from configparser import ConfigParser
+
+__all__ = ['MonitorSensorHub','SensorProtocol']
+__version__ = '1.0.0'
+__author__ = 'Pat Thoyts <patthoyts@users.sourceforge.net>'
+__copyright__ = 'Copyright (c) 2016 Pat Thoyts'
+
+class MonitorSensorHub():
+    """Class to handle database updates for new data items read from
+    the sensor-hub over serial port.
+    Granularity sets the time in seconds between database updates. This
+    reduces the datarate as the sensor-hub issues an update every second.
+    """
+    def __init__(self, granularity = 60):
+        self.granularity = granularity
+        config = ConfigParser()
+        filename = path.join(path.dirname(path.realpath(__file__)), 'sensor-hub.config')
+        config.read(filename)
+        self.uri = config['database']['uri'].strip("\'")
+        client = MongoClient(self.uri)
+        self.db = client.sensorhub.sensorlog
+        self.recent = self.recent()
+
+    def opendb(self):
+        """Open the database"""
+
+    def recent(self):
+        """Get the timestamp of the most recent item in the database."""
+        try:
+            cursor = self.db.find().sort([("timestamp", pymongo.DESCENDING)]).limit(1)
+            last = dict(next(cursor))
+        except StopIteration:
+            last = {'timestamp': 0}
+        r = int(last['timestamp'])
+        return r
+
+    def update(self, item):
+        """Update the database with a new data item only if sufficient
+        time has passed since the last update, as set by the granularity
+        property."""
+        t = int(item['timestamp'])
+        if t > (self.recent + self.granularity):
+            self.recent = t
+            result = self.db.insert_one(item)
+            print("{0} {1}".format(result.inserted_id, self.recent), flush=True)
+
+class SensorProtocol(basic.LineReceiver):
+    """Read input from the sensor-hub and parse each line into a
+    dictionary of values.
+    The temperature is recorded per sensor but the humidity and pressure
+    are reported singly for the hub.
+    """
+    delimiter = b'\r\n' # default delimiter is crlf
+    def __init__(self, monitor):
+        self.monitor = monitor
+        self.log = open('/tmp/sensor-hub.log', 'a+')
+        super().__init__()
+
+    def lineReceived(self, data):
+        try:
+            line = data.decode('ascii').strip()
+            if len(line) > 0:
+                print(line, file=self.log, flush=True)
+                if not line[0] == '#':
+                    item = self.parse(line)
+                    self.monitor.update(item)
+        except Exception as ex:
+            print("error: " + repr(ex), file=sys.stderr)
+
+    def parse(self, data):
+        item = dict(name='spd-office')
+        try:
+            if not data[0] == '#':
+                sensors = []
+                humidity = 0.0
+                pressure = 0.0
+                parts = [x.rstrip(" ]\r\n") for x in data.split('[')][1:]
+                for part in parts:
+                    values = part.split()
+                    sensor = {'id': 'office' + str(values[0])}
+                    #if len(values) > 1: # timestamp
+                    #    #sensor['timestamp'] = values[1]
+                    if len(values) > 2:                             # temperature
+                        sensor['value'] = float(values[2])
+                    if len(values) > 3 and float(values[3]) != 0.0: # humidity
+                        item['humidity'] = float(values[3])
+                    if len(values) > 4 and float(values[4]) != 0.0: # pressure
+                        item['pressure'] = float(values[4])
+                    sensors.append(sensor)
+                item['sensors'] = sensors
+                item['timestamp'] = int(data.split()[0])
+        except IndexError:
+            pass
+        return item
+
+def main(argv = None):
+    """Monitor the serial port for lines from the sensor-hub and update
+    the mongodb database and notify thingspeak at reduced time-rates.
+    """
+    port,baud = r'/dev/ttyUSB0',57600
+    if argv is None:
+        argv = sys.argv
+    if len(argv) > 1:
+        port = argv[1]
+    if len(argv) > 2:
+        baud = argv[2]
+    monitor = MonitorSensorHub()
+    serial = SerialPort(SensorProtocol(monitor), port, reactor, baudrate=baud)
+    serial._serial.parity = PARITY_ODD # work around pyserial issue #30
+    serial._serial.parity = PARITY_NONE
+    try:
+        reactor.run()
+    except KeyboardInterrupt:
+        reactor.stop()
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

diff --git a/sensor-hub.conf b/sensor-hub.conf
index 76f562c76cf6e827d5c9c3390ae30b782575c355..145dad49e1d71b82a734c1241ddd8ba4edb98544 100644 (file)
--- a/sensor-hub.conf
+++ b/sensor-hub.conf
@@ -8,6 +8,6 @@ respawn
 
 script
   [ ! -f /etc/default/sensor-hub ] || . /etc/default/sensor-hub
-  exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/tclsh /home/pat/sensor-hub/monitor.tcl /dev/ttyUSB0 57600 > /tmp/sensor-hub.log"
+  #exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/tclsh /home/pat/sensor-hub/monitor.tcl /dev/ttyUSB0 57600 > /tmp/sensor-hub.log"
+  exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/python3 /opt/django/wsgi-scripts/sensor-hub/monitor.py /dev/ttyUSB0 57600 >> /var/log/sensor-hub.log 2>&1"
 end script
-

diff --git a/sensor-hub.wsgi b/sensor-hub.wsgi
index 4f27a8f172af2fdb56e0b9bf3dcb56f00b204cdb..2f24f7aaab575b36b66f12ae73fc3b8ed10805d6 100755 (executable)
--- a/sensor-hub.wsgi
+++ b/sensor-hub.wsgi
@@ -91,9 +91,9 @@ class SensorHubService():
         uri = cherrypy.request.app.config['database']['uri']
         client = MongoClient(uri)
         db = client.sensorhub.sensorlog
-        selector = [{"timestamp": {"$gt": str(when)}}]
+        selector = [{"timestamp": {"$gt": int(when)}}]
         if not until is None:
-            selector.append({"timestamp": {"$lt": str(until)}})
+            selector.append({"timestamp": {"$lt": int(until)}})
         cursor = db.find({"$and": selector}).sort([("timestamp", pymongo.ASCENDING)])
         return RemoveKeyIterable(cursor, '_id')
 

diff --git a/sensordata.py b/sensordata.py
index 4c0142596fece2ce97966c404037368332a85080..79e3cd7c9aac51300ad3f21ab5f35e50d5348154 100644 (file)
--- a/sensordata.py
+++ b/sensordata.py
@@ -9,6 +9,7 @@ from __future__ import print_function, absolute_import, division
 import sys, re, json, pymongo, unittest
 from pymongo import MongoClient
 from configparser import ConfigParser
+from os import path
 from timeit import timeit
 
 __all__ = ['SensorData','SensorDataIterator']
@@ -38,12 +39,12 @@ class SensorDataIterator():
                 raise StopIteration()
             if line.startswith('#') or line.startswith('!'):
                 continue
-            timestamp = line.split()[0]
             m = self.re.search(line)
             if m:
                 sensors = []
                 for sensor,value in zip(['office1','office2','office3'], m.group('T').split()):
-                    sensors.append({'id': sensor, 'value': value })
+                    sensors.append({'id': sensor, 'value': float(value) })
+                timestamp = int(line.split()[0])
                 item = dict(timestamp=timestamp, name='spd-office', sensors=sensors)
                 humidity = m.group('H').split()
                 if len(humidity) > 0:
@@ -161,7 +162,8 @@ def recent(*args, **kwargs):
 
 def opendb():
     config = ConfigParser()
-    config.read('sensor-hub.config')
+    filename = path.join(path.dirname(path.realpath(__file__)), 'sensor-hub.config')
+    config.read(filename)
     uri = config['database']['uri'].strip("\'")
     client = MongoClient(uri)
     return client