From: Pat Thoyts <Patrick.Thoyts@renishaw.com>
Date: Wed, 18 May 2016 09:00:48 +0000 (+0100)
Subject: Switched to a python serial port monitor which updates mongodb on the fly.
X-Git-Url: https://privyetmir.co.uk/gitweb.cgi?a=commitdiff_plain;h=44f951e1bc9c970fc63c16f8a2c437ca78e65eb7;p=spd%2Fsensor-hub.git

Switched to a python serial port monitor which updates mongodb on the fly.

The database is now directly updated by the serial monitor which is using
twisted to read the serial lines. Timestamps and values are now numerical
and not stored in the db as strings so minor change in the wsgi to handle
that.
---

diff --git a/monitor.py b/monitor.py
new file mode 100755
index 0000000..6ed71ef
--- /dev/null
+++ b/monitor.py
@@ -0,0 +1,129 @@
+#!/usr/bin/python3
+#
+# curl --data "key=$KEY&field1=$Temp0&field2=$Pressure0&field3=$Lux" $URL
+
+from __future__ import print_function, division, absolute_import
+import sys, pymongo
+from twisted.internet import reactor, stdio
+from twisted.internet.serialport import SerialPort, PARITY_ODD, PARITY_NONE
+from twisted.protocols import basic
+from pymongo import MongoClient
+from os import path
+from configparser import ConfigParser
+
+__all__ = ['MonitorSensorHub','SensorProtocol']
+__version__ = '1.0.0'
+__author__ = 'Pat Thoyts <patthoyts@users.sourceforge.net>'
+__copyright__ = 'Copyright (c) 2016 Pat Thoyts'
+
+class MonitorSensorHub():
+    """Class to handle database updates for new data items read from
+    the sensor-hub over serial port.
+    Granularity sets the time in seconds between database updates. This
+    reduces the datarate as the sensor-hub issues an update every second.
+    """
+    def __init__(self, granularity = 60):
+        self.granularity = granularity
+        config = ConfigParser()
+        filename = path.join(path.dirname(path.realpath(__file__)), 'sensor-hub.config')
+        config.read(filename)
+        self.uri = config['database']['uri'].strip("\'")
+        client = MongoClient(self.uri)
+        self.db = client.sensorhub.sensorlog
+        self.recent = self.recent()
+
+    def opendb(self):
+        """Open the database"""
+
+    def recent(self):
+        """Get the timestamp of the most recent item in the database."""
+        try:
+            cursor = self.db.find().sort([("timestamp", pymongo.DESCENDING)]).limit(1)
+            last = dict(next(cursor))
+        except StopIteration:
+            last = {'timestamp': 0}
+        r = int(last['timestamp'])
+        return r
+
+    def update(self, item):
+        """Update the database with a new data item only if sufficient
+        time has passed since the last update, as set by the granularity
+        property."""
+        t = int(item['timestamp'])
+        if t > (self.recent + self.granularity):
+            self.recent = t
+            result = self.db.insert_one(item)
+            print("{0} {1}".format(result.inserted_id, self.recent), flush=True)
+
+class SensorProtocol(basic.LineReceiver):
+    """Read input from the sensor-hub and parse each line into a
+    dictionary of values.
+    The temperature is recorded per sensor but the humidity and pressure
+    are reported singly for the hub.
+    """
+    delimiter = b'\r\n' # default delimiter is crlf
+    def __init__(self, monitor):
+        self.monitor = monitor
+        self.log = open('/tmp/sensor-hub.log', 'a+')
+        super().__init__()
+
+    def lineReceived(self, data):
+        try:
+            line = data.decode('ascii').strip()
+            if len(line) > 0:
+                print(line, file=self.log, flush=True)
+                if not line[0] == '#':
+                    item = self.parse(line)
+                    self.monitor.update(item)
+        except Exception as ex:
+            print("error: " + repr(ex), file=sys.stderr)
+
+    def parse(self, data):
+        item = dict(name='spd-office')
+        try:
+            if not data[0] == '#':
+                sensors = []
+                humidity = 0.0
+                pressure = 0.0
+                parts = [x.rstrip(" ]\r\n") for x in data.split('[')][1:]
+                for part in parts:
+                    values = part.split()
+                    sensor = {'id': 'office' + str(values[0])}
+                    #if len(values) > 1: # timestamp
+                    #    #sensor['timestamp'] = values[1]
+                    if len(values) > 2:                             # temperature
+                        sensor['value'] = float(values[2])
+                    if len(values) > 3 and float(values[3]) != 0.0: # humidity
+                        item['humidity'] = float(values[3])
+                    if len(values) > 4 and float(values[4]) != 0.0: # pressure
+                        item['pressure'] = float(values[4])
+                    sensors.append(sensor)
+                item['sensors'] = sensors
+                item['timestamp'] = int(data.split()[0])
+        except IndexError:
+            pass
+        return item
+
+def main(argv = None):
+    """Monitor the serial port for lines from the sensor-hub and update
+    the mongodb database and notify thingspeak at reduced time-rates.
+    """
+    port,baud = r'/dev/ttyUSB0',57600
+    if argv is None:
+        argv = sys.argv
+    if len(argv) > 1:
+        port = argv[1]
+    if len(argv) > 2:
+        baud = argv[2]
+    monitor = MonitorSensorHub()
+    serial = SerialPort(SensorProtocol(monitor), port, reactor, baudrate=baud)
+    serial._serial.parity = PARITY_ODD # work around pyserial issue #30
+    serial._serial.parity = PARITY_NONE
+    try:
+        reactor.run()
+    except KeyboardInterrupt:
+        reactor.stop()
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/sensor-hub.conf b/sensor-hub.conf
index 76f562c..145dad4 100644
--- a/sensor-hub.conf
+++ b/sensor-hub.conf
@@ -8,6 +8,6 @@ respawn
 
 script
   [ ! -f /etc/default/sensor-hub ] || . /etc/default/sensor-hub
-  exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/tclsh /home/pat/sensor-hub/monitor.tcl /dev/ttyUSB0 57600 > /tmp/sensor-hub.log"
+  #exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/tclsh /home/pat/sensor-hub/monitor.tcl /dev/ttyUSB0 57600 > /tmp/sensor-hub.log"
+  exec su pat -c "/usr/bin/env http_proxy=http://localhost:3128 https_proxy=http://localhost:3128 /usr/bin/python3 /opt/django/wsgi-scripts/sensor-hub/monitor.py /dev/ttyUSB0 57600 >> /var/log/sensor-hub.log 2>&1"
 end script
-
diff --git a/sensor-hub.wsgi b/sensor-hub.wsgi
index 4f27a8f..2f24f7a 100755
--- a/sensor-hub.wsgi
+++ b/sensor-hub.wsgi
@@ -91,9 +91,9 @@ class SensorHubService():
         uri = cherrypy.request.app.config['database']['uri']
         client = MongoClient(uri)
         db = client.sensorhub.sensorlog
-        selector = [{"timestamp": {"$gt": str(when)}}]
+        selector = [{"timestamp": {"$gt": int(when)}}]
         if not until is None:
-            selector.append({"timestamp": {"$lt": str(until)}})
+            selector.append({"timestamp": {"$lt": int(until)}})
         cursor = db.find({"$and": selector}).sort([("timestamp", pymongo.ASCENDING)])
         return RemoveKeyIterable(cursor, '_id')
 
diff --git a/sensordata.py b/sensordata.py
index 4c01425..79e3cd7 100644
--- a/sensordata.py
+++ b/sensordata.py
@@ -9,6 +9,7 @@ from __future__ import print_function, absolute_import, division
 import sys, re, json, pymongo, unittest
 from pymongo import MongoClient
 from configparser import ConfigParser
+from os import path
 from timeit import timeit
 
 __all__ = ['SensorData','SensorDataIterator']
@@ -38,12 +39,12 @@ class SensorDataIterator():
                 raise StopIteration()
             if line.startswith('#') or line.startswith('!'):
                 continue
-            timestamp = line.split()[0]
             m = self.re.search(line)
             if m:
                 sensors = []
                 for sensor,value in zip(['office1','office2','office3'], m.group('T').split()):
-                    sensors.append({'id': sensor, 'value': value })
+                    sensors.append({'id': sensor, 'value': float(value) })
+                timestamp = int(line.split()[0])
                 item = dict(timestamp=timestamp, name='spd-office', sensors=sensors)
                 humidity = m.group('H').split()
                 if len(humidity) > 0:
@@ -161,7 +162,8 @@ def recent(*args, **kwargs):
 
 def opendb():
     config = ConfigParser()
-    config.read('sensor-hub.config')
+    filename = path.join(path.dirname(path.realpath(__file__)), 'sensor-hub.config')
+    config.read(filename)
     uri = config['database']['uri'].strip("\'")
     client = MongoClient(uri)
     return client