bin/radmonAgent.py
b528647b
 #!/usr/bin/python -u
c467ff8b
 # The -u option above turns off block buffering of python output. This 
 # assures that each error message gets individually printed to the log file.
b528647b
 #
 # Module: radmonAgent.py
 #
4a2d49ab
 # Description: This module acts as an agent between the radiation monitoring
 # device and the Internet web server.  The agent periodically sends an http
 # request to the radiation monitoring device and processes the response from
 # the device and performs a number of operations:
207514f1
 #     - conversion of data itemsq
b528647b
 #     - update a round robin (rrdtool) database with the radiation data
 #     - periodically generate graphic charts for display in html documents
 #     - forward the radiation data to other services
4a2d49ab
 #     - write the processed weather data to a JSON file for use by html
 #       documents
b528647b
 #
18691b9a
 # Copyright 2018 Jeff Owrey
b528647b
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation, either version 3 of the License, or
 #    (at your option) any later version.
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see http://www.gnu.org/license.
 #
 # Revision History
4a2d49ab
 #   * v20 released 15 Sep 2015 by J L Owrey; first release
 #   * v21 released 27 Nov 2017 by J L Owrey; bug fixes; updates
0a313e87
 #   * v22 released 03 Mar 2018 by J L Owrey; improved code readability;
 #         improved radmon device offline status handling
b528647b
 #
58737d5f
 
1050ddd1
 _MIRROR_SERVER = False
4a2d49ab
 
58737d5f
 import os
b528647b
 import urllib2
f7a9cacf
 import sys
 import signal
b528647b
 import subprocess
 import multiprocessing
58737d5f
 import time
 import calendar
 
 _USER = os.environ['USER']
 
c467ff8b
    ### DEFAULT RADIATION MONITOR URL ###
58737d5f
 
1050ddd1
 # ip address of radiation monitoring device
18691b9a
 _DEFAULT_RADIATION_MONITOR_URL = "{your radiation monitor url}"
1050ddd1
 # url if this is a mirror server
18691b9a
 _PRIMARY_SERVER_URL = "{your primary server url}"
b528647b
 
66d7b126
     ### FILE AND FOLDER LOCATIONS ###
 
4a2d49ab
 # folder for containing dynamic data objects
1050ddd1
 _DOCROOT_PATH = "/home/%s/public_html/radmon/" % _USER
4a2d49ab
 # folder for charts and output data file
1050ddd1
 _CHARTS_DIRECTORY = _DOCROOT_PATH + "dynamic/"
 # location of data input file
207514f1
 _INPUT_DATA_FILE = _DOCROOT_PATH + "dynamic/radmonInputData.dat"
4a2d49ab
 # location of data output file
1050ddd1
 _OUTPUT_DATA_FILE = _DOCROOT_PATH + "dynamic/radmonOutputData.js"
c467ff8b
 # database that stores weather data
 _RRD_FILE = "/home/%s/database/radmonData.rrd" % _USER
66d7b126
 
     ### GLOBAL CONSTANTS ###
f7a9cacf
 # max number of failed data requests allowed
 _MAX_FAILED_DATA_REQUESTS = 2
4a2d49ab
 # interval in seconds between data requests to radiation monitor
 _DEFAULT_DATA_REQUEST_INTERVAL = 5
 # defines how often the charts get updated in seconds
 _CHART_UPDATE_INTERVAL = 300
 # defines how often the database gets updated
 _DATABASE_UPDATE_INTERVAL = 30
 # number seconds to wait for a response to HTTP request
 _HTTP_REQUEST_TIMEOUT = 3
f7a9cacf
 # standard chart width in pixels
98c8ae93
 _CHART_WIDTH = 600
f7a9cacf
 # standard chart height in pixels
98c8ae93
 _CHART_HEIGHT = 150
b528647b
 
66d7b126
    ### GLOBAL VARIABLES ###
 
4a2d49ab
 # turn on or off of verbose debugging information
b528647b
 debugOption = False
4a2d49ab
 # online status of radiation monitor
f555f41f
 radiationMonitorOnline = True
4a2d49ab
 # number of unsuccessful http requests
207514f1
 failedDataRequestCount = 0
4a2d49ab
 # status of reset command to radiation monitor
 remoteDeviceReset = False
1050ddd1
 # ip address of radiation monitor
207514f1
 radiationMonitorUrl = _DEFAULT_RADIATION_MONITOR_URL
4a2d49ab
 # web update frequency
 dataRequestInterval = _DEFAULT_DATA_REQUEST_INTERVAL
0979d8dd
 
b528647b
   ###  PRIVATE METHODS  ###
 
 def getTimeStamp():
     """
     Sets the error message time stamp to the local system time.
     Parameters: none
     Returns string containing the time stamp.
     """
f555f41f
     return time.strftime( "%m/%d/%Y %T", time.localtime() )
b528647b
 ##end def
 
0979d8dd
 def setOfflineStatus(dData):
     """Set the status of the the upstream device to "offline" and sends
b528647b
        blank data to the downstream clients.
0979d8dd
        Parameters:
            dData - dictionary object containing weather data
b528647b
        Returns nothing.
     """
207514f1
     global radiationMonitorOnline, failedDataRequestCount
f555f41f
 
207514f1
     if os.path.exists(_INPUT_DATA_FILE):
         os.remove(_INPUT_DATA_FILE)
f555f41f
 
207514f1
     if failedDataRequestCount < _MAX_FAILED_DATA_REQUESTS - 1:
         failedDataRequestCount += 1
f555f41f
         return
0979d8dd
 
     # If the radiation monitor was previously online, then send a message
     # that we are now offline.
f555f41f
     if radiationMonitorOnline:
         print "%s: radiation monitor offline" % getTimeStamp()
         radiationMonitorOnline = False
0a313e87
         if os.path.exists(_OUTPUT_DATA_FILE):
             os.remove(_OUTPUT_DATA_FILE)
b528647b
     return
 ##end def
 
f7a9cacf
 def signal_term_handler(signal, frame):
     """Send message to log when process killed
        Parameters: signal, frame - sigint parameters
        Returns: nothing
     """
     print '%s terminating radmon agent process' % \
               (getTimeStamp())
     if os.path.exists(_OUTPUT_DATA_FILE):
         os.remove(_OUTPUT_DATA_FILE)
     if os.path.exists(_INPUT_DATA_FILE):
         os.remove(_INPUT_DATA_FILE)
     sys.exit(0)
 ##end def
 
b528647b
   ###  PUBLIC METHODS  ###
 
f555f41f
 def getRadiationData():
0979d8dd
     """Send http request to radiation monitoring device.  The response
        from the device contains the radiation data.  The data is formatted
        as an html document.
     Parameters: 
f555f41f
         radiationMonitorUrl - url of radiation monitoring device
0979d8dd
         HttpRequesttimeout - how long to wait for device
                              to respond to http request
     Returns a string containing the radiation data, or None if
     not successful.
     """
207514f1
     global radiationMonitorOnline, failedDataRequestCount, \
4a2d49ab
            remoteDeviceReset
 
1050ddd1
     if _MIRROR_SERVER:
         sUrl = _PRIMARY_SERVER_URL
     else:
c467ff8b
         sUrl = radiationMonitorUrl
         if remoteDeviceReset:
             sUrl += "/reset"
         else:
             sUrl += "/rdata"
0979d8dd
 
     try:
4a2d49ab
         conn = urllib2.urlopen(sUrl, timeout=_HTTP_REQUEST_TIMEOUT)
f555f41f
 
         # Format received data into a single string.
         content = ""
         for line in conn:
             content += line.strip()
         del conn
 
0979d8dd
     except Exception, exError:
         # If no response is received from the device, then assume that
         # the device is down or unavailable over the network.  In
         # that case set the status of the device to offline.
         if debugOption:
f555f41f
             print "http error: %s" % exError
0979d8dd
         return None
 
     # If the radiation monitor was previously offline, then send a message
     # that we are now online.
f555f41f
     if not radiationMonitorOnline:
         print "%s radiation monitor online" % getTimeStamp()
         radiationMonitorOnline = True
 
207514f1
     failedDataRequestCount = 0
0979d8dd
     return content
b528647b
 ##end def
 
0979d8dd
 def parseDataString(sData, dData):
b528647b
     """Parse the radiation data JSON string from the radiation 
        monitoring device into its component parts.  
        Parameters:
            sData - the string containing the data to be parsed
            dData - a dictionary object to contain the parsed data items
        Returns true if successful, false otherwise.
     """
     try:
0979d8dd
         sTmp = sData[2:-2]
b528647b
         lsTmp = sTmp.split(',')
     except Exception, exError:
0979d8dd
         print "%s parseDataString: %s" % (getTimeStamp(), exError)
b528647b
         return False
 
     # Load the parsed data into a dictionary for easy access.
0979d8dd
     for item in lsTmp:
b528647b
         if "=" in item:
             dData[item.split('=')[0]] = item.split('=')[1]
0979d8dd
     dData['status'] = 'online'
b528647b
 
1050ddd1
     if len(dData) != 6:
c467ff8b
         print "%s parse failed: corrupted data string" % getTimeStamp()
1050ddd1
         return False;
 
b528647b
     return True
 ##end def
 
0979d8dd
 def convertData(dData):
b528647b
     """Convert individual radiation data items as necessary.
        Parameters:
            lsData - a list object containing the radiation data
            dData - a dictionary object containing the radiation data
        Returns true if successful, false otherwise.
     """
     result = True
  
     try:
c467ff8b
         # Convert the UTC timestamp provided by the radiation monitoring
         # device to epoch local time in seconds.
b528647b
         ts_utc = time.strptime(dData['UTC'], "%H:%M:%S %m/%d/%Y")
c467ff8b
         epoch_local_sec = calendar.timegm(ts_utc)
         dData['ELT'] = epoch_local_sec
 
         # Uncomment the code line below to use a timestamp generated by the
         # requesting server (this) instead of the timestamp provided by the
         # radiation monitoring device.  Using the server generated timestamp
         # prevents errors that occur when the radiation monitoring device
         # fails to synchronize with a valid NTP time server.
         #dData['ELT'] = time.time()
4a2d49ab
         
92145152
         dData['Mode'] = dData['Mode'].lower()
4a2d49ab
  
         dData['uSvPerHr'] = float(dData.pop('uSv/hr'))
         
         dData['CPM'] = int(dData.pop('CPM'))
 
207514f1
         dData['CPS'] = int(dData.pop('CPS'))
 
0979d8dd
     except Exception, exError:
c467ff8b
         print "%s convert data failed: %s" % (getTimeStamp(), exError)
0979d8dd
         result = False
b528647b
 
     return result
 ##end def
 
0979d8dd
 def writeOutputDataFile(dData):
207514f1
     """Write radiation data items to a JSON formatted file for use by
        HTML documents.
b528647b
        Parameters:
            lsData - a list object containing the data to be written
                     to the JSON file
        Returns true if successful, false otherwise.
     """
0979d8dd
     # Set date to current time and data
c467ff8b
     dData['date'] = time.strftime("%m/%d/%Y %T", time.localtime(dData['ELT']))
b528647b
 
0979d8dd
     # Format the weather data as string using java script object notation.
     sData = '[{'
c467ff8b
     sData += "\"date\":\"%s\"," % dData['date']
f7a9cacf
     sData += "\"CPM\":\"%d\"," % dData['CPM']
     sData += "\"CPS\":\"%d\"," % dData['CPS']
     sData += "\"uSvPerHr\":\"%.2f\"," % dData['uSvPerHr']
c467ff8b
     sData += "\"Mode\":\"%s\"," % dData['Mode']
     sData += "\"status\":\"%s\"," % dData['status']
     sData = sData[:-1] + '}]\n'
b528647b
 
0979d8dd
     # Write the string to the output data file for use by html documents.
b528647b
     try:
         fc = open(_OUTPUT_DATA_FILE, "w")
66d7b126
         fc.write(sData)
b528647b
         fc.close()
     except Exception, exError:
0979d8dd
         print "%s writeOutputDataFile: %s" % (getTimeStamp(), exError)
b528647b
         return False
66d7b126
 
b528647b
     return True
 ## end def
 
1050ddd1
 def writeInputDataFile(sData):
207514f1
     """Write raw data from radiation monitor to file for use by mirror
        servers.
        Parameters:
            sData - a string object containing the data string from
                    the radiation monitor
        Returns true if successful, false otherwise.
     """
1050ddd1
     sData += "\n"
58737d5f
     try:
207514f1
         fc = open(_INPUT_DATA_FILE, "w")
58737d5f
         fc.write(sData)
         fc.close()
     except Exception, exError:
         print "%s writeOutputDataFile: %s" % (getTimeStamp(), exError)
         return False
 
     return True
 ##end def
 
b528647b
 def updateDatabase(dData):
     """
     Updates the rrdtool database by executing an rrdtool system command.
     Formats the command using the data extracted from the radiation
     monitor response.   
     Parameters: dData - dictionary object containing data items to be
                         written to the rr database file
     Returns true if successful, false otherwise.
     """
4a2d49ab
     global remoteDeviceReset
 
     # The RR database stores whole units, so convert uSv to Sv.
207514f1
     SvPerHr = dData['uSvPerHr'] * 1.0E-06 
b528647b
 
     # Create the rrdtool update command.
     strCmd = "rrdtool update %s %s:%s:%s" % \
c467ff8b
                        (_RRD_FILE, dData['ELT'], dData['CPM'], SvPerHr)
207514f1
     if debugOption and False:
f555f41f
         print "%s" % strCmd # DEBUG
b528647b
 
     # Run the command as a subprocess.
     try:
         subprocess.check_output(strCmd, shell=True,  \
                              stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError, exError:
         print "%s: rrdtool update failed: %s" % \
207514f1
                     (getTimeStamp(), exError.output)
4a2d49ab
         if exError.output.find("illegal attempt to update using time") > -1:
             remoteDeviceReset = True
207514f1
             print "%s: rebooting radiation monitor" % (getTimeStamp())
b528647b
         return False
207514f1
     else:
         if debugOption:
             print 'database update sucessful'
         return True
b528647b
 ##end def
 
f555f41f
 def createGraph(fileName, dataItem, gLabel, gTitle, gStart,
                 lower, upper, addTrend, autoScale):
b528647b
     """Uses rrdtool to create a graph of specified weather data item.
        Parameters:
            fileName - name of graph image file
            dataItem - data item to be graphed
f555f41f
            gLabel - string containing a graph label for the data item
            gTitle - string containing a title for the graph
            lower - lower bound for graph ordinate #NOT USED
            upper - upper bound for graph ordinate #NOT USED
            addTrend - 0, show only graph data
                       1, show only a trend line
                       2, show a trend line and the graph data
            autoScale - if True, then use vertical axis auto scaling
                (lower and upper parameters are ignored), otherwise use
                lower and upper parameters to set vertical axis scale
b528647b
        Returns true if successful, false otherwise.
     """
4a2d49ab
     gPath = _CHARTS_DIRECTORY + fileName + ".png"
98c8ae93
     trendWindow = { 'end-1day': 7200,
                     'end-4weeks': 172800,
                     'end-12months': 604800 }
  
     # Format the rrdtool graph command.
 
     # Set chart start time, height, and width.
     strCmd = "rrdtool graph %s -a PNG -s %s -e now -w %s -h %s " \
              % (gPath, gStart, _CHART_WIDTH, _CHART_HEIGHT)
    
f555f41f
     # Set the range and scaling of the chart y-axis.
98c8ae93
     if lower < upper:
f555f41f
         strCmd  +=  "-l %s -u %s -r " % (lower, upper)
     elif autoScale:
         strCmd += "-A "
     strCmd += "-Y "
98c8ae93
 
     # Set the chart ordinate label and chart title. 
     strCmd += "-v %s -t %s " % (gLabel, gTitle)
f555f41f
  
98c8ae93
     # Show the data, or a moving average trend line over
     # the data, or both.
f555f41f
     strCmd += "DEF:dSeries=%s:%s:LAST " % (_RRD_FILE, dataItem)
     if addTrend == 0:
         strCmd += "LINE1:dSeries#0400ff "
     elif addTrend == 1:
         strCmd += "CDEF:smoothed=dSeries,%s,TREND LINE3:smoothed#ff0000 " \
                   % trendWindow[gStart]
     elif addTrend == 2:
         strCmd += "LINE1:dSeries#0400ff "
         strCmd += "CDEF:smoothed=dSeries,%s,TREND LINE3:smoothed#ff0000 " \
                   % trendWindow[gStart]
      
207514f1
     if debugOption and False:
b528647b
         print "%s\n" % strCmd # DEBUG
     
98c8ae93
     # Run the formatted rrdtool command as a subprocess.
b528647b
     try:
98c8ae93
         result = subprocess.check_output(strCmd, \
                      stderr=subprocess.STDOUT,   \
b528647b
                      shell=True)
     except subprocess.CalledProcessError, exError:
98c8ae93
         print "rrdtool graph failed: %s" % (exError.output)
b528647b
         return False
 
     if debugOption:
         print "rrdtool graph: %s" % result
     return True
98c8ae93
 
b528647b
 ##end def
 
f555f41f
 def generateGraphs():
     """Generate graphs for display in html documents.
        Parameters: none
        Returns nothing.
     """
     autoScale = False
 
58737d5f
     createGraph('24hr_cpm', 'CPM', 'counts\ per\ minute', 
f555f41f
                 'CPM\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
58737d5f
     createGraph('24hr_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
58737d5f
     createGraph('4wk_cpm', 'CPM', 'counts\ per\ minute',
f555f41f
                 'CPM\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
58737d5f
     createGraph('4wk_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
58737d5f
     createGraph('12m_cpm', 'CPM', 'counts\ per\ minute',
f555f41f
                 'CPM\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
58737d5f
     createGraph('12m_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
 ##end def
 
b528647b
 def getCLarguments():
     """Get command line arguments.  There are three possible arguments
           -d turns on debug mode
           -t sets the radiation device query interval
           -u sets the url of the radiation monitoring device
        Returns nothing.
     """
f555f41f
     global debugOption, dataRequestInterval, radiationMonitorUrl
b528647b
 
     index = 1
     while index < len(sys.argv):
         if sys.argv[index] == '-d':
             debugOption = True
         elif sys.argv[index] == '-t':
             try:
f555f41f
                 dataRequestInterval = abs(int(sys.argv[index + 1]))
b528647b
             except:
                 print "invalid polling period"
                 exit(-1)
             index += 1
         elif sys.argv[index] == '-u':
f555f41f
             radiationMonitorUrl = sys.argv[index + 1]
b528647b
             index += 1
         else:
             cmd_name = sys.argv[0].split('/')
f555f41f
             print "Usage: %s [-d] [-t seconds] [-u url}" % cmd_name[-1]
b528647b
             exit(-1)
         index += 1
 ##end def
 
 def main():
4a2d49ab
     """Handles timing of events and acts as executive routine managing
        all other functions.
b528647b
        Parameters: none
        Returns nothing.
     """
f7a9cacf
     signal.signal(signal.SIGTERM, signal_term_handler)
 
     print '%s starting up radmon agent process' % \
                   (getTimeStamp())
66d7b126
 
4a2d49ab
     # last time output JSON file updated
     lastDataRequestTime = -1
     # last time charts generated
     lastChartUpdateTime = - 1
     # last time the rrdtool database updated
     lastDatabaseUpdateTime = -1
 
b528647b
     ## Get command line arguments.
     getCLarguments()
 
f555f41f
     ## Exit with error if rrdtool database does not exist.
b528647b
     if not os.path.exists(_RRD_FILE):
f7a9cacf
         print 'rrdtool database does not exist\n' \
               'use createWeatherRrd script to ' \
               'create rrdtool database\n'
b528647b
         exit(1)
  
     ## main loop
     while True:
 
f555f41f
         currentTime = time.time() # get current time in seconds
b528647b
 
f555f41f
         # Every web update interval request data from the radiation
         # monitor and process the received data.
         if currentTime - lastDataRequestTime > dataRequestInterval:
             lastDataRequestTime = currentTime
207514f1
             dData = {}
b528647b
             result = True
 
             # Get the data string from the device.
f555f41f
             sData = getRadiationData()
b528647b
             if sData == None:
0979d8dd
                 setOfflineStatus(dData)
b528647b
                 result = False
 
             # If successful parse the data.
             if result:
0979d8dd
                 result = parseDataString(sData, dData)
b528647b
 
             # If parsing successful, convert the data.
             if result:
0979d8dd
                 result = convertData(dData)
66d7b126
 
207514f1
             # If conversion successful, write data to data files.
b528647b
             if result:
1050ddd1
                 writeInputDataFile(sData)
0979d8dd
                 writeOutputDataFile(dData)
58737d5f
                 if debugOption:
                     print "http request successful"
b528647b
 
cc0b13e1
                 # At the rrdtool database update interval, update the database.
4a2d49ab
                 if currentTime - lastDatabaseUpdateTime > \
                         _DATABASE_UPDATE_INTERVAL:   
cc0b13e1
                     lastDatabaseUpdateTime = currentTime
                     ## Update the round robin database with the parsed data.
                     result = updateDatabase(dData)
b528647b
 
         # At the chart generation interval, generate charts.
         if currentTime - lastChartUpdateTime > _CHART_UPDATE_INTERVAL:
             lastChartUpdateTime = currentTime
             p = multiprocessing.Process(target=generateGraphs, args=())
             p.start()
 
         # Relinquish processing back to the operating system until
         # the next update interval.
 
         elapsedTime = time.time() - currentTime
         if debugOption:
f555f41f
             print "processing time: %6f sec\n" % elapsedTime
         remainingTime = dataRequestInterval - elapsedTime
         if remainingTime > 0.0:
b528647b
             time.sleep(remainingTime)
     ## end while
f555f41f
     return
b528647b
 ## end def
 
 if __name__ == '__main__':
207514f1
     try:
         main()
     except KeyboardInterrupt:
f7a9cacf
         print '\n%s terminating radmon agent process' % \
               (getTimeStamp())
207514f1
         if os.path.exists(_OUTPUT_DATA_FILE):
             os.remove(_OUTPUT_DATA_FILE)
         if os.path.exists(_INPUT_DATA_FILE):
             os.remove(_INPUT_DATA_FILE)