bin/radmonAgent.py
d2bf8515
 #!/usr/bin/python3 -u
c467ff8b
 # The -u option above turns off block buffering of python output. This 
 # assures that each error message gets individually printed to the log file.
b528647b
 #
 # Module: radmonAgent.py
 #
4a2d49ab
 # Description: This module acts as an agent between the radiation monitoring
bac3cd9f
 # device and Internet web services.  The agent periodically sends an http
4a2d49ab
 # request to the radiation monitoring device and processes the response from
 # the device and performs a number of operations:
bac3cd9f
 #     - conversion of data items
b528647b
 #     - update a round robin (rrdtool) database with the radiation data
 #     - periodically generate graphic charts for display in html documents
e50a3c08
 #     - write the processed radmon data to a JSON file for use by html
4a2d49ab
 #       documents
b528647b
 #
0bfe4f11
 # Copyright 2015 Jeff Owrey
b528647b
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation, either version 3 of the License, or
 #    (at your option) any later version.
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see http://www.gnu.org/license.
 #
 # Revision History
4a2d49ab
 #   * v20 released 15 Sep 2015 by J L Owrey; first release
 #   * v21 released 27 Nov 2017 by J L Owrey; bug fixes; updates
0a313e87
 #   * v22 released 03 Mar 2018 by J L Owrey; improved code readability;
 #         improved radmon device offline status handling
68be30fb
 #   * v23 released 16 Nov 2018 by J L Owrey: improved fault handling
 #         and data conversion
d2bf8515
 #   * v24 released 14 Jun 2021 by J L Owrey; minor revisions
24d02d7b
 #   * v25 released 9 Jul 2021 by J L Owrey; improved handling of
 #         monitor status function
d2bf8515
 #
e50a3c08
 #2345678901234567890123456789012345678901234567890123456789012345678901234567890
58737d5f
 
 import os
f7a9cacf
 import sys
 import signal
b528647b
 import multiprocessing
58737d5f
 import time
 import calendar
d2bf8515
 import json
 from urllib.request import urlopen
a040909b
 import rrdbase
d2bf8515
 
    ### ENVIRONMENT ###
58737d5f
 
 _USER = os.environ['USER']
d2bf8515
 _SERVER_MODE = "primary"
 _USE_RADMON_TIMESTAMP = True
58737d5f
 
c467ff8b
    ### DEFAULT RADIATION MONITOR URL ###
58737d5f
 
d2bf8515
 _DEFAULT_RADIATION_MONITOR_URL = \
8f3991a5
     "{your radiation monitor URL}"
b528647b
 
66d7b126
     ### FILE AND FOLDER LOCATIONS ###
 
4a2d49ab
 # folder for containing dynamic data objects
1050ddd1
 _DOCROOT_PATH = "/home/%s/public_html/radmon/" % _USER
4a2d49ab
 # folder for charts and output data file
1050ddd1
 _CHARTS_DIRECTORY = _DOCROOT_PATH + "dynamic/"
4a2d49ab
 # location of data output file
d2bf8515
 _OUTPUT_DATA_FILE = _DOCROOT_PATH + "dynamic/radmonData.js"
e50a3c08
 # database that stores radmon data
c467ff8b
 _RRD_FILE = "/home/%s/database/radmonData.rrd" % _USER
66d7b126
 
     ### GLOBAL CONSTANTS ###
bac3cd9f
 
a040909b
 # maximum number of failed data requests allowed
 _MAX_FAILED_DATA_REQUESTS = 2
 # maximum number of http request retries  allowed
 _MAX_HTTP_RETRIES = 5
 # delay time between http request retries
 _HTTP_RETRY_DELAY = 1.119
d2bf8515
 # interval in seconds between data requests
a040909b
 _DEFAULT_DATA_REQUEST_INTERVAL = 5
4a2d49ab
 # number seconds to wait for a response to HTTP request
 _HTTP_REQUEST_TIMEOUT = 3
d2bf8515
 
 # interval in seconds between database updates
 _DATABASE_UPDATE_INTERVAL = 30
 # interval in seconds between chart updates
 _CHART_UPDATE_INTERVAL = 300
f7a9cacf
 # standard chart width in pixels
98c8ae93
 _CHART_WIDTH = 600
f7a9cacf
 # standard chart height in pixels
98c8ae93
 _CHART_HEIGHT = 150
b528647b
 
66d7b126
    ### GLOBAL VARIABLES ###
 
4a2d49ab
 # turn on or off of verbose debugging information
d2bf8515
 verboseMode = False
 debugMode = False
a040909b
 reportUpdateFails = False
bac3cd9f
 
 # The following two items are used for detecting system faults
 # and radiation monitor online or offline status.
 # count of failed attempts to get data from radiation monitor
0bfe4f11
 failedUpdateCount = 0
a040909b
 httpRetries = 0
24d02d7b
 radmonOnline = False
bac3cd9f
 
4a2d49ab
 # status of reset command to radiation monitor
 remoteDeviceReset = False
1050ddd1
 # ip address of radiation monitor
207514f1
 radiationMonitorUrl = _DEFAULT_RADIATION_MONITOR_URL
4a2d49ab
 # web update frequency
 dataRequestInterval = _DEFAULT_DATA_REQUEST_INTERVAL
0979d8dd
 
a040909b
 # rrdtool database interface handler
 rrdb = None
 
b528647b
   ###  PRIVATE METHODS  ###
 
 def getTimeStamp():
     """
bac3cd9f
     Set the error message time stamp to the local system time.
b528647b
     Parameters: none
bac3cd9f
     Returns: string containing the time stamp
b528647b
     """
f555f41f
     return time.strftime( "%m/%d/%Y %T", time.localtime() )
a040909b
 ## end def
b528647b
 
0bfe4f11
 def setStatusToOffline():
bac3cd9f
     """Set the detected status of the radiation monitor to
        "offline" and inform downstream clients by removing input
        and output data files.
        Parameters: none
        Returns: nothing
b528647b
     """
d2bf8515
     global radmonOnline
f555f41f
 
e50a3c08
     # Inform downstream clients by removing output data file.
0bfe4f11
     if os.path.exists(_OUTPUT_DATA_FILE):
        os.remove(_OUTPUT_DATA_FILE)
bac3cd9f
     # If the radiation monitor was previously online, then send
     # a message that we are now offline.
d2bf8515
     if radmonOnline:
         print('%s radiation monitor offline' % getTimeStamp())
     radmonOnline = False
a040909b
 ## end def
b528647b
 
0bfe4f11
 def terminateAgentProcess(signal, frame):
bac3cd9f
     """Send a message to log when the agent process gets killed
        by the operating system.  Inform downstream clients
        by removing input and output data files.
        Parameters:
            signal, frame - dummy parameters
f7a9cacf
        Returns: nothing
     """
a040909b
     # Inform downstream clients by removing output data file.
     if os.path.exists(_OUTPUT_DATA_FILE):
        os.remove(_OUTPUT_DATA_FILE)
d2bf8515
     print('%s terminating radmon agent process' % \
               (getTimeStamp()))
f7a9cacf
     sys.exit(0)
a040909b
 ## end def
f7a9cacf
 
b528647b
   ###  PUBLIC METHODS  ###
 
d2bf8515
 def getRadiationData(dData):
bac3cd9f
     """Send http request to radiation monitoring device.  The
        response from the device contains the radiation data as
        unformatted ascii text.
        Parameters: none 
        Returns: a string containing the radiation data if successful,
                 or None if not successful
0979d8dd
     """
a040909b
     global httpRetries
8e0b939c
 
a040909b
     sUrl = radiationMonitorUrl
8e0b939c
     if remoteDeviceReset:
         sUrl += "/reset" # reboot the radiation monitor
1050ddd1
     else:
8e0b939c
         sUrl += "/rdata" # request data from the monitor
0979d8dd
 
     try:
d2bf8515
         currentTime = time.time()
         response = urlopen(sUrl, timeout=_HTTP_REQUEST_TIMEOUT)
24d02d7b
         requestTime = time.time() - currentTime
f555f41f
 
d2bf8515
         content = response.read().decode('utf-8')
         content = content.replace('\n', '')
         content = content.replace('\r', '')
         if content == "":
             raise Exception("empty response")
f555f41f
 
d2bf8515
     except Exception as exError:
0979d8dd
         # If no response is received from the device, then assume that
         # the device is down or unavailable over the network.  In
bac3cd9f
         # that case return None to the calling function.
a040909b
         httpRetries += 1
 
         if reportUpdateFails:
             print("%s " % getTimeStamp(), end='')
         if reportUpdateFails or verboseMode:
             print("http request failed (%d): %s" % \
                 (httpRetries, exError))
 
         if httpRetries > _MAX_HTTP_RETRIES:
             httpRetries = 0
             return False
         else:
             time.sleep(_HTTP_RETRY_DELAY)
             return getRadiationData(dData)
     ## end try
d2bf8515
 
     if debugMode:
         print(content)
24d02d7b
     if verboseMode:
         print("http request successful: %.4f sec" % requestTime)
d2bf8515
     
a040909b
     httpRetries = 0
d2bf8515
     dData['content'] = content
     return True
a040909b
 ## end def
b528647b
 
d2bf8515
 def parseDataString(dData):
     """Parse the data string returned by the radiation monitor
        into its component parts.
b528647b
        Parameters:
d2bf8515
             dData - a dictionary object to contain the parsed data items
bac3cd9f
        Returns: True if successful, False otherwise
b528647b
     """
d2bf8515
     # Example radiation monitor data string
     # $,UTC=17:09:33 6/22/2021,CPS=0,CPM=26,uSv/hr=0.14,Mode=SLOW,#
     
b528647b
     try:
d2bf8515
         sData = dData.pop('content')
         lData = sData[2:-2].split(',')
     except Exception as exError:
         print("%s parseDataString: %s" % (getTimeStamp(), exError))
b528647b
         return False
 
24d02d7b
     # Verfy the expected number of data items have been received.
     if len(lData) != 5:
         print("%s parse failed: corrupted data string" % getTimeStamp())
         return False;
 
b528647b
     # Load the parsed data into a dictionary for easy access.
d2bf8515
     for item in lData:
b528647b
         if "=" in item:
             dData[item.split('=')[0]] = item.split('=')[1]
24d02d7b
 
d2bf8515
     # Add status to dictionary object
0979d8dd
     dData['status'] = 'online'
24d02d7b
     dData['serverMode'] = _SERVER_MODE
1050ddd1
 
b528647b
     return True
a040909b
 ## end def
b528647b
 
0979d8dd
 def convertData(dData):
b528647b
     """Convert individual radiation data items as necessary.
        Parameters:
            dData - a dictionary object containing the radiation data
bac3cd9f
        Returns: True if successful, False otherwise
b528647b
     """
     try:
e50a3c08
         if _USE_RADMON_TIMESTAMP:
             # Convert the UTC timestamp provided by the radiation monitoring
             # device to epoch local time in seconds.
             ts_utc = time.strptime(dData['UTC'], "%H:%M:%S %m/%d/%Y")
             epoch_local_sec = calendar.timegm(ts_utc)
         else:
             # Use a timestamp generated by the requesting server (this)
             # instead of the timestamp provided by the radiation monitoring
             # device.  Using the server generated timestamp prevents errors
             # that occur when the radiation monitoring device fails to
             # synchronize with a valid NTP time server.
a040909b
             epoch_local_sec = time.time()
d2bf8515
 
         dData['date'] = \
a040909b
             time.strftime("%m/%d/%Y %T", time.localtime(epoch_local_sec))      
d2bf8515
         dData['mode'] = dData.pop('Mode').lower()
a040909b
         dData['uSvPerHr'] = '%.2f' % float(dData['uSv/hr'])
         # The rrdtool database stores whole units, so convert uSv to Sv.
         dData['SvPerHr'] = float(dData.pop('uSv/hr')) * 1.0E-06
207514f1
 
d2bf8515
     except Exception as exError:
         print("%s data conversion failed: %s" % (getTimeStamp(), exError))
bac3cd9f
         return False
b528647b
 
bac3cd9f
     return True
a040909b
 ## end def
b528647b
 
d2bf8515
 def writeOutputFile(dData):
bac3cd9f
     """Write radiation data items to the output data file, formatted as 
d2bf8515
        a JSON file.  This file may then be accessed and used by
bac3cd9f
        by downstream clients, for instance, in HTML documents.
b528647b
        Parameters:
bac3cd9f
            dData - a dictionary object containing the data to be written
                    to the output data file
        Returns: True if successful, False otherwise
b528647b
     """
e50a3c08
     # Format the radmon data as string using java script object notation.
d2bf8515
     jsData = json.loads("{}")
     try:
24d02d7b
         for key in dData:
             jsData.update({key:dData[key]})
d2bf8515
         sData = "[%s]" % json.dumps(jsData)
     except Exception as exError:
         print("%s writeOutputFile: %s" % (getTimeStamp(), exError))
         return False
b528647b
 
d2bf8515
     if debugMode:
         print(sData)
8e0b939c
 
0979d8dd
     # Write the string to the output data file for use by html documents.
b528647b
     try:
         fc = open(_OUTPUT_DATA_FILE, "w")
66d7b126
         fc.write(sData)
b528647b
         fc.close()
d2bf8515
     except Exception as exError:
         print("%s writeOutputFile: %s" % (getTimeStamp(), exError))
b528647b
         return False
66d7b126
 
b528647b
     return True
 ## end def
 
24d02d7b
 def setRadmonStatus(updateSuccess):
     """Detect if radiation monitor is offline or not available on
        the network. After a set number of attempts to get data
        from the monitor set a flag that the radmon is offline.
        Parameters:
            updateSuccess - a boolean that is True if data request
                            successful, False otherwise
        Returns: nothing
     """
     global failedUpdateCount, radmonOnline
 
     if updateSuccess:
         failedUpdateCount = 0
         # Set status and send a message to the log if the device
         # previously offline and is now online.
         if not radmonOnline:
             print('%s radiation monitor online' % getTimeStamp())
             radmonOnline = True
         return
a040909b
     else:
         # The last attempt failed, so update the failed attempts
         # count.
         failedUpdateCount += 1
 
     if failedUpdateCount == _MAX_FAILED_DATA_REQUESTS:
24d02d7b
         # Max number of failed data requests, so set
         # device status to offline.
         setStatusToOffline()
a040909b
 ## end def
98c8ae93
 
a040909b
     ### GRAPH FUNCTIONS ###
b528647b
 
f555f41f
 def generateGraphs():
     """Generate graphs for display in html documents.
        Parameters: none
bac3cd9f
        Returns: nothing
f555f41f
     """
     autoScale = False
 
d2bf8515
     # past 24 hours
a040909b
     rrdb.createAutoGraph('24hr_cpm', 'CPM', 'counts\ per\ minute', 
f555f41f
                 'CPM\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
a040909b
     rrdb.createAutoGraph('24hr_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
d2bf8515
     # past 4 weeks
a040909b
     rrdb.createAutoGraph('4wk_cpm', 'CPM', 'counts\ per\ minute',
f555f41f
                 'CPM\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
a040909b
     rrdb.createAutoGraph('4wk_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
d2bf8515
     # past year
a040909b
     rrdb.createAutoGraph('12m_cpm', 'CPM', 'counts\ per\ minute',
f555f41f
                 'CPM\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
a040909b
     rrdb.createAutoGraph('12m_svperhr', 'SvperHr', 'Sv\ per\ hour',
f555f41f
                 'Sv/Hr\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
a040909b
 ## end def
f555f41f
 
b528647b
 def getCLarguments():
bac3cd9f
     """Get command line arguments.  There are four possible arguments
b528647b
           -d turns on debug mode
d2bf8515
           -v turns on verbose mode
b528647b
           -t sets the radiation device query interval
           -u sets the url of the radiation monitoring device
bac3cd9f
        Returns: nothing
b528647b
     """
d2bf8515
     global verboseMode, debugMode, dataRequestInterval, \
a040909b
            radiationMonitorUrl, reportUpdateFails
b528647b
 
     index = 1
     while index < len(sys.argv):
d2bf8515
         if sys.argv[index] == '-v':
             verboseMode = True
         elif sys.argv[index] == '-d':
             verboseMode = True
             debugMode = True
a040909b
         elif sys.argv[index] == '-r':
             reportUpdateFails = True
 
         # Update period and url options
         elif sys.argv[index] == '-p':
             try:
                 dataRequestInterval = abs(float(sys.argv[index + 1]))
             except:
                 print("invalid polling period")
                 exit(-1)
b528647b
             index += 1
         elif sys.argv[index] == '-u':
f555f41f
             radiationMonitorUrl = sys.argv[index + 1]
d2bf8515
             if radiationMonitorUrl.find('http://') < 0:
                 radiationMonitorUrl = 'http://' + radiationMonitorUrl
b528647b
             index += 1
         else:
             cmd_name = sys.argv[0].split('/')
a040909b
             print("Usage: %s [-d] [-p seconds] [-u url}" % cmd_name[-1])
b528647b
             exit(-1)
         index += 1
a040909b
 ## end def
b528647b
 
a040909b
 def setup():
4a2d49ab
     """Handles timing of events and acts as executive routine managing
        all other functions.
b528647b
        Parameters: none
bac3cd9f
        Returns: nothing
b528647b
     """
a040909b
     global rrdb
4a2d49ab
 
b528647b
     ## Get command line arguments.
     getCLarguments()
 
a040909b
     print('====================================================')
     print('%s starting up radmon agent process' % \
                   (getTimeStamp()))
 
f555f41f
     ## Exit with error if rrdtool database does not exist.
b528647b
     if not os.path.exists(_RRD_FILE):
d2bf8515
         print('rrdtool database does not exist\n' \
e50a3c08
               'use createRadmonRrd script to ' \
d2bf8515
               'create rrdtool database\n')
b528647b
         exit(1)
a040909b
 
     signal.signal(signal.SIGTERM, terminateAgentProcess)
     signal.signal(signal.SIGINT, terminateAgentProcess)
 
     # Define object for calling rrdtool database functions.
     rrdb = rrdbase.rrdbase( _RRD_FILE, _CHARTS_DIRECTORY, _CHART_WIDTH, \
                             _CHART_HEIGHT, verboseMode, debugMode )
 ## end def
 
 def loop():
      # last time output JSON file updated
     lastDataRequestTime = -1
     # last time charts generated
     lastChartUpdateTime = - 1
     # last time the rrdtool database updated
     lastDatabaseUpdateTime = -1
 
b528647b
     while True:
 
f555f41f
         currentTime = time.time() # get current time in seconds
b528647b
 
d2bf8515
         # Every data update interval request data from the radiation
f555f41f
         # monitor and process the received data.
         if currentTime - lastDataRequestTime > dataRequestInterval:
             lastDataRequestTime = currentTime
207514f1
             dData = {}
b528647b
 
             # Get the data string from the device.
d2bf8515
             result = getRadiationData(dData)
b528647b
 
             # If successful parse the data.
             if result:
d2bf8515
                 result = parseDataString(dData)
b528647b
 
             # If parsing successful, convert the data.
             if result:
0979d8dd
                 result = convertData(dData)
66d7b126
 
207514f1
             # If conversion successful, write data to data files.
b528647b
             if result:
d2bf8515
                 writeOutputFile(dData)
b528647b
 
d2bf8515
             # At the rrdtool database update interval, update the database.
             if result and (currentTime - lastDatabaseUpdateTime > \
                            _DATABASE_UPDATE_INTERVAL):   
                 lastDatabaseUpdateTime = currentTime
                 ## Update the round robin database with the parsed data.
a040909b
                 result = rrdb.updateDatabase(dData['date'], \
                              dData['CPM'], dData['SvPerHr'])
0bfe4f11
 
d2bf8515
             # Set the radmon status to online or offline depending on the
0bfe4f11
             # success or failure of the above operations.
d2bf8515
             setRadmonStatus(result)
0bfe4f11
 
b528647b
 
         # At the chart generation interval, generate charts.
         if currentTime - lastChartUpdateTime > _CHART_UPDATE_INTERVAL:
             lastChartUpdateTime = currentTime
             p = multiprocessing.Process(target=generateGraphs, args=())
             p.start()
 
         # Relinquish processing back to the operating system until
         # the next update interval.
 
         elapsedTime = time.time() - currentTime
d2bf8515
         if verboseMode:
             if result:
                 print("update successful: %6f sec\n"
                       % elapsedTime)
             else:
                 print("update failed: %6f sec\n"
                       % elapsedTime)
f555f41f
         remainingTime = dataRequestInterval - elapsedTime
         if remainingTime > 0.0:
b528647b
             time.sleep(remainingTime)
     ## end while
 ## end def
 
 if __name__ == '__main__':
a040909b
     setup()
     loop()
d2bf8515