#!/usr/bin/python3 -u # The -u option above turns off block buffering of python output. This # assures that each error message gets individually printed to the log file. # # Module: radmonAgent.py # # Description: This module acts as an agent between the radiation monitoring # device and Internet web services. The agent periodically sends an http # request to the radiation monitoring device and processes the response from # the device and performs a number of operations: # - conversion of data items # - update a round robin (rrdtool) database with the radiation data # - periodically generate graphic charts for display in html documents # - write the processed radmon data to a JSON file for use by html # documents # # Copyright 2015 Jeff Owrey # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/license. # # Revision History # * v20 released 15 Sep 2015 by J L Owrey; first release # * v21 released 27 Nov 2017 by J L Owrey; bug fixes; updates # * v22 released 03 Mar 2018 by J L Owrey; improved code readability; # improved radmon device offline status handling # * v23 released 16 Nov 2018 by J L Owrey: improved fault handling # and data conversion # * v24 released 14 Jun 2021 by J L Owrey; minor revisions # * v25 released 9 Jul 2021 by J L Owrey; improved handling of # monitor status function # #2345678901234567890123456789012345678901234567890123456789012345678901234567890 import os import sys import signal import subprocess import multiprocessing import time import calendar import json from urllib.request import urlopen ### ENVIRONMENT ### _USER = os.environ['USER'] _SERVER_MODE = "primary" _USE_RADMON_TIMESTAMP = True ### DEFAULT RADIATION MONITOR URL ### _DEFAULT_RADIATION_MONITOR_URL = \ "http://192.168.1.24" ### FILE AND FOLDER LOCATIONS ### # folder for containing dynamic data objects _DOCROOT_PATH = "/home/%s/public_html/radmon/" % _USER # folder for charts and output data file _CHARTS_DIRECTORY = _DOCROOT_PATH + "dynamic/" # location of data output file _OUTPUT_DATA_FILE = _DOCROOT_PATH + "dynamic/radmonData.js" # database that stores radmon data _RRD_FILE = "/home/%s/database/radmonData.rrd" % _USER ### GLOBAL CONSTANTS ### # max number of failed data requests allowed _MAX_FAILED_DATA_REQUESTS = 3 # interval in seconds between data requests _DEFAULT_DATA_REQUEST_INTERVAL = 2 # number seconds to wait for a response to HTTP request _HTTP_REQUEST_TIMEOUT = 3 # interval in seconds between database updates _DATABASE_UPDATE_INTERVAL = 30 # interval in seconds between chart updates _CHART_UPDATE_INTERVAL = 300 # standard chart width in pixels _CHART_WIDTH = 600 # standard chart height in pixels _CHART_HEIGHT = 150 ### GLOBAL VARIABLES ### # turn on or off of verbose debugging information verboseMode = False debugMode = False # The following two items are used for detecting system faults # and radiation monitor online or offline status. # count of failed attempts to get data from radiation monitor failedUpdateCount = 0 # detected status of radiation monitor device radmonOnline = False # status of reset command to radiation monitor remoteDeviceReset = False # ip address of radiation monitor radiationMonitorUrl = _DEFAULT_RADIATION_MONITOR_URL # web update frequency dataRequestInterval = _DEFAULT_DATA_REQUEST_INTERVAL ### PRIVATE METHODS ### def getTimeStamp(): """ Set the error message time stamp to the local system time. Parameters: none Returns: string containing the time stamp """ return time.strftime( "%m/%d/%Y %T", time.localtime() ) ##end def def setStatusToOffline(): """Set the detected status of the radiation monitor to "offline" and inform downstream clients by removing input and output data files. Parameters: none Returns: nothing """ global radmonOnline # Inform downstream clients by removing output data file. if os.path.exists(_OUTPUT_DATA_FILE): os.remove(_OUTPUT_DATA_FILE) # If the radiation monitor was previously online, then send # a message that we are now offline. if radmonOnline: print('%s radiation monitor offline' % getTimeStamp()) radmonOnline = False ##end def def terminateAgentProcess(signal, frame): """Send a message to log when the agent process gets killed by the operating system. Inform downstream clients by removing input and output data files. Parameters: signal, frame - dummy parameters Returns: nothing """ print('%s terminating radmon agent process' % \ (getTimeStamp())) setStatusToOffline() sys.exit(0) ##end def ### PUBLIC METHODS ### def getRadiationData(dData): """Send http request to radiation monitoring device. The response from the device contains the radiation data as unformatted ascii text. Parameters: none Returns: a string containing the radiation data if successful, or None if not successful """ sUrl = radiationMonitorUrl if remoteDeviceReset: sUrl += "/reset" # reboot the radiation monitor else: sUrl += "/rdata" # request data from the monitor try: currentTime = time.time() response = urlopen(sUrl, timeout=_HTTP_REQUEST_TIMEOUT) requestTime = time.time() - currentTime content = response.read().decode('utf-8') content = content.replace('\n', '') content = content.replace('\r', '') if content == "": raise Exception("empty response") except Exception as exError: # If no response is received from the device, then assume that # the device is down or unavailable over the network. In # that case return None to the calling function. if verboseMode: print("%s getRadiationData: %s" % (getTimeStamp(), exError)) return False ##end try if debugMode: print(content) if verboseMode: print("http request successful: %.4f sec" % requestTime) dData['content'] = content return True ##end def def parseDataString(dData): """Parse the data string returned by the radiation monitor into its component parts. Parameters: dData - a dictionary object to contain the parsed data items Returns: True if successful, False otherwise """ # Example radiation monitor data string # $,UTC=17:09:33 6/22/2021,CPS=0,CPM=26,uSv/hr=0.14,Mode=SLOW,# try: sData = dData.pop('content') lData = sData[2:-2].split(',') except Exception as exError: print("%s parseDataString: %s" % (getTimeStamp(), exError)) return False # Verfy the expected number of data items have been received. if len(lData) != 5: print("%s parse failed: corrupted data string" % getTimeStamp()) return False; # Load the parsed data into a dictionary for easy access. for item in lData: if "=" in item: dData[item.split('=')[0]] = item.split('=')[1] # Add status to dictionary object dData['status'] = 'online' dData['serverMode'] = _SERVER_MODE return True ##end def def convertData(dData): """Convert individual radiation data items as necessary. Parameters: dData - a dictionary object containing the radiation data Returns: True if successful, False otherwise """ try: if _USE_RADMON_TIMESTAMP: # Convert the UTC timestamp provided by the radiation monitoring # device to epoch local time in seconds. ts_utc = time.strptime(dData['UTC'], "%H:%M:%S %m/%d/%Y") epoch_local_sec = calendar.timegm(ts_utc) dData['ELT'] = epoch_local_sec else: # Use a timestamp generated by the requesting server (this) # instead of the timestamp provided by the radiation monitoring # device. Using the server generated timestamp prevents errors # that occur when the radiation monitoring device fails to # synchronize with a valid NTP time server. dData['ELT'] = time.time() dData['date'] = \ time.strftime("%m/%d/%Y %T", time.localtime(dData['ELT'])) dData['mode'] = dData.pop('Mode').lower() dData['uSvPerHr'] = '%.2f' % float(dData.pop('uSv/hr')) except Exception as exError: print("%s data conversion failed: %s" % (getTimeStamp(), exError)) return False return True ##end def def writeOutputFile(dData): """Write radiation data items to the output data file, formatted as a JSON file. This file may then be accessed and used by by downstream clients, for instance, in HTML documents. Parameters: dData - a dictionary object containing the data to be written to the output data file Returns: True if successful, False otherwise """ # Format the radmon data as string using java script object notation. jsData = json.loads("{}") try: for key in dData: jsData.update({key:dData[key]}) sData = "[%s]" % json.dumps(jsData) except Exception as exError: print("%s writeOutputFile: %s" % (getTimeStamp(), exError)) return False if debugMode: print(sData) # Write the string to the output data file for use by html documents. try: fc = open(_OUTPUT_DATA_FILE, "w") fc.write(sData) fc.close() except Exception as exError: print("%s writeOutputFile: %s" % (getTimeStamp(), exError)) return False return True ## end def def setRadmonStatus(updateSuccess): """Detect if radiation monitor is offline or not available on the network. After a set number of attempts to get data from the monitor set a flag that the radmon is offline. Parameters: updateSuccess - a boolean that is True if data request successful, False otherwise Returns: nothing """ global failedUpdateCount, radmonOnline if updateSuccess: failedUpdateCount = 0 # Set status and send a message to the log if the device # previously offline and is now online. if not radmonOnline: print('%s radiation monitor online' % getTimeStamp()) radmonOnline = True return elif failedUpdateCount == _MAX_FAILED_DATA_REQUESTS - 1: # Max number of failed data requests, so set # device status to offline. setStatusToOffline() ## end if failedUpdateCount += 1 ##end def ### DATABASE FUNCTIONS ### def updateDatabase(dData): """ Update the rrdtool database by executing an rrdtool system command. Format the command using the data extracted from the radiation monitor response. Parameters: dData - dictionary object containing data items to be written to the rr database file Returns: True if successful, False otherwise """ global remoteDeviceReset # The RR database stores whole units, so convert uSv to Sv. SvPerHr = float(dData['uSvPerHr']) * 1.0E-06 # Format the rrdtool update command. strCmd = "rrdtool update %s %s:%s:%s" % \ (_RRD_FILE, dData['ELT'], dData['CPM'], SvPerHr) if debugMode: print("%s" % strCmd) # DEBUG # Run the command as a subprocess. try: subprocess.check_output(strCmd, shell=True, \ stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exError: print("%s: rrdtool update failed: %s" % \ (getTimeStamp(), exError.output)) if exError.output.find("illegal attempt to update using time") > -1: remoteDeviceReset = True print("%s: rebooting radiation monitor" % (getTimeStamp())) return False if verboseMode and not debugMode: print("database update successful") return True ##end def def createGraph(fileName, dataItem, gLabel, gTitle, gStart, lower, upper, addTrend, autoScale): """Uses rrdtool to create a graph of specified radmon data item. Parameters: fileName - name of file containing the graph dataItem - data item to be graphed gLabel - string containing a graph label for the data item gTitle - string containing a title for the graph gStart - beginning time of the graphed data lower - lower bound for graph ordinate #NOT USED upper - upper bound for graph ordinate #NOT USED addTrend - 0, show only graph data 1, show only a trend line 2, show a trend line and the graph data autoScale - if True, then use vertical axis auto scaling (lower and upper parameters are ignored), otherwise use lower and upper parameters to set vertical axis scale Returns: True if successful, False otherwise """ gPath = _CHARTS_DIRECTORY + fileName + ".png" trendWindow = { 'end-1day': 7200, 'end-4weeks': 172800, 'end-12months': 604800 } # Format the rrdtool graph command. # Set chart start time, height, and width. strCmd = "rrdtool graph %s -a PNG -s %s -e now -w %s -h %s " \ % (gPath, gStart, _CHART_WIDTH, _CHART_HEIGHT) # Set the range and scaling of the chart y-axis. if lower < upper: strCmd += "-l %s -u %s -r " % (lower, upper) elif autoScale: strCmd += "-A " strCmd += "-Y " # Set the chart ordinate label and chart title. strCmd += "-v %s -t %s " % (gLabel, gTitle) # Show the data, or a moving average trend line over # the data, or both. strCmd += "DEF:dSeries=%s:%s:LAST " % (_RRD_FILE, dataItem) if addTrend == 0: strCmd += "LINE1:dSeries#0400ff " elif addTrend == 1: strCmd += "CDEF:smoothed=dSeries,%s,TREND LINE2:smoothed#006600 " \ % trendWindow[gStart] elif addTrend == 2: strCmd += "LINE1:dSeries#0400ff " strCmd += "CDEF:smoothed=dSeries,%s,TREND LINE2:smoothed#006600 " \ % trendWindow[gStart] if debugMode: print("\n%s" % strCmd) # DEBUG # Run the formatted rrdtool command as a subprocess. try: result = subprocess.check_output(strCmd, \ stderr=subprocess.STDOUT, \ shell=True) except subprocess.CalledProcessError as exError: print("rrdtool graph failed: %s" % (exError.output)) return False if verboseMode: print("rrdtool graph: %s" % result.decode('utf-8'), end='') return True ##end def def generateGraphs(): """Generate graphs for display in html documents. Parameters: none Returns: nothing """ autoScale = False # past 24 hours createGraph('24hr_cpm', 'CPM', 'counts\ per\ minute', 'CPM\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale) createGraph('24hr_svperhr', 'SvperHr', 'Sv\ per\ hour', 'Sv/Hr\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale) # past 4 weeks createGraph('4wk_cpm', 'CPM', 'counts\ per\ minute', 'CPM\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale) createGraph('4wk_svperhr', 'SvperHr', 'Sv\ per\ hour', 'Sv/Hr\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale) # past year createGraph('12m_cpm', 'CPM', 'counts\ per\ minute', 'CPM\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale) createGraph('12m_svperhr', 'SvperHr', 'Sv\ per\ hour', 'Sv/Hr\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale) ##end def def getCLarguments(): """Get command line arguments. There are four possible arguments -d turns on debug mode -v turns on verbose mode -t sets the radiation device query interval -u sets the url of the radiation monitoring device Returns: nothing """ global verboseMode, debugMode, dataRequestInterval, \ radiationMonitorUrl index = 1 while index < len(sys.argv): if sys.argv[index] == '-v': verboseMode = True elif sys.argv[index] == '-d': verboseMode = True debugMode = True elif sys.argv[index] == '-t': dataRequestInterval = abs(int(sys.argv[index + 1])) index += 1 elif sys.argv[index] == '-u': radiationMonitorUrl = sys.argv[index + 1] if radiationMonitorUrl.find('http://') < 0: radiationMonitorUrl = 'http://' + radiationMonitorUrl index += 1 else: cmd_name = sys.argv[0].split('/') print("Usage: %s [-d] [-t seconds] [-u url}" % cmd_name[-1]) exit(-1) index += 1 ##end def def main(): """Handles timing of events and acts as executive routine managing all other functions. Parameters: none Returns: nothing """ signal.signal(signal.SIGTERM, terminateAgentProcess) signal.signal(signal.SIGINT, terminateAgentProcess) print('===================') print('%s starting up radmon agent process' % \ (getTimeStamp())) # last time output JSON file updated lastDataRequestTime = -1 # last time charts generated lastChartUpdateTime = - 1 # last time the rrdtool database updated lastDatabaseUpdateTime = -1 ## Get command line arguments. getCLarguments() ## Exit with error if rrdtool database does not exist. if not os.path.exists(_RRD_FILE): print('rrdtool database does not exist\n' \ 'use createRadmonRrd script to ' \ 'create rrdtool database\n') exit(1) ## main loop while True: currentTime = time.time() # get current time in seconds # Every data update interval request data from the radiation # monitor and process the received data. if currentTime - lastDataRequestTime > dataRequestInterval: lastDataRequestTime = currentTime dData = {} # Get the data string from the device. result = getRadiationData(dData) # If successful parse the data. if result: result = parseDataString(dData) # If parsing successful, convert the data. if result: result = convertData(dData) # If conversion successful, write data to data files. if result: writeOutputFile(dData) # At the rrdtool database update interval, update the database. if result and (currentTime - lastDatabaseUpdateTime > \ _DATABASE_UPDATE_INTERVAL): lastDatabaseUpdateTime = currentTime ## Update the round robin database with the parsed data. result = updateDatabase(dData) # Set the radmon status to online or offline depending on the # success or failure of the above operations. setRadmonStatus(result) # At the chart generation interval, generate charts. if currentTime - lastChartUpdateTime > _CHART_UPDATE_INTERVAL: lastChartUpdateTime = currentTime p = multiprocessing.Process(target=generateGraphs, args=()) p.start() # Relinquish processing back to the operating system until # the next update interval. elapsedTime = time.time() - currentTime if verboseMode: if result: print("update successful: %6f sec\n" % elapsedTime) else: print("update failed: %6f sec\n" % elapsedTime) remainingTime = dataRequestInterval - elapsedTime if remainingTime > 0.0: time.sleep(remainingTime) ## end while return ## end def if __name__ == '__main__': main()