#!/usr/bin/python3 -u
# The -u option above turns off block buffering of python output. This
# assures that each error message gets individually printed to the log file.
#
# Module: arednsigAgent.py
#
# Description: This module acts as an agent between the aredn node
# and aredn mest services. The agent periodically sends an http
# request to the aredn node, processes the response from
# the node, and performs a number of operations:
# - conversion of data items
# - update a round robin (rrdtool) database with the node data
# - periodically generate graphic charts for display in html documents
# - write the processed node status to a JSON file for use by html
# documents
#
# Copyright 2020 Jeff Owrey
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/license.
#
# Revision History
# * v20 released 11 Jan 2020 by J L Owrey; first release
# * v21 released 13 Feb 2020 by J L Owrey; fixed bug occuring when node
# powers on and signal data memory is empty. Data points with N/A data
# are discarded.
# * v22 released 31 Mar 2020 by J L Owrey; upgraded for compatibility with
# Aredn firmware version 3.20.3.0. This agent now downloads the node's
# status page and parsed the signal data from the html.
# * v23 released 11 Jun 2021 by J L Owrey; remove unused code.
# * v24 released 14 Jun 2021 by J L Owrey; minor revisions
# * v25 released 9 Jul 2021 by J L Owrey; improved handling of
# node status function
#
#2345678901234567890123456789012345678901234567890123456789012345678901234567890
import os
import sys
import signal
import multiprocessing
import time
import json
from urllib.request import urlopen
import rrdbase
### ENVIRONMENT ###
_USER = os.environ['USER']
_SERVER_MODE = "primary"
### DEFAULT AREDN NODE URL ###
_DEFAULT_AREDN_NODE_URL = "http://localnode.local.mesh/cgi-bin/status"
### FILE AND FOLDER LOCATIONS ###
# folder for containing dynamic data objects
_DOCROOT_PATH = "/home/%s/public_html/arednsig/" % _USER
# folder for charts and output data file
_CHARTS_DIRECTORY = _DOCROOT_PATH + "dynamic/"
# location of data output file
_OUTPUT_DATA_FILE = _DOCROOT_PATH + "dynamic/arednsigData.js"
# database that stores node data
_RRD_FILE = "/home/%s/database/arednsigData.rrd" % _USER
### GLOBAL CONSTANTS ###
# maximum number of failed data requests allowed
_MAX_FAILED_DATA_REQUESTS = 2
# maximum number of http request retries allowed
_MAX_HTTP_RETRIES = 3
# delay time between http request retries
_HTTP_RETRY_DELAY = 1.1199
# interval in seconds between data requests
_DEFAULT_DATA_REQUEST_INTERVAL = 60
# number seconds to wait for a response to HTTP request
_HTTP_REQUEST_TIMEOUT = 3
# interval in seconds between database updates
_DATABASE_UPDATE_INTERVAL = 60
# chart update interval in seconds
_CHART_UPDATE_INTERVAL = 600
# standard chart width in pixels
_CHART_WIDTH = 600
# standard chart height in pixels
_CHART_HEIGHT = 150
# Set this to True only if this server is intended to relay raw
### GLOBAL VARIABLES ###
# turn on or off of verbose debugging information
verboseMode = False
debugMode = False
reportUpdateFails = False
# The following two items are used for detecting system faults
# and aredn node online or offline status.
# count of failed attempts to get data from aredn node
failedUpdateCount = 0
httpRetries = 0
nodeOnline = False
# ip address of aredn node
arednNodeUrl = _DEFAULT_AREDN_NODE_URL
# frequency of data requests to aredn node
dataRequestInterval = _DEFAULT_DATA_REQUEST_INTERVAL
# chart update interval
chartUpdateInterval = _CHART_UPDATE_INTERVAL
# rrdtool database interface handler instance
rrdb = None
### PRIVATE METHODS ###
def getTimeStamp():
"""
Set the error message time stamp to the local system time.
Parameters: none
Returns: string containing the time stamp
"""
return time.strftime( "%m/%d/%Y %T", time.localtime() )
##end def
def setStatusToOffline():
"""Set the detected status of the aredn node to
"offline" and inform downstream clients by removing input
and output data files.
Parameters: none
Returns: nothing
"""
global nodeOnline
# Inform downstream clients by removing output data file.
if os.path.exists(_OUTPUT_DATA_FILE):
os.remove(_OUTPUT_DATA_FILE)
# If the aredn node was previously online, then send
# a message that we are now offline.
if nodeOnline:
print('%s aredn node offline' % getTimeStamp())
nodeOnline = False
##end def
def terminateAgentProcess(signal, frame):
"""Send a message to log when the agent process gets killed
by the operating system. Inform downstream clients
by removing input and output data files.
Parameters:
signal, frame - dummy parameters
Returns: nothing
"""
# Inform downstream clients by removing output data file.
if os.path.exists(_OUTPUT_DATA_FILE):
os.remove(_OUTPUT_DATA_FILE)
print('%s terminating arednsig agent process' % getTimeStamp())
sys.exit(0)
##end def
### PUBLIC METHODS ###
def getNodeData(dData):
"""Send http request to aredn node. The response from the
node contains the node signal data as unformatted ascii text.
Parameters: none
Returns: True if successful,
or False if not successful
"""
global httpRetries
try:
currentTime = time.time()
response = urlopen(arednNodeUrl, timeout=_HTTP_REQUEST_TIMEOUT)
requestTime = time.time() - currentTime
content = response.read().decode('utf-8')
content = content.replace('\n', '')
content = content.replace('\r', '')
if content == "":
raise Exception("empty response")
except Exception as exError:
# If no response is received from the device, then assume that
# the device is down or unavailable over the network. In
# that case return None to the calling function.
httpRetries += 1
if reportUpdateFails:
print("%s " % getTimeStamp(), end='')
if reportUpdateFails or verboseMode:
print("http request failed (%d): %s" % \
(httpRetries, exError))
if httpRetries > _MAX_HTTP_RETRIES:
httpRetries = 0
return False
else:
time.sleep(_HTTP_RETRY_DELAY)
return getNodeData(dData)
## end try
if debugMode:
print(content)
if verboseMode:
print("http request successful: "\
"%.4f seconds" % requestTime)
httpRetries = 0
dData['content'] = content
return True
##end def
def parseDataString(dData):
"""Parse the node signal data JSON string from the aredn node
into its component parts.
Parameters:
sData - the string containing the data to be parsed
dData - a dictionary object to contain the parsed data items
Returns: True if successful, False otherwise
"""
sData = dData.pop('content')
try:
strBeginSearch = '<nobr>Signal/Noise/Ratio</nobr></th>' \
'<td valign=middle><nobr><big><b>'
strEndSearch = 'dB</b>'
iBeginIndex = sData.find(strBeginSearch) + len(strBeginSearch)
iEndIndex = sData.find(strEndSearch, iBeginIndex)
#print("search params: %d, %d" % (iBeginIndex, iEndIndex))
if iBeginIndex == -1 or iEndIndex == -1:
raise Exception("signal data not found in status page")
snr = sData[iBeginIndex:iEndIndex]
snr = snr.replace(' ','')
lsnr = snr.split('/')
dData['signal'] = lsnr[0]
dData['noise'] = lsnr[1]
dData['snr'] = lsnr[2]
except Exception as exError:
print("%s parse failed: %s" % (getTimeStamp(), exError))
return False
## end try
# Add status information to dictionary object.
dData['date'] = getTimeStamp()
dData['chartUpdateInterval'] = chartUpdateInterval
dData['dataRequestInterval'] = dataRequestInterval
dData['serverMode'] = _SERVER_MODE
return True
##end def
def writeOutputFile(dData):
"""Write node data items to the output data file, formatted as
a Javascript file. This file may then be accessed and used by
by downstream clients, for instance, in HTML documents.
Parameters:
sData - a string object containing the data to be written
to the output data file
Returns: True if successful, False otherwise
"""
# Write file for use by html clients. The following two
# data items are sent to the client file.
# * The last database update date and time
# * The data request interval
# Format data into a JSON string.
jsData = json.loads("{}")
try:
for key in dData:
jsData.update({key:dData[key]})
sData = "[%s]" % json.dumps(jsData)
except Exception as exError:
print("%s writeOutputFile: %s" % (getTimeStamp(), exError))
return False
if debugMode:
print(sData)
try:
fc = open(_OUTPUT_DATA_FILE, "w")
fc.write(sData)
fc.close()
except Exception as exError:
print("%s write output file failed: %s" % (getTimeStamp(), exError))
return False
return True
## end def
def setNodeStatus(updateSuccess):
"""Detect if aredn node is offline or not available on
the network. After a set number of attempts to get data
from the node set a flag that the node is offline.
Parameters:
updateSuccess - a boolean that is True if data request
successful, False otherwise
Returns: nothing
"""
global failedUpdateCount, nodeOnline
if updateSuccess:
failedUpdateCount = 0
# Set status and send a message to the log if the device
# previously offline and is now online.
if not nodeOnline:
print('%s aredn node online' % getTimeStamp())
nodeOnline = True
return
else:
# The last attempt failed, so update the failed attempts
# count.
failedUpdateCount += 1
if failedUpdateCount == _MAX_FAILED_DATA_REQUESTS:
# Max number of failed data requests, so set
# device status to offline.
setStatusToOffline()
##end def
### GRAPH FUNCTIONS ###
def generateGraphs():
"""Generate graphs for display in html documents.
Parameters: none
Returns: nothing
"""
autoScale = False
# The following will force creation of charts
# of only signal strength and S/N charts. Note that the following
# data items appear constant and do not show variation with time:
# noise level, rx mcs, rx rate, tx mcs, tx rate. Therefore, until
# these parameters are demonstrated to vary in time, there is no point
# in creating the charts for these data items.
createAllCharts = False
# 24 hour stock charts
#### REPLACE WITH createRadGraph ####
rrdb.createAutoGraph('24hr_signal', 'S', 'dBm',
'RSSI\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
rrdb.createAutoGraph('24hr_snr', 'SNR', 'dB',
'SNR\ -\ Last\ 24\ Hours', 'end-1day', 0, 0, 2, autoScale)
# 4 week stock charts
rrdb.createAutoGraph('4wk_signal', 'S', 'dBm',
'RSSI\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
rrdb.createAutoGraph('4wk_snr', 'SNR', 'dB',
'SNR\ -\ Last\ 4\ Weeks', 'end-4weeks', 0, 0, 2, autoScale)
# 12 month stock charts
rrdb.createAutoGraph('12m_signal', 'S', 'dBm',
'RSSI\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
rrdb.createAutoGraph('12m_snr', 'SNR', 'dB',
'SNR\ -\ Past\ Year', 'end-12months', 0, 0, 2, autoScale)
if verboseMode:
#print() # print a blank line to improve readability when in debug mode
pass
##end def
def getCLarguments():
"""Get command line arguments. There are four possible arguments
-d turns on debug mode
-v turns on verbose debug mode
-p sets the aredn node query interval
-u sets the url of the aredn nodeing device
Returns: nothing
"""
global verboseMode, debugMode, dataRequestInterval, \
arednNodeUrl, reportUpdateFails
index = 1
while index < len(sys.argv):
if sys.argv[index] == '-v':
verboseMode = True
elif sys.argv[index] == '-d':
verboseMode = True
debugMode = True
elif sys.argv[index] == '-r':
reportUpdateFails = True
# Update period and url options
elif sys.argv[index] == '-p':
try:
dataRequestInterval = abs(float(sys.argv[index + 1]))
except:
print("invalid polling period")
exit(-1)
index += 1
elif sys.argv[index] == '-u':
arednNodeUrl = sys.argv[index + 1]
if arednNodeUrl.find('http://') < 0:
arednNodeUrl = 'http://' + arednNodeUrl
index += 1
else:
cmd_name = sys.argv[0].split('/')
print("Usage: %s [-v|d] [-p seconds] [-u url]" % cmd_name[-1])
exit(-1)
index += 1
##end def
def setup():
"""Handles timing of events and acts as executive routine managing
all other functions.
Parameters: none
Returns: nothing
"""
global rrdb
## Get command line arguments.
getCLarguments()
print('======================================================')
print('%s starting up arednsig agent process' % \
(getTimeStamp()))
## Exit with error if rrdtool database does not exist.
if not os.path.exists(_RRD_FILE):
print('rrdtool database does not exist\n' \
'use createArednsigRrd script to ' \
'create rrdtool database\n')
exit(1)
signal.signal(signal.SIGTERM, terminateAgentProcess)
signal.signal(signal.SIGINT, terminateAgentProcess)
# Define object for calling rrdtool database functions.
rrdb = rrdbase.rrdbase( _RRD_FILE, _CHARTS_DIRECTORY, _CHART_WIDTH, \
_CHART_HEIGHT, verboseMode, debugMode )
## end def
def loop():
# last time output JSON file updated
lastDataRequestTime = -1
# last time charts generated
lastChartUpdateTime = -1
# last time the rrdtool database updated
lastDatabaseUpdateTime = -1
## main loop
while True:
currentTime = time.time() # get current time in seconds
# Every web update interval request data from the aredn
# node and process the received data.
if currentTime - lastDataRequestTime > dataRequestInterval:
lastDataRequestTime = currentTime
dData = {}
# Get the data string from the device.
result = getNodeData(dData)
# If successful parse the data.
if result:
result = parseDataString(dData)
# If parse successful, write data output data file.
if result:
writeOutputFile(dData)
# At the rrdtool database update interval, update the database.
if result and (currentTime - lastDatabaseUpdateTime > \
_DATABASE_UPDATE_INTERVAL):
lastDatabaseUpdateTime = currentTime
# If write output file successful, update the database.
if result:
result = rrdb.updateDatabase(dData['date'], \
dData['signal'], dData['noise'], dData['snr'], \
'0', '0', '0', '0')
# Set the node status to online or offline depending on the
# success or failure of the above operations.
setNodeStatus(result)
# At the chart generation interval, generate charts.
if currentTime - lastChartUpdateTime > chartUpdateInterval:
lastChartUpdateTime = currentTime
p = multiprocessing.Process(target=generateGraphs, args=())
p.start()
# Relinquish processing back to the operating system until
# the next update interval.
elapsedTime = time.time() - currentTime
if verboseMode:
if result:
print("update successful: %s sec\n" % elapsedTime)
else:
print("update failed: %s sec\n" % elapsedTime)
remainingTime = dataRequestInterval - elapsedTime
if remainingTime > 0.0:
time.sleep(remainingTime)
## end while
## end def
if __name__ == '__main__':
setup()
loop()