import os import json import datetime from slackclient import SlackClient def dataCompareTest(new,old,threshPercent): factor1 = (100.0 + float(threshPercent)) / 100.0 factor2 = (100.0 - float(threshPercent)) / 100.0 if(new > (old*factor1)): return True else: return False str2ShortCode2Thres= { 'GPS_L1CA' : 0.64, 'GPS_L2E' : 1.00, 'GPS_L2C' : 0.72, 'GPS_L5' : 0.72, 'GLONASS_G1C' : 2.4, 'GLONASS_G1P' : 1.7, 'GLONASS_G2C' : 1.2, 'GLONASS_G2P' : 0.72, 'Galileo_E1' : 0.66, 'Galileo_E5A' : 0.75, 'Galileo_E5B' : 0.80, 'Galileo_E5L' : 0.32, 'Galileo_E6' : 0.50, # No data yet 'Beidou_B1' : 1.8, 'Beidou_B2' : 1.0, 'Beidou_B3' : 0.5, # No data yet 'Beidou_B1C' : 1.0, 'Beidou_B2A' : 1.0} str2ShortCarr2Thres= { 'GPS_L1CA' : 55, 'GPS_L2E' : 46, 'GPS_L2C' : 46, 'GPS_L5' : 49, 'GLONASS_G1C' : 14, 'GLONASS_G1P' : 14, 'GLONASS_G2C' : 20, 'GLONASS_G2P' : 20, 'Galileo_E1' : 45, 'Galileo_E5A' : 50, 'Galileo_E5B' : 50, 'Galileo_E5L' : 45, 'Galileo_E6' : 45, # No data yet 'Beidou_B1' : 60, 'Beidou_B2' : 60, 'Beidou_B3' : 60, # No data yet 'Beidou_B1C' : 60, # No data yet 'Beidou_B2A' : 60} combo2str = { 0 : 'GPS_L1CA', 1 : 'GPS_L2E', 2 : 'GPS_L2C', 3 : 'GPS_L5', 4 : 'GLONASS_G1C', 5 : 'GLONASS_G1P', 6 : 'GLONASS_G2C', 7 : 'GLONASS_G2P', 8 : 'Galileo_E1', 9 : 'Galileo_E5A', 10 : 'Galileo_E5B', 11 : 'Galileo_E5L', 12 : 'Galileo_E6', 13 : 'Beidou_B1', 14 : 'Beidou_B2', 15 : 'Beidou_B3', 16 : 'Beidou_B1C', 17 : 'Beidou_B2A'} file2name = {'-noGLN-L2E-MP12.txt' : 'MP12-GPS_L1CA_L2E', '-noGLN-L2C-MP12.txt' : 'MP12-GPS_L1CA_L2C', '-GLN_CA-MP12.txt' : 'MP12-GLN_L1CA_L2CA', '-GLN_P-MP12.txt' : 'MP12-GLN_L1P_L2P', '-noGLN-L2E-MP15.txt' : 'MP15-GPS_L1CA_L5__GAL_E1_E5A', '-noGLN-L2E-MP16.txt' : 'MP16-GAL_E1_E6__BDS_B1_B3', '-noGLN-L2E-MP17.txt' : 'MP17-GAL_E1_E5B__BDS_B1_B2', '-noGLN-L2E-MP18.txt' : 'MP18-GAL_E1_E5AltBOC', '-noGLN-L2E-MP21.txt' : 'MP21-GPS_L2E_L1CA', '-noGLN-L2C-MP21.txt' : 'MP21-GPS_L2C_L1CA', '-GLN_CA-MP21.txt' : 'MP21-GLN_L2CA_L1CA', '-GLN_P-MP21.txt' : 'MP21-GLN_L2P_L1P', '-noGLN-L2E-MP51.txt' : 'MP51-GPS_L5_L1CA__GAL_E5A_E1', '-noGLN-L2E-MP61.txt' : 'MP61-GAL_E6_E1__BDS_B3_B1', '-noGLN-L2E-MP71.txt' : 'MP71-GAL_E5B_E1__BDS_B2_B1', '-noGLN-L2E-MP81.txt' : 'MP81-GAL_E5AltBOC_E1' } # Teqc threshold MP2Thresh = { 'MP12-GLN_L1CA_L2CA.png' : 0.42, 'MP12-GLN_L1P_L2P.png' : 0.40, 'MP12-GPS_L1CA_L2C.png' : 0.26, 'MP12-GPS_L1CA_L2E.png' : 0.26, 'MP15-GPS_L1CA_L5__GAL_E1_E5A.png' : 0.24, 'MP16-GAL_E1_E6__BDS_B1_B3.png' : 0.25, 'MP17-GAL_E1_E5B__BDS_B1_B2.png' : 0.26, 'MP18-GAL_E1_E5AltBOC.png' : 0.23, 'MP21-GLN_L2CA_L1CA.png' : 0.52, 'MP21-GLN_L2P_L1P.png' : 0.22, 'MP21-GPS_L2C_L1CA.png' : 0.27, 'MP21-GPS_L2E_L1CA.png' : 0.31, 'MP51-GPS_L5_L1CA__GAL_E5A_E1.png' : 0.23, 'MP61-GAL_E6_E1__BDS_B3_B1.png' : 0.29, 'MP71-GAL_E5B_E1__BDS_B2_B1.png' : 0.25, 'MP81-GAL_E5AltBOC_E1.png' : 0.08} TeqcRXData = [ "BD935_1_ROVER", "BD935_2_ROVER", "BD935_3_ROVER", "BD940_1_ROVER", "BD940_2_ROVER", "BD940_3_ROVER", "BD992_ROVER_A0", "BD992_ROVER_A1"] def DDTest(suffix,codeOrCarr): DDReport = [] slackStr = '' for i in range(len(combo2str)): DDCodeSigma = [] DDCarrSigma = [] filename = combo2str.get(i) + suffix if(os.path.isfile(filename + '.txt')): with open(filename + '.txt','r') as f: for line in f: data = line.rstrip().split() DDCodeSigma.append((float)(data[15])) DDCarrSigma.append((float)(data[10])) if(codeOrCarr == True): data = DDCodeSigma thresh = str2ShortCode2Thres.get(combo2str.get(i)) else: data = DDCarrSigma thresh = str2ShortCarr2Thres.get(combo2str.get(i)) if(len(data) > 0): if(data[-1] > thresh): slackStr = slackStr + "DD " + combo2str.get(i) + ' ' + str.format('{0:.2f}',data[-1]) if(codeOrCarr == True): units = 'm' else: if("GLONASS" in combo2str.get(i)): units = 'mm' else: units = 'mcycles' slackStr = slackStr + units + ' ' slackStr = slackStr + 'Threshold = ' + str(thresh) + units + '\n' d = {} d['signal'] = suffix[1:6] + ' ' + combo2str.get(i) d['err'] = data[-1] d['threshold'] = thresh DDReport.append(d) return slackStr,DDReport # # testData() tests the data we are monitoring in the dash board # against a set of thresholds. It returns two objects: # errData - dictionary which can be turned into JSON # slackStr - string containing the error data that can be # sent to slack, emailed etc # chatStr - string containing data for Google Chat # # To convert errData to JSON run the following commands: # # jsonStr = json.dumps(eData) # jsonOut = json.loads(jsonStr) # print(json.dumps(jsonOut,indent=4,sort_keys=True)) # # def testData(): slackStr = '' chatStr = '' errData = {} ########################################## ####### Cycle slip test ########################################## SlipData = [] maxCombo = 18 # json reporting slipReport = [] for combos in range(maxCombo): SlipData.append([]) filename = '/net/fermion/mnt/data_drive/DataDump/DashBoard/NoPiSlips.txt' if(os.path.isfile(filename)): with open(filename,'r') as f: for line in f: data = line.rstrip().split() if(int(data[1]) < maxCombo): SlipData[int(data[1])].append(line) testStr = '' tokens = None for combos in range(maxCombo): if(len(SlipData[combos]) > 0): dataCurrent = SlipData[combos][-1].rstrip().split() if(int(dataCurrent[2])): PPM = float(dataCurrent[9])*1e6/float(dataCurrent[2]) thresh = 0.25 if(PPM > thresh): tokens = dataCurrent[0].split('/') linkRoot = 'http://teller.eng.trimble.com/GPSRxEng/GNSSResults/' linkRoot += tokens[-2] + '/' + tokens[-1] + '/' linkRoot += 'plots/time_carr_all_' + str(combos) + '.png\n' linkRoot += 'Undetected slip event history:\n' linkRoot += 'http://quark.eng.trimble.com/DataDump/DashBoard/SlipEventsPPM-' + str(combos) + '.png\n' linkRoot += 'http://quark.eng.trimble.com/DataDump/DashBoard/SlipEpochsPPM-' + str(combos) + '.png\n' testStr = testStr + "Combo " + str(combos) + "(" + combo2str.get(combos) testStr = testStr + ") Cycle Slip PPM=" + str.format('{0:.4f}',PPM) testStr = testStr + " Threshold =" + str(thresh) + "PPM\n" testStr += linkRoot + '\n' d = {} d['signal'] = 'BD940 ' + combo2str.get(combos) d['err'] = PPM d['threshold'] = thresh slipReport.append(d) if(len(testStr) > 0): slackStr = slackStr + '\n`NoPi Cycle Slip Test`\n' + "```" + testStr + "```" chatStr += '*NoPi Cycle Slip Test*:\n' if(tokens is not None): chatStr += 'http://teller.eng.trimble.com/GPSRxEng/GNSSResults/' chatStr += tokens[-2] + '/' + tokens[-1] + '/' chatStr += 'NoPiDI_Top.html\n' chatStr += testStr errData['slips'] = slipReport ########################################## ####### Reacquisition ########################################## rxStr = ['Wamel','BD970', 'BD935','BD940'] GPSStr = ['L1CA','L2E','L2C','L5'] AcqThres = [6.0,10.0,8.0,8.0] hyst = 5.0; testStr = '' # json reporting reacqReport = [] for signal in range(len(GPSStr)): for rx in range(len(rxStr)): AcqTime = [] filename = '/net/fermion/mnt/data_drive/DataDump/DashBoard/' + rxStr[rx] + '-' + GPSStr[signal] + '-reAcq.txt' if(os.path.isfile(filename)): with open(filename,'r') as f: for line in f: data = line.rstrip().split() AcqTime.append(float(data[1])) if(len(AcqTime) >= 2): if(dataCompareTest(AcqTime[-1],AcqTime[-2],hyst)): testStr = testStr + 'Acquisition ' + rxStr[rx] + ' ' + GPSStr[signal] + ' ' testStr = testStr + str(AcqTime[-1]) + 'secs [Today] vs ' + str(AcqTime[-2]) + 'secs [Yesterday]\n' if(AcqTime[-1] > AcqThres[signal]): testStr = testStr + 'Acquisition ' + rxStr[rx] + ' ' + GPSStr[signal] + ' ' testStr = testStr + str(AcqTime[-1]) + 'secs vs Abs. Threshold ' + str(AcqThres[signal]) +'secs\n' d = {} d['signal'] = rxStr[rx] + ' ' + GPSStr[signal] d['err'] = AcqTime[-1] d['threshold'] = AcqThres[signal] reacqReport.append(d) if(len(testStr) > 0): slackStr += '\n`Reacquisition`\n' + "```" + testStr + "```" chatStr += '*Reacquisition*:' + testStr errData['reacq'] = reacqReport [testStr,DDReport] = DDTest("-BD940_BD940_Short",True) # Code if(len(testStr) > 0): slackStr += '\n`NoPi DD Code (BD940-BD940 Short)`\n' + "```" + testStr + "```" chatStr += '*NoPi DD Code (BD940-BD940 Short)*:' + testStr errData['DDCodeShort'] = DDReport [testStr,DDReport] = DDTest("-BD940_BD940_Short",False) # Carrier if(len(testStr) > 0): slackStr += '\n`NoPi DD Carrier (BD940-BD940 Short)`\n' + "```"+ testStr + "```" chatStr += '*NoPi DD Carrier (BD940-BD940 Short)*:' + testStr errData['DDCarrShort'] = DDReport [testStr,DDReport] = DDTest("-BD940_BD940_Zero",True) # Code if(len(testStr) > 0): slackStr += '\n`NoPi DD Code (BD940-BD940 Zero)`\n' + "```" + testStr + "```" chatStr += '*NoPi DD Code (BD940-BD940 Zero)*:' + testStr errData['DDCodeZero'] = DDReport [testStr,DDReport] = DDTest("-BD940_BD940_Zero",False) # Carrier if(len(testStr) > 0): slackStr += '\n`NoPi DD Carrier (BD940-BD940 Zero)`\n' + "```" + testStr + "```" chatStr += '*NoPi DD Carrier (BD940-BD940 Zero)*:' + testStr errData['DDCarrShort'] = DDReport ########################################## ####### Teqc Test ########################################## testStr = '' teqcReport = [] # There are no duplicates so we can invert the dictionary ivd = {v: k for k, v in file2name.items()} for i in range(len(file2name)): key = ivd.get(list(file2name.values())[i]) Thresh = MP2Thresh.get(list(file2name.values())[i] + '.png') for rx in range(len(TeqcRXData)): MPX = [] now = datetime.datetime.utcnow() for year in range(2018,(now.year + 1)): filename = '/net/quark/mnt/data_drive/DataDump/TeqcStingerLab/' + str(year) + '/' + TeqcRXData[rx] + list(file2name.keys())[i] if(os.path.isfile(filename)): with open(filename,'r') as f: for line in f: data = line.rstrip().split() MPX.append(float(data[3])) if(MPX): if(MPX[-1] > Thresh): testStr += TeqcRXData[rx] + ' ' + list(file2name.values())[i] + ' ' + str.format('{0:.2f}',MPX[-1]) + ' ' testStr += 'Threshold = ' + str(Thresh) + '\n' testStr += 'http://quark.eng.trimble.com/DataDump/TeqcStingerLab/' testStr += list(file2name.values())[i] + '-' +TeqcRXData[rx] + '.png\n' d = {} d['signal'] = TeqcRXData[rx] + ' ' + list(file2name.values())[i] d['err'] = MPX[-1] d['threshold'] = Thresh teqcReport.append(d) if(len(testStr)>0): slackStr = slackStr + '\n`Teqc`\n' + "```" + testStr + "```\n" chatStr += '*Teqc*:\n' + testStr + "\n" errData['teqcMPX'] = teqcReport return errData,slackStr,chatStr def sendTestToSlack(): [errData, slackStr,chatStr] = testData() if(len(slackStr) > 0): print(slackStr) os.environ['http_proxy'] = 'proxy.trimble.com:3128' slackStr = ':boom:Exception\n' + slackStr slack_token = os.environ["SLACK_API_TOKEN"] sc = SlackClient(slack_token) # Channel from https://my.slack.com/messages sc.api_call( "chat.postMessage", channel="CAC19ABHV", text=slackStr) if __name__ == "__main__": sendTestToSlack()