import matplotlib # Allow running headless from the command line matplotlib.use("agg") import math import os import datetime import ftplib import argparse import sys; import json; from numpy import * from pylab import * import mutils as m try: from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener except ImportError: from urllib2 import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener def find( x ) : return where(x)[0] # Simple class to how the task key look up class dotdict(dict): def __getattr__(self, name): try: return self[name] except KeyError: return def getFileFTP(fileRoot,ftpServer,ftpPath,year,month,day,ext): # Get the file if(os.path.isfile(fileRoot + '.' + ext) and (os.path.getsize(fileRoot + '.' + ext) > 0)): print("Already downloaded the file - skipping FTP") else: # Download the file via FTP from Dave's server remoteFile = str(year) + "_" + str(month).zfill(2) + "_" + str(day).zfill(2) + ".T04" localFile = fileRoot + "." + ext print(ftpServer + " " + ftpPath + " " + remoteFile + " " + localFile) ftp = ftplib.FTP(ftpServer) ftp.login("anonymous", "anything") ftp.cwd(ftpPath) try: ftp.retrbinary("RETR " + remoteFile ,open(localFile, 'wb').write) except: print("Error") def processFile(fileRoot, outputRoot, resDir, year, month, day, cacheLoad): global maxTokens # Process the file if(os.path.isfile(fileRoot+ '.T04') and (os.path.getsize(fileRoot+ '.T04') > 0)): if( (cacheLoad == True) and (os.path.isfile(outputRoot+ '.258')) and (os.path.getsize(outputRoot+ '.258') > 0)): print("Found cached task load information") else: os.system("viewdat -d35:258 -m " + fileRoot+ ".T04 > " + outputRoot + ".258") if( (cacheLoad == True) and (os.path.isfile(outputRoot+ '.259')) and (os.path.getsize(outputRoot+ '.259') > 0)): print("Found cached task list information") else: os.system("viewdat -d35:259 -m " + fileRoot+ ".T04 > " + outputRoot + ".259") ################## # --- Get the number of satellites tracked / used from the position record ################## (ref,k)=m.vd2arr(fileRoot+ '.T04',rec='-d29',opt=['--dec=1000']) T = ref[:,k.TIME] Trk = ref[:,k.NTRK] used = ref[:,k.NUSED] print("Processed the satellite tracked/used data") gotKeyStart = False key = dotdict({}) FTPDServerCount = 0 EthWriteCount = 0 HTTPDCount = 0 RTKDPPPCount = 0 RTKFSSCount = 0 if(os.path.isfile(outputRoot + '.259')): f = open(outputRoot + ".259","r") for line in f.readlines(): tokens = line.split() if(gotKeyStart == False): if(len(tokens) == 2 and tokens[0] == "Task"): gotKeyStart = True else: if(len(tokens) == 2 and tokens[0] == "Task"): # If there's a Task line we've probably # concatenated files break if( len(tokens) > 2 and tokens[0] == "TYPE" and tokens[1] == "35:"): break if(len(tokens) == 4): keyStr = tokens[3].rstrip() # Remove return # Some tasks have multiple instances that are not unique, add a # number to the end of the task number if(keyStr == "FTPDserver"): keyStr = keyStr + str(FTPDServerCount) FTPDServerCount = FTPDServerCount + 1 if(keyStr == "Ethernetwritetask"): keyStr = keyStr + str(EthWriteCount) EthWriteCount = EthWriteCount + 1 if(keyStr == "HTTPDworker"): keyStr = keyStr + str(HTTPDCount) HTTPDCount = HTTPDCount + 1 # We get two RTKDPPP tasks when running a heading solution, RTX # etc if(keyStr == "RTKDPPP"): keyStr = keyStr + str(RTKDPPPCount) RTKDPPPCount = RTKDPPPCount + 1 if(keyStr == "RTKFSS"): keyStr = keyStr + str(RTKFSSCount) RTKFSSCount = RTKFSSCount + 1 if(keyStr == "Anti-JamTask"): keyStr = "AntiJamTask" # +1 as we want this to index into the 35:258 record, the first # column in that record is time key[keyStr] = int(tokens[0]) + 1; if(keyStr == "BM" or keyStr == "ISR" or keyStr == "OS"): # BM ISR, Misc ISRs and the OS do not have valid FE IDs print("Key." + str(keyStr) + " = " + str(key[keyStr]) + " core = " + tokens[2]) else: print("Key." + str(keyStr) + " = " + str(key[keyStr]) + " core = " + tokens[2] + " FE ID = " + tokens[1]) f.close() print("Processed the Key data") ################## # -- Read the load data and compute the task loading ################## # List containing the time tags time = [] # List containing the load as a percent loadPerc = [] lastCount = [] NumTypes = 14 for LoadType in range(0,NumTypes): loadPerc.append([]) lastCount.append(0) lastSum = 0; with open(outputRoot + '.258') as f: for line in f: tokens = line.split() if(maxTokens == -1): maxTokens = len(tokens) # If the number of tokens changes the firmware was changed, # this will lead to mismatches in the key and can cause the # script to crash. Even if it doesn't crash it may lead to # erroneous information so don't do any further processing if( (len(tokens) > 50) and (len(tokens) == maxTokens)): countSum = 0; # Start from 1 to skip the time for x in range(1,len(tokens)): countSum = countSum + int(tokens[x]) time.append(float(tokens[0])/1000.0) # --- Loop around collecting what we want to plot # 0 == RTK tasks # 1 == Stinger (inclding BM) # 2 == Background # 3 == Stinger BM # 4 == Stinger LM # 5 == Stinger MM # 6 == Stinger PVT # 7 == Stinger LDPC # 8 == Stinger Anti-Jam # 9 == Stinger Acq # 10 == Stinger NM # 11 == Stinger TS # 12 == Stinger SD # 13 == Stinger AM RTKTasks = 0 StingerTasks = 1 BackgroundTasks = 2 for LoadType in range(0,NumTypes): if(LoadType == RTKTasks): if(key.RTKDPPP0): # Astra installed taskCount = int(tokens[key.RTKDPPP0]) + int(tokens[key.PreRTK]) + int(tokens[key.RTKFSS0]) if(key.RTKDPPP1): taskCount = taskCount + int(tokens[key.RTKDPPP1]) if(key.RTKFSS1): taskCount = taskCount + int(tokens[key.RTKFSS1]) if(key.INSIN): # Not all systems will have INS-RTK tasks taskCount = taskCount + int(tokens[key.INSIN]) + int(tokens[key.INSINKF]) else: # Titan installed taskCount = ( int(tokens[key.PreRTK]) + int(tokens[key.PreTN])) if(key.Interface): taskCount += int(tokens[key.Interface]) if(key.RovDataPrepModule): taskCount += int(tokens[key.RovDataPrepModule]) if(key.DynamicStatusModule): taskCount += int(tokens[key.DynamicStatusModule]) if(key.FrameDecoded): taskCount += int(tokens[key.FrameDecoder]) if(key.FrameProcessor): taskCount += int(tokens[key.FrameProcessor]) if(key.GlobalVirtualRefStnModu): taskCount += int(tokens[key.GlobalVirtualRefStnModu]) if(key.RefDataPrepModule): taskCount += int(tokens[key.RefDataPrepModule]) if(key.EphemerisModule): taskCount += int(tokens[key.EphemerisModule]) if(key.TN_ITF): taskCount += int(tokens[key.TN_ITF]) if(key.TN_FDEC): taskCount += int(tokens[key.TN_FDEC]) if(key.TN_FPRO): taskCount += int(tokens[key.TN_FPRO]) if(key.TN_EPH): taskCount += int(tokens[key.TN_EPH]) if(key.TN_GVRS): taskCount += int(tokens[key.TN_GVRS]) if(key.TN_ANAVP): taskCount += int(tokens[key.TN_ANAVP]) if(key.TN_NAVDP): taskCount += int(tokens[key.TN_NAVDP]) if(key.TN_ROVDP): taskCount += int(tokens[key.TN_ROVDP]) if(key.TN_REFDP): taskCount += int(tokens[key.TN_REFDP]) if(key.TN_DYNS): taskCount += int(tokens[key.TN_DYNS]) if(key.TN_TE): taskCount += int(tokens[key.TN_TE]) if(key.TN_PE): taskCount += int(tokens[key.TN_PE]) if(key.TN_XFILL): taskCount += int(tokens[key.TN_XFILL]) if(key.TN_AHRS): taskCount += int(tokens[key.TN_AHRS]) if(key.TN_FM): taskCount += int(tokens[key.TN_FM]) if(key.TN_NAV): taskCount += int(tokens[key.TN_NAV]) if(key.TN_MNAV): taskCount += int(tokens[key.TN_MNAV]) if(key.TN_COL): taskCount += int(tokens[key.TN_COL]) if(key.TN_CDIFDP): taskCount += int(tokens[key.TN_CDIFDP]) if(key.TN_CONTROL): taskCount += int(tokens[key.TN_CONTROL]) if(key.TNxFILL): taskCount += int(tokens[key.TNxFILL]) if(key.TN_CBC): taskCount += int(tokens[key.TN_CBC]) # Optional modules ... some of these modules changed names # during the Titan development if(key.AmbiguitySearchModule): taskCount = taskCount + int(tokens[key.AmbiguitySearchModule]) if(key.PreciseEstimatorModule): taskCount = taskCount + int(tokens[key.PreciseEstimatorModule]) if(key.PreciseEstimationModule): taskCount = taskCount + int(tokens[key.PreciseEstimationModule]) if(key.PreciseGNSSProcessorModu): taskCount = taskCount + int(tokens[key.PreciseGNSSProcessorModu]) if(key.XFillModule): taskCount = taskCount + int(tokens[key.XFillModule]) if(key.TimelyEstimatorModule): taskCount = taskCount + int(tokens[key.TimelyEstimatorModule]) if(key.TimelyEstimationModule): taskCount = taskCount + int(tokens[key.TimelyEstimationModule]) if(key.TimelyProcessorModule): taskCount = taskCount + int(tokens[key.TimelyProcessorModule]) if(key.PreciseProcessorModule): taskCount = taskCount + int(tokens[key.PreciseProcessorModule]) if(key.FilterManagerModule): taskCount = taskCount + int(tokens[key.FilterManagerModule]) if(key.AHRSModule): taskCount = taskCount + int(tokens[key.AHRSModule]) if(key.CollatorModule): taskCount = taskCount + int(tokens[key.CollatorModule]) if(key.NavEngineControlThread): taskCount = taskCount + int(tokens[key.NavEngineControlThread]) if(key.NavDataPrepModule): taskCount = taskCount + int(tokens[key.NavDataPrepModule]) if(key.ANAVDataPrepModule): taskCount = taskCount + int(tokens[key.ANAVDataPrepModule]) if(key.TNxFILL): taskCount = taskCount + int(tokens[key.TNxFILL ]) if(key.TitanSingleThread): taskCount = taskCount + int(tokens[key.TitanSingleThread]) if(key.NavigatorModule): taskCount = taskCount + int(tokens[key.NavigatorModule]) if(key.CodeDifDataPrepModule): taskCount = taskCount + int(tokens[key.CodeDifDataPrepModule]) if(key.TN_CBC): taskCount = taskCount + int(tokens[key.TN_CBC]) elif(LoadType == StingerTasks): # Stinger taskCount = ( int(tokens[key.StingerPF]) + int(tokens[key.StingerLM]) + int(tokens[key.StingerAM]) + int(tokens[key.StingerMM]) + int(tokens[key.StingerPF]) + int(tokens[key.StingerNM]) + int(tokens[key.StingerTS]) + int(tokens[key.StingerSD]) + int(tokens[key.StingerLMBackground])) # New background MM task for neural net processing if(key.StingerMMbackground): taskCount += int(tokens[key.StingerMMbackground]) # Single core BM and old primary core BM if(key.BM): taskCount = taskCount + int(tokens[key.BM]) # Core 0 - 3 BM load for modern multi-core builds if(key.BM0): taskCount = taskCount + int(tokens[key.BM0]) if(key.BM1): taskCount = taskCount + int(tokens[key.BM1]) if(key.BM2): taskCount = taskCount + int(tokens[key.BM2]) if(key.BM3): taskCount = taskCount + int(tokens[key.BM3]) # Old 2nd core BM if(key.BM_SLAVE): taskCount = taskCount + int(tokens[key.BM_SLAVE]) if(key.Stingertrace): taskCount = taskCount + int(tokens[key.Stingertrace]) if(key.StingerFFTacq): taskCount = taskCount + int(tokens[key.StingerFFTacq]) if(key.AntiJamTask): taskCount = taskCount + int(tokens[key.AntiJamTask]) if(key.StingerMMLDPC): taskCount = taskCount + int(tokens[key.StingerMMLDPC]) elif(LoadType == BackgroundTasks): # Background taskCount = int(tokens[key.StingerBIT]) if(key.CheetahBackgroundTest): taskCount += int(tokens[key.CheetahBackgroundTest]) if(key.CoreBuildBackgroundTest): taskCount += int(tokens[key.CoreBuildBackgroundTest]) if(key.CoreBuildBackground): taskCount += int(tokens[key.CoreBuildBackground]) # Dual core background if(key.StingerBITCPU1): taskCount = taskCount + int(tokens[key.StingerBITCPU1]) # 3rd core if(key.StingerBITCPU2): taskCount = taskCount + int(tokens[key.StingerBITCPU2]) # 4th core if(key.StingerBITCPU3): taskCount = taskCount + int(tokens[key.StingerBITCPU3]) if(key.CheetahBackgroundTestCPU1): taskCount = taskCount + int(tokens[key.CheetahBackgroundTestCPU1]) if(key.CoreBuildBackgroundTestCPU1): taskCount = taskCount + int(tokens[key.CoreBuildBackgroundTestCPU1]) if(key.CoreBuildBackgroundCPU1): taskCount = taskCount + int(tokens[key.CoreBuildBackgroundCPU1]) elif(LoadType == 3): # BM taskCount = 0 # Single core BM and old primary core BM if(key.BM): taskCount = taskCount + int(tokens[key.BM]) # Core 0 - 3 BM load for modern multi-core builds if(key.BM0): taskCount = taskCount + int(tokens[key.BM0]) if(key.BM1): taskCount = taskCount + int(tokens[key.BM1]) if(key.BM2): taskCount = taskCount + int(tokens[key.BM2]) if(key.BM3): taskCount = taskCount + int(tokens[key.BM3]) # Old 2nd core BM if(key.BM_SLAVE): taskCount = taskCount + int(tokens[key.BM_SLAVE]) elif(LoadType == 4): # LM taskCount = int(tokens[key.StingerLM]) elif(LoadType == 5): # MM taskCount = int(tokens[key.StingerMM]) elif(LoadType == 6): # PVT taskCount = int(tokens[key.StingerPF]) elif(LoadType == 7): # LDPC if(key.StingerMMLDPC): taskCount = int(tokens[key.StingerMMLDPC]) else: taskCount = 0 elif(LoadType == 8): # Anti-Jam if(key.AntiJamTask): taskCount = int(tokens[key.AntiJamTask]) else: taskCount = 0 elif(LoadType == 9): # Stinger FFT if(key.StingerFFTacq): taskCount = int(tokens[key.StingerFFTacq]) else: taskCount = 0 elif(LoadType == 10): # Stinger NM taskCount = int(tokens[key.StingerNM]) elif(LoadType == 11): # Stinger TS taskCount = int(tokens[key.StingerTS]) elif(LoadType == 12): # Stinger SD taskCount = int(tokens[key.StingerSD]) else: # Stinger AM taskCount = int(tokens[key.StingerAM]) # load as percentage, difference between current and last counts load = 100.0 * float(taskCount - lastCount[LoadType])/float(countSum - lastSum) # Store the load and time information loadPerc[LoadType].append(load) # Update for next time lastCount[LoadType] = taskCount lastSum = countSum f.close() print("Processed the load data") ################## # -- Now plot the data ################## RTK = array(loadPerc[RTKTasks]) Stinger = array(loadPerc[StingerTasks]) Background = array(loadPerc[BackgroundTasks]) App = 100 - RTK - Stinger - Background print("Stinger Load Summary") print("Stinger Mean = %.2f" % mean(loadPerc[StingerTasks])) print("Stinger BM = %.2f" % mean(loadPerc[3])) print("Stinger LM = %.2f" % mean(loadPerc[4])) print("Stinger MM = %.2f" % mean(loadPerc[5])) print("Stinger PVT = %.2f" % mean(loadPerc[6])) print("Stinger LDPC = %.2f" % mean(loadPerc[7])) print("Stinger AJ = %.2f" % mean(loadPerc[8])) print("Stinger Acq = %.2f" % mean(loadPerc[9])) print("Stinger NM = %.2f" % mean(loadPerc[10])) print("Stinger TS = %.2f" % mean(loadPerc[11])) print("Stinger SD = %.2f" % mean(loadPerc[12])) print("Stinger AM = %.2f" % mean(loadPerc[13])) fid = open(resDir + '/loadStats.txt','a') fid.write("%d %d %d %.3f %.3f %.3f %.3f %.3f %.3f\n" % (year, month, day, mean(Trk), mean(used), mean(Stinger), mean(App), mean(RTK), mean(Background))) fid.close() figure(figsize=(20,10)) plot( array(time)/3600.0, Stinger, label='Stinger') plot( array(time)/3600.0, App, label='App') plot( array(time)/3600.0, RTK, label='RTK') plot( array(time)/3600.0, Background, label='Reserve') plot( T/3600.0, Trk, label='SVs Tracked') plot( T/3600.0, used, label='SVs Used') xlabel('Time [Hour]') ylabel('Load [%]') grid() legend() tight_layout() show() # Save the data as a PNG file savefig(outputRoot + ".Loading.png",dpi=150) close() # 0 == RTK tasks # 1 == Stinger (inclding BM) # 2 == Background # 3 == Stinger BM # 4 == Stinger LM # 5 == Stinger MM # 6 == Stinger PVT # 7 == Stinger LDPC # 8 == Stinger Anti-Jam # 9 == Stinger Acq # 10 == Stinger NM # 11 == Stinger TS # 12 == Stinger SD # 13 == Stinger AM BM = array(loadPerc[3]) LM = array(loadPerc[4]) MM = array(loadPerc[5]) PVT = array(loadPerc[6]) LDPC = array(loadPerc[7]) AJ = array(loadPerc[8]) ACQ = array(loadPerc[9]) NM = array(loadPerc[10]) TS = array(loadPerc[11]) SD = array(loadPerc[12]) AM = array(loadPerc[13]) fid = open(resDir + '/loadStinger.txt','a') fid.write("%d %d %d %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f\n" % (year, month, day, mean(Trk), mean(used), mean(BM), mean(LM), mean(MM), mean(PVT), mean(LDPC), mean(AJ), mean(ACQ), mean(NM), mean(TS), mean(SD), mean(AM))) fid.close() # We run out of the built in colours so generate our own cmap = get_cmap('jet') NumColours = 11 colors = cmap(linspace(0, 1.0, NumColours)) figure(figsize=(20,10)) plot( array(time)/3600.0, BM, label='BM',color=colors[9]) plot( array(time)/3600.0, LM, label='LM',color=colors[10]) plot( array(time)/3600.0, MM, label='MM',color=colors[8]) plot( array(time)/3600.0, PVT, label='PVT',color=(1,0,1)) plot( array(time)/3600.0, ACQ, label='ACQ',color=colors[5]) plot( array(time)/3600.0, LDPC, label='LDPC',color=(0.75,0.75,0.75)) plot( array(time)/3600.0, AJ, label='AJ',color='k') plot( array(time)/3600.0, NM, label='NM',color=colors[4]) #plot( array(time)/3600.0, TS, label='TS',color=colors[6]) plot( array(time)/3600.0, TS, label='TS',color=(1,1,0)) plot( array(time)/3600.0, SD, label='SD',color=colors[7]) plot( array(time)/3600.0, AM, label='AM',color=colors[0]) line = plot( T/3600.0, Trk, label='SVs Tracked',color='k') #line[0].set_linestyle(':') line[0].set_linewidth(5) line = plot( T/3600.0, used, label='SVs Used',color='orange') #line[0].set_linestyle(':') line[0].set_linewidth(5) xlabel('Time [Hour]') ylabel('Load [%]') grid() legend() tight_layout() show() # Save the data as a PNG file savefig(outputRoot + ".Stinger.png",dpi=150) close() ################################################# # # Now find the average load as a function of satellites in the PVT # ################################################# ArrTime = rint(time) TimeList = [] SVsUsed = [] TitanLoad = [] StingerTimeList = [] StingerSVsTracked= [] StingerLoad = [] StingerAppLoad = [] ReserveLoad = [] #RTKLoad = [] print("find SVs") # load as a function of satellites used for i in range(int(T[0]),int(T[-1])): loadIndex = find(ArrTime == i) if(loadIndex.size > 0): timeindex = find(T == i) if( timeindex.size > 0 ): StingerTimeList.append(i) StingerLoad.append(loadPerc[StingerTasks][loadIndex[0]]) appStinger = ( 100 - loadPerc[BackgroundTasks][loadIndex[0]] - loadPerc[RTKTasks][loadIndex[0]]) StingerAppLoad.append(appStinger) StingerSVsTracked.append(Trk[timeindex[0]]) ReserveLoad.append( loadPerc[BackgroundTasks][loadIndex[0]]) SVsUsed.append(used[timeindex[0]]) if(len(StingerTimeList) > 0): fid = open(resDir + '/loadStingerPerSV.txt','a') fid.write("%d %d %d %.3f %.3f " % (year, month, day, mean(Trk), mean(used))) ArrTime = array(StingerTimeList) ArrSVs = array(StingerSVsTracked) ArrStingerLoad = array(StingerLoad) ArrStingerAppLoad = array(StingerAppLoad) ArrReserveLoad = array(ReserveLoad) # Stinger load SVLoad = [] SVLoadPer = [] SVReserve = [] for i in range(0,100): j = find(ArrSVs == i) # make sure we have at least 10 epochs if(len(j) > 10): SVLoad.append(sum(ArrStingerLoad[j]) / len(j)) # Load per SV SVLoadPer.append( (sum(ArrStingerLoad[j]) / len(j)) / i ) SVReserve.append(sum(ArrReserveLoad[j]) / len(j)) fid.write("%.3f " % SVLoad[-1]) else: SVLoad.append(float('nan')) SVLoadPer.append(float('nan')) SVReserve.append(float('nan')) fid.write("nan ") fid.write("\n") fid.close() plot(SVLoad,'bo') plot(SVLoad,'k') xlabel('Number of Satellites Tracked') ylabel('Stinger Load [%]') grid(True) tight_layout() show() savefig(outputRoot + ".StingerLoadingvsSVs.png",dpi=150) close() fid = open(resDir + '/loadStingerAppPerSV.txt','a') fid.write("%d %d %d %.3f %.3f " % (year, month, day, mean(Trk), mean(used))) # Stinger+App load SVLoad = [] SVLoadPer = [] SVReserve = [] for i in range(0,100): j = find(ArrSVs == i) # make sure we have at least 10 epochs if(len(j) > 10): SVLoad.append(sum(ArrStingerAppLoad[j]) / len(j)) # Load per SV SVLoadPer.append( (sum(ArrStingerAppLoad[j]) / len(j)) / i ) SVReserve.append(sum(ArrReserveLoad[j]) / len(j)) fid.write("%.3f " % SVLoad[-1]) else: SVLoad.append(float('nan')) SVLoadPer.append(float('nan')) SVReserve.append(float('nan')) fid.write("nan ") fid.write("\n") fid.close() plot(SVLoad,'bo') plot(SVLoad,'k') xlabel('Number of Satellites Tracked') ylabel('Stinger + App Load [%]') grid(True) tight_layout() show() savefig(outputRoot + ".StingerAppLoadingvsSVs.png",dpi=150) close() if(__name__ == "__main__"): # Defaults until these are set ftpData = False localData = False genFileName = 99 today = True cacheLoadFiles = True removeT04 = False date = [] ###################################################################### # Parse arguments # # The script has three fundamental operating modes: # # 1. Provide a single file "--file" # 2. Process data from a single known station (e.g. --btnjam) # 3. Process all known stations (default) # # For options 2 and 3 a JSON file (receivers.json) is read which defines # the data location, how to get it (FTP etc) format etc for each station. # By default options 2 and 3 will process yesterday's data based on Linux # time (e.g. if it is the 20th June based on the Linux local time clock # we will process data from the 19th GPS time). There is an optional # parameter "--date YYYY,MM,DD" which provides a start date, the script # will iterate for each day from that date to the current date attempting # to process the data. The --date option is used if a new station is added # and we want to process old data or if the analysis is changed and we want # to reprocess the data. parser = argparse.ArgumentParser(description='Extracts CPU loading information from a T04 file') parser.add_argument('-f','--file', help='Parse a single file - provide the filename --file myFile.T04') parser.add_argument('-d','--date', help='Optional start date year,month,day e.g. --date YYYY,MM,DD') parser.add_argument('-j','--btnjam', help='Process CO BTN-Jam station data', action="store_true") parser.add_argument('-m','--melb', help='Process Melbourne data', action="store_true") parser.add_argument('-s','--sing', help='Process Singapore data', action="store_true") parser.add_argument('-l','--svale', help='Process Sunnyvale BD940 data', action="store_true") parser.add_argument('-k','--bd992', help='Process Sunnyvale BD992 data', action="store_true") parser.add_argument('-b','--beijastra', help='Process Beijing Astra control', action="store_true") parser.add_argument('-t','--beijtitan', help='Process Beijing Titan Test', action="store_true") parser.add_argument('-n','--beijnetr9', help='Process Beijing NetR9', action="store_true") args = parser.parse_args() ###################################################################### if(args.file): MaxRX = 1 else: # Load the JSON file with open('receivers.json', 'r') as f: receivers = json.load(f) if(args.date): if(args.file): print("Can't add a date if you provide a file for processing") sys.exit(1) date = args.date.split(",") today = False if(args.file): localDir = "." filename = args.file.split('.')[0] ext = args.file.split('.')[-1] removeT04 = False cacheLoadFiles = False fileSource = "CLI" else: lst = receivers['receivers'] index = None if(args.melb): print("Checking Melb") # BD935 in Melbourne index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'Melb-BD935'][0] elif(args.sing): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'Sing-BD935'][0] elif(args.svale): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'BD940-1'][0] elif(args.bd992): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'BD992'][0] elif(args.btnjam): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'BTN-Jam'][0] elif(args.beijastra): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'Beijing-BD935'][0] elif(args.beijtitan): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'Beijing-BD935-2'][0] elif(args.beijnetr9): index = [i for i,_ in enumerate(lst) if _['LocalDir'] == 'Beijing-NetR9'][0] if(index is None): # Process the full json file MaxRX = len(lst) index = 0 else: MaxRX = 1 for rx in range(MaxRX): if(not args.file): localDir = lst[index]['LocalDir'] RemoteDir = lst[index]['DataDir'] ext = lst[index]['ext'] fileSource = lst[index]['fileSource'] if(lst[index]['removeFile'] == 'Yes'): removeT04 = True else: removeT04 = False if('prefix' in lst[index]): prefix = lst[index]['prefix'] else: prefix = [] if(not os.path.exists(localDir)): os.makedirs(localDir) # Move to the next receiver ready for the next loop index += 1 print(localDir) if(today == True): # Many of the files aren't ready until the following day so # actually process yesterday's data start = datetime.datetime.now() - datetime.timedelta(days=1) num = 1 else: start = datetime.datetime(int(date[0]),int(date[1]),int(date[2]),0,0,0) stop = datetime.datetime.now() delta = ((stop-start).total_seconds()/86400) num = int(math.ceil(delta)) for i in range(num): maxTokens = -1 # Now get the file - depending on the source we may simply point to # it, copy it, concat it, FTP or HTTP the data if(fileSource == "NoPi"): ############################################# # Simply need to point to a NoPi T04 day file ############################################# filename = ( (RemoteDir + '%4d/%4d%02d%02d/' + prefix + '%4d%02d%02d') % (start.year,start.year,start.month,start.day,start.year,start.month,start.day)) elif(fileSource == "CLI"): ############################################### # The file name was passed on the command line ############################################### pass elif(fileSource == "BTN"): ############################################# # Need to download via HTTP BTN files ############################################# filename = ( '%s/%4d%02d%02d' % (localDir,start.year,start.month,start.day)) # Get the file if(os.path.isfile(filename + '.' + ext) and (os.path.getsize(filename + '.' + ext) > 0)): print("Already downloaded the file - skipping HTTP") else: # Download the file via HTTP from Geoffrey's server remoteFile = str(start.year) + "_" + str(start.month).zfill(2) + "_" + str(start.day).zfill(2) + "." + ext localFile = filename + "." + ext for hour in range(24): fileRemote = ("%s%4d%02d%02d%02d.T04" % (prefix,start.year,start.month,start.day,hour)) fileURL = (RemoteDir + "%s" % fileRemote) print(fileURL) # This is copy and paste - don't need the login. ToDo review how to # remove this. password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, 'gnssplot.eng.trimble.com', "admin", "password") handler = HTTPBasicAuthHandler(password_manager) try: opener = build_opener(handler) filehandle = opener.open(fileURL) with open(filename + str(hour).zfill(2) + '.' + ext,'wb') as output: output.write(filehandle.read()) os.system( "cat " + filename + str(hour).zfill(2) + "." + ext + " >> " + filename + ".T04") os.system( "rm " + filename + str(hour).zfill(2) + "." + ext) except: print("Missing " + fileRemote) elif(fileSource == "COFTP"): ############################### # FTP from Dave's server in CO ############################### filename = ( '%s/%4d%02d%02d' % (localDir,start.year,start.month,start.day)) getFileFTP(filename, "dave.trimble.com", "/testing/MON_" + str(start.year) + "/" + prefix + "/" + str(start.year) + "/" + str(start.month).zfill(2) + "/", start.year, start.month, start.day, ext) else: # "Local" file type ############################################################### # Need to concat into a day file for the rest of the stations ############################################################### filename = ( '%s/%4d%02d%02d' % (localDir,start.year,start.month,start.day)) # Make sure there isn't already an output file as we'll concat to this file os.system( "rm " + localDir + "/" + str(start.year) + str(start.month).zfill(2) + str(start.day).zfill(2) + "." + ext) # Check for a 24 hour file remoteFile = ( RemoteDir + str(start.year) + str(start.month).zfill(2) + str(start.day).zfill(2) + "." + ext) if(os.path.isfile(remoteFile) and (os.path.getsize(remoteFile) > 0)): # We occassionally generated day files print("Got Remote") os.system("cp " + remoteFile + " " + localDir) else: # Concat the 10 minute files into a local T04 file for hour in range(24): for minIndex in range(6): os.system( "cat " + RemoteDir + "*" + str(start.year) + str(start.month).zfill(2) + str(start.day).zfill(2) + str(hour).zfill(2) + str(minIndex*10).zfill(2) + "*.T04 >> " + localDir + "/" + str(start.year) + str(start.month).zfill(2) + str(start.day).zfill(2) + ".T04") print(filename) output = ( '%s/CPU_%4d%02d%02d' % (localDir,start.year,start.month,start.day)) print("%d %02d %02d %s %s" % (start.year,start.month,start.day,filename,output)) processFile(filename, output, localDir, start.year, start.month, start.day,cacheLoadFiles) # Now clean up if requested if(removeT04 == True): print("Removing File") os.system("rm " + filename + "." + ext) start = start + datetime.timedelta(days=1) print("Finished")