# This script requires "matplotlib" it also requires "mutils.py" which # can be found in CVS GPSTools/pythontools/ it requires pandas # # Copyright Trimble 2016 # # Concanenates 10 minute T04 files from the current day and plots the load from the # T04 load diagnostic records # import matplotlib # Allow running headless from the command line matplotlib.use("agg") # For headless/scripting operation: # yum install xorg-x11-server-Xvfb from numpy import * from pylab import * import mutils as m import os; import sys; class dotdict(dict): def __getattr__(self, name): try: return self[name] except KeyError: return if(len(sys.argv) != 3): print("Usage:\n python makeDayLoad.py serialnumber titleString\n") exit(1) serialnumber = sys.argv[1] titleString = sys.argv[2] #start = datetime.date.today() + datetime.timedelta(days=1) start = datetime.date.today() day = start.day month = start.month year = start.year YYYYMMDD = str(year) + str(month).zfill(2) + str(day).zfill(2) # Set to zero to skip the cat file section, assumes you already have the # file available CatFiles = 1 OrigFilename = YYYYMMDD + ".T04" if CatFiles: os.system("rm " + YYYYMMDD + ".T04") for hour in range(0,24): for minute in range(0,60,10): HHMM = str(hour).zfill(2) + str(minute).zfill(2) Filename = serialnumber + YYYYMMDD + HHMM + ".T04" if(os.path.isfile(Filename) == True): os.system("cat " + Filename + " >> " + YYYYMMDD + ".T04") filename = "Dec_" + OrigFilename print("Decimating " + OrigFilename) # As we are going to scan the data multiple times remove all records except the ones we want os.system("t0x2t0x -pos_dec=1 -i35:2,35:258,35:259 " + OrigFilename + " " + filename); print("Processing " + filename) ################## # --- Get the number of satellites tracked / used from the position record ################## (ref,k)=m.vd2arr(filename,rec='-d29',opt=['--dec=1000']) T = ref[:,k.TIME] Trk = ref[:,k.NTRK] used = ref[:,k.NUSED] print("Processed the satellite tracked/used data") ################## # --- Build up the task key # # Once this is setup can access the tasks based on a name, e.g. # key.StingerPF will give the index into the type 258 ################## gotKeyStart = False key = dotdict({}) FTPDServerCount = 0 EthWriteCount = 0 HTTPDCount = 0 os.system("viewdat -d35:259 " + filename + " > 259.all") f = open("259.all","r") for line in f.readlines(): tokens = line.split() if(gotKeyStart == False): if(len(tokens) == 2 and tokens[0] == "Task"): gotKeyStart = True else: if( len(tokens) > 2 and tokens[0] == "TYPE" and tokens[1] == "35:"): break if(len(tokens) == 4): keyStr = tokens[3].rstrip() # Remove return # Some tasks have multiple instances that are not unique, add a # number to the end of the task number if(keyStr == "FTPDserver"): keyStr = keyStr + str(FTPDServerCount) FTPDServerCount = FTPDServerCount + 1 if(keyStr == "Ethernetwritetask"): keyStr = keyStr + str(EthWriteCount) EthWriteCount = EthWriteCount + 1 if(keyStr == "HTTPDworker"): keyStr = keyStr + str(HTTPDCount) HTTPDCount = HTTPDCount + 1 # +1 as we want this to index into the 35:258 record, the first # column in that record is time key[keyStr] = int(tokens[0]) + 1; if(keyStr == "BM" or keyStr == "ISR" or keyStr == "OS"): # BM ISR, Misc ISRs and the OS do not have valid FE IDs print("Key." + str(keyStr) + " = " + str(key[keyStr]) + " core = " + tokens[2]) else: print("Key." + str(keyStr) + " = " + str(key[keyStr]) + " core = " + tokens[2] + " FE ID = " + tokens[1]) f.close() print("Processed the Key data") ################## # -- Read the load data and compute the task loading ################## os.system("viewdat -d35:258 " + filename + " > 258.all") # List containing the time tags time = [] # List containing the load as a percent loadPerc = [] lastCount = [] NumTypes = 27 for LoadType in range(0,NumTypes): loadPerc.append([]) lastCount.append(0) lastSum = 0; # Which RTK engine do we have? HaveAstra = False HaveTitan = False with open("258.all") as f: for line in f: tokens = line.split() if(len(tokens) > 50): countSum = 0; # Start from 1 to skip to time for x in range(1,len(tokens)): countSum = countSum + int(tokens[x]) time.append(float(tokens[0])/1000.0) # --- Loop around collecting what we want to plot # 0 == Astra RTK tasks # 1 == Stinger PVT # 2 == Background # 3 == BM # 4 == LM # 5 == MM # 6 == File syste related tasks (logging, FTP push ...) # 7 == Titan RTK Tasks # # Major Titan tasks # 8 == PreTN # 9 == RovP # 10 == Interface # 11 == Nav # 12 == Timely # 13 == Auto # 14 == PKEE # 15 =- Amb # # Astra tasks # 16 == RTKDPPP # 17 == PreRTK # 18 == RTKFSS # # File system breakdown: # 19 - Flash Manager # 20 - File System Manager # 21 - Data Logger # 22 - Autodelte # 23 - FlashFileSystem # 24 - FTP Push # 25 - FTPD # 26 - NAND # New tasks here - make sure you bump "NumTypes" # # The code below gives examples of how to extract individual tasks # as well as combine tasks to get a load number. # # If you want to look at more groups of tasks or individual tasks # just add new "LoadType" values below. # # *** Make sure you also increase "NumTypes" to match the maximum # number of types of data you want to extract, this will dimension # the lists appropriately. for LoadType in range(0,NumTypes): if(LoadType == 0): if(key.RTKDPPP): # We have Astra tasks HaveAstra = True taskCount = int(tokens[key.RTKDPPP]) + int(tokens[key.PreRTK]) + int(tokens[key.RTKFSS]) if(key.INSIN): # Not all systems will have INS-RTK tasks taskCount = taskCount + int(tokens[key.INSIN]) + int(tokens[key.INSINKF]) else: taskCount = 0 elif(LoadType == 1): taskCount = int(tokens[key.StingerPF]) elif(LoadType == 2): taskCount = int(tokens[key.StingerBIT]) elif(LoadType == 3): taskCount = int(tokens[key.BM]) elif(LoadType == 4): taskCount = int(tokens[key.StingerLM]) elif(LoadType == 5): taskCount = int(tokens[key.StingerMM]) elif(LoadType == 6): # File system related loading taskCount = ( int(tokens[key.FlashManager]) + int(tokens[key.FileSystemManager]) + int(tokens[key.DataLogger]) + int(tokens[key.DLAutoDelete]) + int(tokens[key.DLADScanner]) + int(tokens[key.FlashFileSystem]) + int(tokens[key.NAND]) + int(tokens[key.FTPPush]) + int(tokens[key.FTPDmain]) + int(tokens[key.FTPDserver0]) + int(tokens[key.FTPDserver1]) + int(tokens[key.FTPDserver2]) ) elif(LoadType == 7): if(key.PreTN): HaveTitan = True # As of 2016-06-16 this is all the Titan tasks, if new tasks # are added this needs to be updated! taskCount = ( int(tokens[key.PreTN]) + int(tokens[key.Interface]) + int(tokens[key.NavigatorModule]) + int(tokens[key.RovDataPrepModule]) + int(tokens[key.DynamicStatusModule]) + int(tokens[key.TimelyEstimatorModule]) + int(tokens[key.GNSSAutonomousPVTModule]) + int(tokens[key.FrameDecoder]) + int(tokens[key.FrameProcessor]) + int(tokens[key.GlobalVirtualRefStnModu]) + int(tokens[key.RefDataPrepModule]) + int(tokens[key.PreciseEstimatorModule]) + int(tokens[key.AmbiguitySearchModule]) + int(tokens[key.EphemerisModule]) ) else: taskCount = 0 elif(LoadType == 8): if(key.PreTN): taskCount = int(tokens[key.PreTN]) else: taskCount = 0 elif(LoadType == 9): if(key.PreTN): taskCount = int(tokens[key.RovDataPrepModule]) else: taskCount = 0 elif(LoadType == 10): if(key.PreTN): taskCount = int(tokens[key.Interface]) else: taskCount = 0 elif(LoadType == 11): if(key.PreTN): taskCount = int(tokens[key.NavigatorModule]) else: taskCount = 0 elif(LoadType == 12): if(key.PreTN): taskCount = int(tokens[key.TimelyEstimatorModule]) else: taskCount = 0 elif(LoadType == 13): if(key.PreTN): taskCount = int(tokens[key.GNSSAutonomousPVTModule]) else: taskCount = 0 elif(LoadType == 14): if(key.PreTN): taskCount = int(tokens[key.PreciseEstimatorModule]) else: taskCount = 0 elif(LoadType == 15): if(key.PreTN): taskCount = int(tokens[key.AmbiguitySearchModule]) else: taskCount = 0 elif(LoadType == 16): if(key.RTKDPPP): taskCount = int(tokens[key.RTKDPPP]) else: taskCount = 0 elif(LoadType == 17): if(key.RTKDPPP): taskCount = int(tokens[key.PreRTK]) else: taskCount = 0 elif(LoadType == 18): if(key.RTKDPPP): taskCount = int(tokens[key.RTKFSS]) else: taskCount = 0 elif(LoadType == 19): taskCount = int(tokens[key.FlashManager]) elif(LoadType == 20): taskCount = int(tokens[key.FileSystemManager]) elif(LoadType == 21): taskCount = int(tokens[key.DataLogger]) elif(LoadType == 22): taskCount = int(tokens[key.DLAutoDelete]) + int(tokens[key.DLADScanner]) elif(LoadType == 23): taskCount = int(tokens[key.FlashFileSystem]) elif(LoadType == 24): taskCount = int(tokens[key.FTPPush]) elif(LoadType == 25): taskCount = ( int(tokens[key.FTPDmain]) + int(tokens[key.FTPDserver0]) + int(tokens[key.FTPDserver1]) + int(tokens[key.FTPDserver2]) ) else: # (LoadType == 26): taskCount = int(tokens[key.NAND]) # load as percentage, difference between current and last counts load = 100.0 * float(taskCount - lastCount[LoadType])/float(countSum - lastSum) # Store the load and time information loadPerc[LoadType].append(load) # Update for next time lastCount[LoadType] = taskCount lastSum = countSum f.close() print("Processed the load data") ################## # -- Now plot the data ################## # Force to plot Astra #HaveAstra = True if(HaveAstra == False): String = "Titan-8032" else: String = "Astra-8032" colList= ['b','g','#FFCCCC','m',"#555555",'k','#FF8C00','#FF99FF','#C0C0C0','#CCFFCC','r','c']; figure(figsize=(20,10)) if(HaveAstra == True): plot( array(time)/3600.0, array(loadPerc[0]), label='Astra RTK',color=colList[0]) else: plot( array(time)/3600.0, array(loadPerc[7]), label='Titan RTK',color=colList[0]) plot( array(time)/3600.0, array(loadPerc[1]), label='Stinger PVT',color=colList[5]) plot( array(time)/3600.0, array(loadPerc[3]), label='Stinger BM',color=colList[2]) plot( array(time)/3600.0, array(loadPerc[4]), label='Stinger LM',color=colList[3]) plot( array(time)/3600.0, array(loadPerc[6]), label='Logging/FS',color=colList[4]) plot( array(time)/3600.0, array(loadPerc[2]), label='Reserve',color=colList[1]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) xlabel('Time [Hour]') ylabel('Load [%] & Number of SVs') grid() legend() show() # Save the data as a PNG file title(titleString) savefig(OrigFilename + ".GeneralLoading.png",dpi=150,bbox_inches='tight') close() figure(figsize=(20,10)) plot( array(time)/3600.0, array(loadPerc[1]), label='Stinger PF',color=colList[5]) plot( array(time)/3600.0, array(loadPerc[3]), label='Stinger BM',color=colList[2]) plot( array(time)/3600.0, array(loadPerc[4]), label='Stinger LM',color=colList[3]) plot( array(time)/3600.0, array(loadPerc[5]), label='Stinger MM',color=colList[4]) plot( array(time)/3600.0, array(loadPerc[2]), label='Reserve',color=colList[1]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) xlabel('Time [Hour]') ylabel('Load [%] & Number of SVs') grid() legend() show() # Save the data as a PNG file title(titleString) savefig(OrigFilename + ".StingerLoading.png",dpi=150,bbox_inches='tight') close() figure(figsize=(20,10)) plot( array(time)/3600.0, array(loadPerc[19]), label='Flash Manager',color=colList[2]) plot( array(time)/3600.0, array(loadPerc[20]), label='File System Manager',color=colList[3]) plot( array(time)/3600.0, array(loadPerc[21]), label='Data Logger',color=colList[4]) plot( array(time)/3600.0, array(loadPerc[22]), label='Auto Delete',color=colList[5]) plot( array(time)/3600.0, array(loadPerc[23]), label='Flash File System',color=colList[6]) plot( array(time)/3600.0, array(loadPerc[24]), label='FTP Push',color=colList[7]) plot( array(time)/3600.0, array(loadPerc[25]), label='FTPD',color=colList[8]) plot( array(time)/3600.0, array(loadPerc[26]), label='NAND',color=colList[9]) plot( array(time)/3600.0, array(loadPerc[2]), label='Reserve',color=colList[1]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) xlabel('Time [Hour]') ylabel('Load [%] & Number of SVs') grid() legend() show() # Save the data as a PNG file title(titleString) savefig(OrigFilename + ".FileSystem.png",dpi=150,bbox_inches='tight') close() figure(figsize=(20,10)) if(HaveAstra == True): plot( array(time)/3600.0, array(loadPerc[0]), label='Astra RTK',color=colList[0]) else: plot( array(time)/3600.0, array(loadPerc[7]), label='Titan RTK',color=colList[0]) plot( array(time)/3600.0, array(loadPerc[2]), label='Reserve',color=colList[1]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) xlabel('Time [Hour]') ylabel('Load [%] & Number of SVs') grid() legend() show() # Save the data as a PNG file title(titleString) savefig(OrigFilename + ".RTKLoading." + String + ".png",dpi=150,bbox_inches='tight') close() figure(figsize=(20,10)) if(HaveAstra == True): plot( array(time)/3600.0, array(loadPerc[0]), label='Total Astra RTK',color=colList[0]) plot( array(time)/3600.0, array(loadPerc[2]), label='Sys Reserve',color=colList[1]) plot( array(time)/3600.0, array(loadPerc[16]), label='RTKDPPP',color=colList[2]) plot( array(time)/3600.0, array(loadPerc[17]), label='PreRTK',color=colList[3]) plot( array(time)/3600.0, array(loadPerc[18]), label='RTKFSS',color=colList[4]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) else: plot( array(time)/3600.0, array(loadPerc[7]), label='Total Titan RTK',color=colList[0]) plot( array(time)/3600.0, array(loadPerc[2]), label='Sys Reserve',color=colList[1]) plot( array(time)/3600.0, array(loadPerc[14]), label='PKEE',color=colList[2]) plot( array(time)/3600.0, array(loadPerc[12]), label='Timely',color=colList[3]) plot( array(time)/3600.0, array(loadPerc[9]), label='RovP',color=colList[4]) plot( array(time)/3600.0, array(loadPerc[11]), label='Navigator',color=colList[5]) plot( array(time)/3600.0, array(loadPerc[15]), label='Ambiguity',color=colList[6]) plot( array(time)/3600.0, array(loadPerc[8]), label='PreTN',color=colList[7]) plot( array(time)/3600.0, array(loadPerc[13]), label='Auto',color=colList[8]) plot( array(time)/3600.0, array(loadPerc[10]), label='Interface',color=colList[9]) plot( T/3600.0, Trk, label='SVs Tracked',color=colList[10]) plot( T/3600.0, used, label='SVs Used',color=colList[11]) xlabel('Time [Hour]') ylabel('Load [%] & Number of SVs') grid() legend() show() title(titleString) savefig(OrigFilename + ".RTKLoading." + String + "-Tasks.png",dpi=150,bbox_inches='tight') close() ################################################# # # Now find the average load as a function of satellites in the PVT # ################################################# ArrTime = rint(time) TimeList = [] SVsUsed = [] TitanLoad = [] print("find SVs") # load as a function of satellites used for i in range(int(T[0]),int(T[-1])): loadIndex = find(ArrTime == i) if(loadIndex.size > 0): timeindex = find(T == i) if(HaveAstra == False): SearchLoad = loadPerc[14][loadIndex[0]] else: SearchLoad = loadPerc[18][loadIndex[0]] # By testing PKEE we know that Titan is receiving corrections and doing something if( (SearchLoad > 1) and (timeindex.size > 0) ): TimeList.append(i) if(HaveAstra == False): TitanLoad.append(loadPerc[7][loadIndex[0]]) else: TitanLoad.append(loadPerc[0][loadIndex[0]]) SVsUsed.append(used[timeindex[0]]) if(len(TimeList) > 0): ArrTime = array(TimeList) ArrSVs = array(SVsUsed) ArrLoad = array(TitanLoad) SVLoad = [] SVLoadPer = [] for i in range(0,60): j = find(ArrSVs == i) # make sure we have at least 10 epochs if(len(j) > 10): SVLoad.append(sum(ArrLoad[j]) / len(j)) # Load per SV SVLoadPer.append( (sum(ArrLoad[j]) / len(j)) / i ) else: SVLoad.append(float('nan')) SVLoadPer.append(float('nan')) print(i,SVLoad[i]) plot(SVLoad,'bo') plot(SVLoad,'k') xlabel('Number of Satellites in PVT solution') ylabel(String + ' Load [%]') grid(True) show() title(titleString) savefig(OrigFilename + ".RTKLoading." + String + "-TasksVsSVs.png",dpi=150,bbox_inches='tight') close() plot(SVLoadPer,'bo') plot(SVLoadPer,'k') xlabel('Number of Satellites in PVT solution') ylabel(String + ' Load per SV [%]') grid(True) show() title(titleString) savefig(OrigFilename + ".RTKLoading." + String + "-TasksVsSVs-PerSV.png",dpi=150,bbox_inches='tight')