import matplotlib
# Allow running headless from the command line
matplotlib.use("agg")

import sys
import os
import glob
import time
import datetime
import signal
import os
import math
from numpy import *
from pylab import *

# Override pylab find() to avoid deprecation warning
# pylint: disable=function-redefined
def find(x):
    return where(x)[0]

logY = True

def plot_local(X,Y,marker,label=None):

  if((logY == False) or (sum(Y) == 0)):
    # If all the Y data is 0 don't attempt to plot a log scale
    ret = False
    plot(X,Y,marker,label=label)
  else:
    ret = True
    # Can't represent 0 in a log scale. Find these values, and set
    # to a low value
    i = find( Y == 0 )
    Y[i] = 0.01
    semilogy(X,Y,marker,label=label)

  return ret


fid = open("NoPiSlips.txt",'a')

#datetimeformat = "%Y-%m-%d %H:%M:%S"
#stop   = datetime.datetime.now()

# We run the day after the data is created so subract 1 day to get to
# the day we want to process.
start = datetime.datetime.now() - datetime.timedelta(days=1)

# Loop for each receiver data set we want to analyze
for combo in range(18):
  #start  = datetime.datetime.strptime("2018-02-01 12:00:00",datetimeformat)
  #delta = ((stop-start).total_seconds()/86400)

  # Loop for each day we want to analyze
  #for index in range((int)(ceil(delta))):
  if True:
    day   = start.day
    month = start.month
    year  = start.year
    #start = start + datetime.timedelta(days=1)

    print("get file: " + str(year) + str(month).zfill(2) + str(day).zfill(2) + " combo = " + str(combo))
    # Make sure an old file doesn't exist
    os.system(  "rm diffs_combo_" + str(combo) + ".mtb")
    fileroot = (  "/net/proton/mnt/data_drive2/WWW/GNSSResults/" 
                + str(year) + "-Res/" 
                + str(year) + str(month).zfill(2) + str(day).zfill(2) 
                + "_BD940_BD940_Short" )
    os.system(  "cp " + fileroot + "/diffs_combo_" + str(combo) + ".mtb.bz2 .")
    os.system(  "bzip2 -d diffs_combo_" + str(combo) + ".mtb.bz2")

    filename = 'diffs_combo_' + str(combo) + '.mtb'

    # Total number of unique slips, we try to find multi-epoch slips and
    # only bump this when the slip occurs rather than each epoch
    slipCount = 0
    # How many epochs are impacted by the slip
    slipEpochs = 0
    timeTag = 0

    # How many epochs of each cycle slip type?
    ClassicEpochs = 0
    DriftEpochs   = 0
    GangLuEpochs  = 0 
    UnknownEpochs = 0 

    # Store data for the current epoch (common timetags)
    currentEpoch_SV   = []
    currentEpoch_Carr = []
    currentEpoch_Time = []
    currentEpoch_Slip = []


    # Fifo for storing data (actually a circular buffer). Use this so we
    # can try to detect different types of slips
    lastEpoch_SV = []
    lastEpoch_Carr = []
    lastEpoch_Time = []
    lastEpoch_Slip = []

    fifoDepth = 30
    for i in range(fifoDepth):
      lastEpoch_SV.append([])
      lastEpoch_Carr.append([])
      lastEpoch_Time.append([])
      lastEpoch_Slip.append([])

    head = 0
    tail = 0 

    # Total measurements (as we are parsing double differences we add one
    # each time the time tag changes to account for the reference)
    TotalMeas   = 0
    # Total measurements less than 5 degrees
    TotalSub5   = 0
    # Total measurements 5-10 degrees
    Total5to10  = 0
    # Total measurements 10-15 degrees
    Total10to15 = 0

    slipsSub5   = 0
    slips5to10  = 0
    slips10to15 = 0

    if(os.path.isfile(filename)): 
      with open(filename,'r') as f:
        for line in f:
          data = line.rstrip().split()
          # We have data
          TotalMeas = TotalMeas + 1
          elev = int(data[3])

          if(elev < 5):
            TotalSub5 = TotalSub5 + 1
          elif((elev >= 5) and (elev < 10)):
            Total5to10 = Total5to10 + 1
          elif((elev >= 10) and (elev < 15)):
            Total10to15 = Total10to15 + 1

          # Is this a new time tag?
          if(int(data[0]) != timeTag):
            # Bump by one to account for the reference
            TotalMeas = TotalMeas + 1
            lastEpoch_SV[head]   = currentEpoch_SV
            lastEpoch_Carr[head] = currentEpoch_Carr
            lastEpoch_Time[head] = currentEpoch_Time
            lastEpoch_Slip[head] = currentEpoch_Slip
            head = (head + 1) % fifoDepth
            tail = (tail + 1) % fifoDepth

            currentEpoch_SV   = []
            currentEpoch_Carr = []
            currentEpoch_Time = []
            currentEpoch_Slip= []
            timeTag = int(data[0])

          carrPhase = float(data[7])
          currentEpoch_SV.append(int(data[1]))
          currentEpoch_Carr.append(carrPhase)
          currentEpoch_Time.append(int(data[0]))

          # Phase is greater than 1 cycle == a slip
          if( abs(carrPhase) > 0.75 ):
            # Analyze the first half of the fifo
            SumStart = 0
            NumStart = 0
            localTail = tail
            for j in range(int(fifoDepth/2)):
              for i in range(len(lastEpoch_SV[localTail])):
                 if(lastEpoch_SV[localTail][i] == int(data[1])):
                   SumStart = SumStart + lastEpoch_Carr[localTail][i]
                   NumStart = NumStart + 1
                   break;
              localTail = (localTail + 1) % fifoDepth

            # Analyze the second half of the fifo
            SumEnd = 0
            NumEnd = 0
            # Bump the tail so we'll look at the 2nd half of the buffer
            localTail = (tail + int(fifoDepth/2)) % fifoDepth
            for j in range(int(fifoDepth/2)):
              for i in range(len(lastEpoch_SV[localTail])):
                 if(lastEpoch_SV[localTail][i] == int(data[1])):
                   SumEnd = SumEnd + lastEpoch_Carr[localTail][i]
                   NumEnd = NumEnd + 1
                   break;
              localTail = (localTail + 1) % fifoDepth

            # Now find data from the last epoch, check whether we have data
            # for this satellite, if we do see if we had a slip last time.
            # GotSlip has a count of consecutive epochs from this satellite
            # with a slip
            localTail = (tail + fifoDepth - 1) % fifoDepth
            GotSlip = 0
            for i in range(len(lastEpoch_SV[localTail])):
               if(lastEpoch_SV[localTail][i] == int(data[1])):
                 GotSlip = lastEpoch_Slip[localTail][i]
                 break;
 
            if(NumStart > 0):
              #print data[0],data[1],data[3],data[7],SumStart,NumStart,SumStart/NumStart,SumEnd,NumEnd,SumEnd/NumEnd 
              #print "Stuck Test ",abs(SumStart/NumStart),abs(SumEnd/NumEnd),GotSlip 
              if( (abs(carrPhase) > 2) and (GotSlip > 5) ):
                print(data[0], "Tracking Drift")
                DriftEpochs += 1

              # We are stuck with a cycle slip - let's assume this is a
              # classic slip 
              elif(     (abs(SumStart/NumStart) >= 0.8) 
                    and (abs(SumEnd/NumEnd) >= 0.8) 
                    and (GotSlip > 5) ):
                print(data[0], "ClassicSlip")
                ClassicEpochs += 1

              # Compare the current carrier phase with the first half of the
              # buffer, if we've jumped more than 0.5 cycles consider it a
              # classic slip
              elif( abs(carrPhase - (SumStart/NumStart)) > 0.5):
                print(data[0], "ClassicSlip")
                ClassicEpochs += 1

              # If the change from the beginning to the end of the buffer is
              # subtle this may be the "Gang Lu" drift effect
              # ToDo - we need an example and then we can tune the
              # detector
              elif((NumEnd > 0) and ( abs( (NumStart/NumStart) - (SumEnd/NumEnd) ) < 0.1) ):
                print(data[0], "Gang Lu effect")
                GangLuEpochs += 1

              # Unknown slip type
              else:
                print(data[0], "Unknown")
                UnknownEpochs += 1
            else:
              # Assume a classic slip as we don't have much data in the FIFO
              print(data[0], "Assume ClassicSlip")
              ClassicEpochs = ClassicEpochs + 1
              # print data[0],data[1],data[3],data[7]


            GotSlip = GotSlip + 1
            if(GotSlip == 1):
              slipCount += 1
        
            slipEpochs += 1
            if(elev < 5):
              slipsSub5 += 1
            elif((elev >= 5) and (elev < 10)):
              slips5to10 += 1
            elif((elev >= 10) and (elev < 15)):
              slips10to15 += 1

            currentEpoch_Slip.append(GotSlip)
          else:
            currentEpoch_Slip.append(0)


    if(TotalMeas > 0):
      fid.write("%s %d %d %d %d %d %d %d %d %d %d %d %d %d %d\n" % 
              ( fileroot,
                combo,
                TotalMeas,
                TotalSub5,
                Total5to10,
                Total10to15,
                slipsSub5,
                slips5to10,
                slips10to15,
                slipCount,
                slipEpochs,
                ClassicEpochs,
                DriftEpochs,
                GangLuEpochs, 
                UnknownEpochs))
      print("SlipEpochs", (slipEpochs * 1e6) / TotalMeas, "PPM")
      print("SlipCount",  (slipCount * 1e6) / TotalMeas, "PPM")
    
    # We've not processed the file so we can delete the .mtb file
    os.system(  "rm diffs_combo_" + str(combo) + ".mtb")

fid.close()

#
# Now generate some plots and HTML pages
#

SlipData = []
maxCombo = 18
for combos in range(maxCombo):
  SlipData.append([])

# First load the file that contains all the slip data
filename = '/net/higgs/mnt/data_drive/DataDump/DashBoard/NoPiSlips.txt'
if(os.path.isfile(filename)): 
  with open(filename,'r') as f:
    for line in f:
      data = line.rstrip().split()
      if(int(data[1]) < maxCombo):
        SlipData[int(data[1])].append(line)

# Now seperate the data we've read into each combo and generate a table
# and plots
for combos in range(maxCombo):
  fid = open("DDSlipsCombo-" + str(combos) + ".html",'w')
  
  webStr = "<html>"
  webStr = webStr + ("<body>")
  webStr = webStr + ("<table border='1'>")
  webStr = webStr + ("<tr>")
  webStr = webStr + ("<th>Combo</th>")
  webStr = webStr + ("<th>Epochs</th>")
  webStr = webStr + ("<th>Slip Count</th>")
  webStr = webStr + ("<th>Slip Epochs</th>")
  webStr = webStr + ("<th>Count [PPM]</th>")
  webStr = webStr + ("<th>Epochs [PPM]</th>")
  webStr = webStr + ("<th>Classic</th>")
  webStr = webStr + ("<th>Drift</th>")
  webStr = webStr + ("<th>Gang Lu</th>")
  webStr = webStr + ("<th>Uncharacterized</th>")
  webStr = webStr + ("</tr>")
  webStr = webStr + ("<tr>")
  fid.write(webStr)

  DOY = []
  epochs = []
  slipCount = []
  slipEpochs = []
  slipCountPPM = []
  slipEpochsPPM = []
  for i in range(len(SlipData[combos])):
    data = SlipData[combos][i].rstrip().split()
    if(int(data[2]) > 0):
      year  = data[0].split("/")[8][:4]
      month = data[0].split("/")[8][4:6]
      day   = data[0].split("/")[8][6:8]

      DOY.append( (   datetime.datetime(int(year), int(month), int(day), 0, 0)
                    - datetime.datetime(2018,1,1)).days)
      epochs.append(int(data[2]))
      slipCount.append(int(data[9]))
      slipEpochs.append(int(data[10]))
      slipCountPPM.append(float(data[9])*1e6/float(data[2]))
      slipEpochsPPM.append(float(data[10])*1e6/float(data[2]))

      date  = year + '-' + month + '-' + day

      webLink = 'http://teller.eng.trimble.com/GPSRxEng' + data[0].split("WWW")[-1] + '/NoPiDI_Top.html'
      webStr = "<td><a href='" + webLink + "' target='_blank'>" + date + "</a></td>"
      webStr = webStr + "<td>" + data[2] + "</td>"
      # Slip count
      webStr = webStr + "<td>" + data[9] + "</td>"
      # Slip Epochs
      webStr = webStr + "<td>" + data[10] + "</td>"
      # Slip count as a PPM
      webStr = webStr + "<td>" + str.format('{0:.4f}',float(data[9])*1e6/float(data[2])) + "</td>"
      # Slip Epochs as a PPM
      webStr = webStr + "<td>" + str.format('{0:.4f}',float(data[10])*1e6/float(data[2])) + "</td>"
      # Classic Slips
      webStr = webStr + "<td>" + data[11] + "</td>"
      # Drift
      webStr = webStr + "<td>" + data[12] + "</td>"
      # Gang Lu Effect
      webStr = webStr + "<td>" + data[13] + "</td>"
      # Uncharacterized
      webStr = webStr + "<td>" + data[14] + "</td>"
      webStr = webStr + ("</tr>")
      fid.write(webStr)
      
  webStr = "</table>"
  webStr = webStr + "</body>"
  webStr = webStr + "</html>"
  fid.write(webStr)
  fid.close()

  # Now generate plots for this combo
  for plots in range(5):
    fig=figure()
    ax=fig.add_subplot(111)

    if(plots == 0):
      plotType = plot_local( array(DOY),array(epochs),'bo',label='Epochs')
    elif(plots == 1):
      plotType = plot_local( array(DOY),array(slipCount),'bo',label='Slip Events')
    elif(plots == 2):
      plotType = plot_local( array(DOY),array(slipEpochs),'bo',label='Slip Epochs')
    elif(plots == 3):
      plotType = plot_local( array(DOY),array(slipCountPPM),'bo',label='Slip Events PPM')
    else:
      plotType = plot_local( array(DOY),array(slipEpochsPPM),'bo',label='Slip Epochs PPM')
    grid(True)
    legend()
    # Prevent the axis numers having an offset
    ax.get_xaxis().get_major_formatter().set_useOffset(False)
    if(plotType == False):
      ax.get_yaxis().get_major_formatter().set_useOffset(False)
    # Save the data as a PNG file
    xlabel('Time [Days since 2018-01-01]')
    if(plots == 0):
      ylabel('Number of Epochs')
    elif(plots == 1):
      ylabel('Number of Slip Events')
    elif(plots == 2):
      ylabel('Number of Slip Epochs')
    elif(plots == 3):
      ylabel('Slip Events PPM')
    else:
      ylabel('Slip Epochs PPM')
    title('Combo ' + str(combos))
    tight_layout()
    show()
    if(plots == 0):
      savefig('Epochs-'+ str(combos) + '.png', dpi=150)
    elif(plots == 1):
      savefig('SlipEvents-'+ str(combos) + '.png', dpi=150)
    elif(plots == 2):
      savefig('SlipEpochs-'+ str(combos) + '.png', dpi=150)
    elif(plots == 3):
      savefig('SlipEventsPPM-'+ str(combos) + '.png', dpi=150)
    else:
      savefig('SlipEpochsPPM-'+ str(combos) + '.png', dpi=150)
    close()