################################################################################
# Copyright (c) 2011-2019 Trimble Inc.
# $Id: concat_NoPi_stats.py,v 1.2 2019/09/11 00:50:54 ddelore Exp $
################################################################################
#
# concat_NoPi_stats.py
#
# Concatenates all the measurement statistics generated by NoPi Display
# for multiple days into a single file per combo.
# See show_help() for the command line
#
# The output files (stats_combo_X.mtb) are loadable in Matlab
# It has the following column format:
#   1) Day Number
#   2) Measurement Type (0 = DD Carrier; 1 = DD Code; 2 = SD CNo)
#   3) SV Id (0 = All satellites)
#   4) Number of Epochs
#   5) Min. value of valid data
#   6) Max. value of valid data
#   7) Mean value of valid data
#   8) Std value of valid data
#   9) Mean absolute value of valid data
#
################################################################################

import sys
import os


################################################################################
# Define some global variables to make things easier
################################################################################

global tst_day
global tst_mon
global tst_year


################################################################################
# Show the help if the command line is not understood
################################################################################

def show_help() :
  print "python concat_NoPi_stats.py <sday> <smon> <syear> <ndays> [dir_ext]"
  print ""
  print "Where:"
  print "  sday, smon and syear define the start date"
  print "  ndays defines the number of days to process"
  print "  dir_ext optionally specifies an extension to the expected"
  print "    directory name of YYYYMMDD"
  print "    i.e. dir_ext '_Short' --> YYYYMMDD_Short"


################################################################################
# Increment the date to use to search for data
################################################################################

def inc_tst_date() :
  global tst_day
  global tst_mon
  global tst_year

  # Assume that this is not a new month
  inc_mon  = 0

  # Check for the end of the month
  if ( (tst_mon == 2) & (tst_day >= 28) ) :
    if ( (tst_year % 4) == 0 ) :
      if ( tst_day == 29 ) :
        inc_mon = 1
    else :
      inc_mon = 1
  elif ( ( tst_mon == 4
           | tst_mon == 6
           | tst_mon == 9
           | tst_mon == 11 )
         & (tst_day == 30) ) :
    inc_mon = 1
  elif ( tst_day == 31 ) :
    inc_mon = 1

  # Increment the month (and possibly the year) and reset
  # the day
  # OR
  # just increment the day of month
  if ( inc_mon ) :
    tst_day = 1
    if ( tst_mon == 12 ) :
      tst_mon = 1
      tst_year = tst_year + 1
    else :
      tst_mon = tst_mon + 1
  else :
    tst_day = tst_day + 1


################################################################################
# Recover values from the end of the results line and append to the input string
# Assumes format is: <something> = <value>;
################################################################################

def append_value( line, str_in ) :
  slne = line.split( '=' )
  slne = slne[1].split( ';' )
  return( str_in + ' ' + slne[0] )


################################################################################
# Recover values from the end of the results line and append to the input string
# The line of data is read from the supplied file
################################################################################

def read_append_value( fin, str_in ) :
  line = fin.readline()
  return( append_value( line, str_in ) )


################################################################################
# SCRIPT STARTS HERE
################################################################################


################################################################################
# Parse the command line
################################################################################

if ( ( len( sys.argv ) != 5 ) & ( len( sys.argv ) != 6 ) ) :
  show_help()
  sys.exit()

# Store the requested start date from the command line
# Assume that it's valid
st_day   = int( sys.argv[1] )
st_mon   = int( sys.argv[2] )
st_year  = int( sys.argv[3] )
num_days = int( sys.argv[4] )

# Store the requested directory name extension
if ( len( sys.argv ) == 6 ) :
  dir_ext = sys.argv[5]
else :
  dir_ext = ''


################################################################################
# Grab the NoPi results from the web page generated by NoPi Display
# Concatenate the results data for multiple days into a file per combo
# The resultant files are loadable in Matlab
################################################################################

# Initialise the current test date
tst_day  = st_day
tst_mon  = st_mon
tst_year = st_year

# Bit-field used to determine which combos have already
# had data stored
# i.e. should the new file write data or append data
created_file = 0

# Loop through each day and grab the NoPi statistics for each
for x in range( num_days ) :
  # Create the directory name to search
  dir_name = ( str( "%4d"  % tst_year )
             + str( "%02d" % tst_mon )
             + str( "%02d" % tst_day )
             )

  # Try to open the NoPi Display results for this date
  fname = dir_name + dir_ext + '/' + 'NoPiDI_Top.html'
  if ( not( os.access( fname, os.F_OK ) ) ) :
    print 'Failed to open file ' + fname
  else :
    print 'Processing file ' + fname
    fin = open( fname )

    # Wait to find the results data
    in_data = 0

    # Read line from file
    line = fin.readline()
    while( len( line ) ) :
      # Find the start of the results data for one combo
      if ( line.find( 'combo_vld' ) != -1 ) :
        # If open, close the previous combo file
        if ( in_data ) :
          fout.close()

        # Determine the current combo number
        slne  = line.split( '[' )
        combo = slne[1].split( ']' )

        # Open the output file
        if ( created_file & ( 1 << int( combo[0] ) ) ) :
          fout = open( 'stats_combo_' + combo[0] + '.mtb', 'at' )
          print 'Append to file stats_combo_' + combo[0] + '.mtb'
        else :
          fout = open( 'stats_combo_' + combo[0] + '.mtb', 'wt' )
          print 'Write to file stats_combo_' + combo[0] + '.mtb'
          created_file = created_file | ( 1 << int( combo[0] ) )

        # Flag that we are now in the results data
        in_data = 1

      if ( in_data ) :
        # Find the start of the results data for one satellite
        if ( line.find( 'sv_id' ) != -1 ) :
          # Store the day number
          lout = str( x )

          # Store the measurement type
          slne = line.split( '[' )
          meas = slne[2].split( ']' )
          lout = lout + ' ' + meas[0]

          # Store the SV Id
          # 0 = All satellites
          lout = append_value( line, lout )     # SVId

          # Store the results
          lout = read_append_value( fin, lout ) # epochs
          lout = read_append_value( fin, lout ) # vld_min
          lout = read_append_value( fin, lout ) # vld_max
          lout = read_append_value( fin, lout ) # vld_mean
          lout = read_append_value( fin, lout ) # vld_std
          lout = read_append_value( fin, lout ) # vld_mav

          # Write the data to the output file
          fout.write( lout + '\n' )

      # Read next line from file
      line = fin.readline()

    # Close the NoPi display file and output file
    fin.close()
    if ( in_data ) :
      fout.close()

  # Increment to the next day
  inc_tst_date()