import json
import threading
import requests
import xmltodict
import numpy as np
import time
import datetime
import os
import signal

#
# Copyright Trimble Inc
#
# Provided to Geotronics Poland to demonstrate accessing Trimble receiver FFT data
#
#

# Used to control the Ctrl-C program termination
ThreadingActive = True

# Adjust if a proxy is required
proxies = {"http":  "",
           "https": ""}


# Use requests to get data via HTTP from a receiver. By default, don't
# adjust the headers and use the global proxy setting.
def getReceiverData(session, URL, user, password, headers='', proxies=proxies):
    #with mutex:
    #    print(datetime.datetime.now().isoformat(),URL)

    r = session.get(URL, auth=(user,password), headers=headers, proxies=proxies, timeout=5)
    r.raise_for_status()
    if('.xml' in URL):
        d = xmltodict.parse(r.text)
    else:
        d = r.text # Just return the raw text for csv
    return(d)

# Called when CTRL=C is pressed. Kills the threads and exits
def signal_handler(signal,frame):
  global ThreadingActive
  print('Ctrl+C Detected - shutting down ...')
  ThreadingActive = False

  for num in range(numThreads):
    threadHandle[num].join()

  print('Terminated via CTRL-C')

# Called to get the FFT data from the receiver
def getFFT(URL, session, headers, user, password, showTimeTag=True):
    d = getReceiverData(session, URL, user, password, headers=headers)

    if('.xml' in URL):
        # Older firmware that does not support the .csv format, parse the XML - this is
        # slower and consumes more network bandwidth
        data = np.array(d['saData']['points']['y']).astype('int')
        # Get the time and convert to milliseconds
        timeTag = int(float(d['saData']['FFT_time'] )*1000)
    else:
        # Newer firmware that supports the .csv format
        tokens = d.split(',')
        data = np.array(tokens[1:2049]).astype(int)
        timeTag = tokens[0] # in milliseconds in the first field
        
    if(showTimeTag):
      with mutex:
        print(timeTag)

    return(timeTag, data)

# Called by each thread to get the FFT data from a band of a single receiver
def fftWorker(thisRX,thisBand):
    # Get the requested band and ensure it is valid
    band = thisBand
    if(band not in ('B1','L1','L2','L5','E6')):
        print('Invalid band:',band)
        return

    # Get the receiver's IP address and login details
    IPAddr = thisRX['addr']
    user   = thisRX['user']
    password = thisRX['pw']

    # Get the HTTP data transfer encoding we want to use. Valid options
    # are 'gzip' and 'None'. 'None' is the default and means no compression
    # is used. 'gzip' is used to reduce the network bandwidth used. The no
    # compression option will consume more network bandwidth, but reduce the
    # CPU load on this server and the receiver. The encoding type can be
    # set for each receiver. The data is uncompressed when it is saved to 
    # the log file irrespective of the encoding to transfer the data over
    # the network. You may get a slightly higher update rate by adjusting
    # this on a receiver by receiver basis depending on the network bandwidth/latency
    # to each receiver, and the load on this server (which will depend on the
    # number of receivers and bands processed by this script)
    encoding = thisRX['encoding']
    if(encoding not in ('gzip','None')):
        print('Invalid encoding:',encoding)
        return

    headers={'Accept-Encoding':encoding}

    # Get the receiver's "long" name. We'll use this to create the log file name
    fileBase = thisRX['long']
    # We'll store the data in a different directory for each receiver, make the
    # directory if it does not exist
    if(not os.path.exists(fileBase)):
        os.makedirs(fileBase)

    # Do a full XML read of the FFT to get the spectrum details, e.g. center ferequency
    # and whether the spectrum is inverted. This extra information is not in the .csv 
    # version. In the main loop, we get the .csv version as it is faster to process on
    # both the receiver and the server running this code, and it consumes less network
    # bandwidth.
    configURL  = 'http://' + IPAddr
    configURL += '/xml/dynamic/rfSpectrumAnalyzer.xml?rfBand=' + band + '&filterMode=NoFilter'

    # Use requests' Session to create a keep-alive connection.
    session = requests.Session()

    # Now get the configuration details of the FFT
    try:
        d = getReceiverData(session, configURL, user, password)
    except:
        print('Problem with %s:%s - killing thead' % (thisRX, thisBand))
        return

    # If we have all zeros, the FFT is not supported. Test for this and kill this
    # thread if we detect a problem
    numZeroes = 0
    for thisData in d['saData']['points']['y']:
        if(thisData == '0'):
            numZeroes +=1

    if(numZeroes == 2048):
        # All zeroes == band not supported
        print(band,': No FFT data - terminating thread')
        return

    # Get the center frequency of the FFT data in MHz
    centerFreq = float(d['saData']['center_freq_mhz'])

    # Check if the spectrum is inverted. If it is, we need to flip the
    # FFT data before saving to the log file
    if(d['saData']['inverted_spectrum'] == '1'):
        inverted = True
    else:
        inverted = False

    # Get the current week number
    weekNum = int(d['saData']['time']['week'])
    lastTime = -1

    # Get the firmware version, newer code supports a CSV method to get
    # the FFT data. If that isn't supported, use the older XML method
    versionURL  = 'http://' + IPAddr
    versionURL += '/xml/dynamic/sysData.xml'
    try:
        d = getReceiverData(session, versionURL, user, password)
    except:
        print('Problem with %s:%s - killing thead' % (thisRX, thisBand))
        return

    # Betas add a "-", handle customer and beta version numbers
    version = float(d['sysData']['FWVersion'].split('-')[0])
    print('Firmware Version: %.2f' % version)
    if(version < 5.64):
        mainURL = configURL
    else:
        # This URL gets just the FFT information from the receiver as comma seperated data
        # Notice the ".csv" in the URL. We'll use this URL in the main procesing loop as
        # it processes faster and less data is transferred from the receiver.
        loc = '/xml/dynamic/rfSpectrumAnalyzer.csv?rfBand=' + band + '&filterMode=NoFilter'
        mainURL = 'http://' + IPAddr + loc

    # Get the current time, we'll use this to provide a diagnostic
    # indicating the FFT rate we are achieving for this thread
    startTime = datetime.datetime.now()
    numFFTs = 0

    gps_ref = datetime.datetime(1980,1,6)
    # Main "forever" processing loop
    while(ThreadingActive):
        try:
            timeTag, data = getFFT(mainURL,session,headers,user,password,showTimeTag=False)
        except:
            print('HTTP Err:',mainURL)
            time.sleep(0.1)
            continue

        # With the latest firmware, we should not get any duplicates,
        # incase anything changes add some protection by testing whether
        # we've already received an FFT with this timetag
        if(int(timeTag) == lastTime):
            with mutex:
                print('duplicate FFT',fileBase,band)
            continue

        # The FFT data does not contain the week number. We need to check
        # for week rollovers to ensure we keep the week number up to date.
        # If the time tag is currently less than the last time tag, the
        # week number could have rolled. We run a check to see if that
        # is really the case. For new firmware, we'll be using the CSV
        # record to get the FFT, that does not contain full time
        # information, so we need to be careful in case the receiver
        # was power cycled.
        if(int(timeTag) < lastTime):
            with mutex:
                print('Testing for week roll')
            # Get the full XML config data to get the week number to see
            # if there was a roll over. If the receiver has been rebooted,
            # it may not have time and that could cause an apparent wrap.
            # The XML data includes time status information
            try:
                d = getReceiverData(session, configURL, user, password, headers=headers)
                timeState = d['saData']['time']['status']
            except:
                print('HTTP Err:',configURL)
                time.sleep(0.1)
                continue

            if(timeState != 'UTC'):
                continue
            
            # Get the time difference between the original FFT time tag and
            # the full time tag. This should be within processing and network
            # delays. If it is, we can use the week number from the full config
            # data. If it isn't, something isn't right and we should drop this
            # FFT and try again
            
            timeDelta = abs(int(timeTag) - int(d['saData']['time']['msecs']))
            # 100s (we are in millseconds) - far longer than any reasonable delay
            if(timeDelta > 100000):
                continue

            # All good - use the weeknumber from the full config record
            weekNum = int(d['saData']['time']['week'])
            print(weekNum)

        # Save the time for next time through the loop
        lastTime = int(timeTag)

        # The FFT data is on a linear scale, convert to dB
        thisFFT = 10.0 * np.log10(data.astype(float))
    
        # This FFT is inverted, so we need to flip the data!
        if(inverted):
            thisFFT = np.flip(thisFFT)

        # Now save the FFT
        offset = datetime.timedelta(days=weekNum*7,seconds=int(timeTag)/1000)
        now = gps_ref + offset
        # Get the date
        dateTimeStr  = str(now.year) + '-' + str(now.month).zfill(2) + '-' + str(now.day).zfill(2)
        # Add the Hour
        dateTimeStr += '-' + str(now.hour).zfill(2)
        logFile = dateTimeStr + '-' + fileBase + '-' + band + '.log'
        # Add the directory path (the receiver "long" name)
        logFile = os.path.join(fileBase,logFile)

        #print(fileBase, band, centerFreq, timeTag, logFile)
        with open(logFile,'a') as fid:
            fid.write('%s %d %.4f ' % (timeTag, weekNum, centerFreq))
            for thisData in thisFFT:
                fid.write('%.4f ' % thisData)
            fid.write('\n')

        # As a diagnostic, keep track of the number of FFTs we've accessed
        numFFTs += 1

        # Periodically output a diagnostic that indicates the FFT rate we are processing
        # for each band.
        if((numFFTs % 100) == 0):
            delta = (datetime.datetime.now() - startTime).total_seconds()
            # Keep track of the effective FFT update rate (frequency) since the test started
            freq = numFFTs / delta
            with mutex:
                print("%s %s: Rate=%.3fHz" % (fileBase,band,freq))

    # We've been asked to terminate, so return from the thread
    print('Processing Complete')


# Main entry point
if __name__ == '__main__':

    # Load the receiver details
    with open('receivers.json') as f:
        receivers = json.load(f)
    
    # Load the signal details
    with open('signals.json') as f:
        signals = json.load(f)

    # Setup a Ctrl-C handler
    signal.signal(signal.SIGINT, signal_handler)

    # Setup a mutex - we'll use this to protect stdout printing.
    mutex = threading.Lock()

    # Kick off a thread for each band
    threadHandle = []
    numThreads = len(receivers) * len(signals)
    for thisBand in signals:
        for thisReceiver in receivers:
            t = threading.Thread(target=fftWorker, args=(thisReceiver,thisBand,))
            threadHandle.append(t)
            t.start()