from logging import raiseExceptions
import multiprocessing
import multiprocessing.pool
import multiprocessing.connection as mp_con
import os, sys, traceback, re, glob
import json
from urllib.request import urlopen
import requests
from requests.auth import HTTPBasicAuth
from datetime import datetime
from six import print_
import smtplib, email.mime.text, email.mime.multipart, email.mime.application
import subprocess
import psutil
import inspect
import time
import shutil
import copy
import csv
import mutils
import numpy
import xmltodict
from datetime import datetime

import RXTools as rx
import SeptRXTools as sept_rx
import NovRXTools as novt_rx
from RunConfig import get_config_xml

import socket
import time

try:
  import xml.etree.cElementTree as ET
except ImportError:
  import xml.etree.ElementTree as ET


verbose = True
run_build_install_in_background = False
run_setup_in_parallel = False
user = {'username':'username', 'password':'password'}
use_simulator = True

class LogInfo(object):
  def __init__(self, log_to_file):
    '''Print info to screen and optionally a file for the web/ app.'''
    self.info = {}
    self.log_to_file = log_to_file
    if log_to_file:
      self.diag_f = open("regression_diag.txt","w")
    else:
      self.diag_f = sys.stdout

  def close(self):
    if self.diag_f != sys.stdout:
      self.diag_f.close()

  def raise_(self, *args ):
    self.print_(args)
    raise RuntimeError(args)

  def print_(self, *args ):
    log_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + ": " + " ".join(map(str,args))
    print_( log_str, file=self.diag_f, flush=True )
    self.info['detail'] = log_str
    if self.log_to_file:
      with open("regression_progress.json","w") as f:
        json.dump( self.info, f )


# Using __slots__ lets pylint check for typos
class ScenarioInfo(object):
  '''All info from a single scenario XML file'''
  __slots__ = ('fileName',
               'play_secs',
               'scn_file')
  def __init__(self):
    pass
  def __repr__(self):
    mystr = 'ScenarioInfo:\n  '
    mystr += '\n  '.join("%s: %s" % (item,getattr(self,item,None)) for item in self.__slots__)
    mystr += '\n'
    return mystr


def fn(forcePrint=False):
  '''
  Inputs: forcePrt = True/False - force it to return function name and ignore verbose
  Return caller's name
  '''
  if verbose == False and forcePrint == False:
    return ""
  else:
    frameinfo = inspect.getouterframes(inspect.currentframe())
    (frame, source, lineno, func, lines, index) = frameinfo[1]
    return func+"()::"


def RandomFrequency(random_seed=221, start_freq_Hz = 1555.0e6, stop_freq_Hz = 1625.0e6, n_points = 10):
  ''' 
  Return a serious of random frequency for eg. scenario 11b based on random_seed(for the purpose of reproducible), input range and number of points.
  '''
  # Initialize the random number generator
  numpy.random.seed(random_seed)
  lst = []
  for i in range(n_points):
    # Return a uniformly-distributed random floating point number
    freq_Hz = numpy.random.uniform(start_freq_Hz, stop_freq_Hz)
    freq_MHz = '%f' % (freq_Hz/1e6) # 6 decimal
    lst.append(freq_MHz)
  return lst


def gpsTimeToWnTow(time_str):
  '''
  Calculate Wn, Tow from GPS time
  '''
  # number of seconds in a week
  weekSeconds = 60 * 60 * 24 * 7
  # epoch time
  epoch_time = datetime(1980, 1, 6)

  gps_time = datetime.strptime(time_str.strip(),'%d-%b-%Y %H:%M:%S')
  # subtract Datetime from epoch datetime
  delta = (gps_time - epoch_time)
  
  total_sec = delta.total_seconds()
  
  wn = int(total_sec / weekSeconds)
  tow = total_sec - wn * weekSeconds

  return {'wn':wn, 'tow':tow}


def send_request(ip, port, resource, PARAMS = None, username = None, password = None, timeout=15):
  '''
    Send http request to other server
  '''
  URL = 'http://' + ip + ':' + port + resource # api-endpoint
  r = None
  try:
    r = requests.get(url = URL, params = PARAMS, auth = HTTPBasicAuth(username, password), timeout=timeout)
    if r.status_code != 200:
      print( datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + ": url = " + URL )
      print('status_code:\t',r.status_code)
      print('text:\t\t',r.text)
      print('url:\t\t',r.url)
  except requests.exceptions.ConnectionError as ex:
    r = 'Error. Request URL failed. '+str(ex)
    print(r)
  except requests.exceptions.ReadTimeout as ex:
    r = 'Error. Request URL failed. '+str(ex)
    print(r)
  except Exception as ex:
    r = 'Error. Request URL failed. '+str(ex)
    print(r)
  return r


def get_jamming_device_info():
  '''
  Provide dictionary with info on all regression system jamming devices.  This
  is basically config.xml info.  Here's the data format:
   'jamdevs' : [{ 'deviceType':'HPSignalGen', 'desc':'HP8648C GPIB 16', 'long_desc':'Some desc...', 'enable':True, 'gpid_address':16,...},
                { 'deviceType':'Attenuator', 'desc':'DAT64L COM 3', 'long_desc':'Some desc...', 'enable':True, 'com_port':3,...},
               ...]
  '''
  cfg = get_config_xml()

  d= {'jamdevs':[]}
  for n,dev in enumerate(cfg.jamming_devices):
    info = {}
    for k,v in (vars(dev)).items():
      if v is not None:
        info[k]=v
    d['jamdevs'].append(info)

  return d


def get_receiver_info():
  '''
  Provide dictionary with info on all regression system receivers.  This
  is basically config.xml info.  Here's the data format:
   'devs' : [{ 'ip':'10.1.2.3', 'long_desc':'Some desc..'}, ...]
  '''
  cfg = get_config_xml()

  d= {'devs':[]}
  for n,dev in enumerate(cfg.devices):
    info = {'ip':dev.ip, 'long_desc':dev.long_desc,'n':dev.n, 'trimble':(dev.novatel is None and dev.septentrio is None), 'novatel':(dev.novatel is not None), 'septentrio':(dev.septentrio is not None)}
    d['devs'].append(info)
  return d


def remove_one_bad_device( loginfo, cfg, device, reason ):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             cfg = ConfigXML = info from config.xml
             device = device to remove
             reason = text describing removal reason
     Remove a bad device from list to test
  '''
  loginfo.print_("Removing device {}".format(device.ip))
  cfg.devices.remove( device )
  if len(cfg.email_recipients) > 0:
    sender = 'RFregression@fermion.eng.trimble.com'
    msg_text = "{} - removing unit {}".format(reason,device.ip)
    msg = email.mime.text.MIMEText(msg_text)
    msg['Subject'] = 'Anti-jamming regression device problem'
    msg['From'] = sender
    msg['To'] = ', '.join(cfg.email_recipients)
    s = smtplib.SMTP( cfg.email_host )
    s.sendmail(sender, cfg.email_recipients, msg.as_string())
    s.quit()


def remove_bad_devices( loginfo, cfg ):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             cfg = ConfigXML = info from config.xml
     Check all cfg.devices and remove any that are not responding.
  '''
  loginfo.print_("Removing bad devices...")
  bad_devices = []
  for device in cfg.devices:
    if device.ag == None: # only check GNSS receivers
      try:
        rx.SendHttpGet( device.ip, '/', device.user, device.pw )
        if device == cfg.current_config_device:
          bad_devices.append( device )
      except:
        bad_devices.append( device )
    else:
      loginfo.print_("Removing bad ag devices is skipped.")

  for device in bad_devices:
    remove_one_bad_device( loginfo, cfg, device, "Unresponsive" )
  if len(cfg.devices) == 0:
    raise RuntimeError("remove_bad_devices() removed everything!")


def disable_all_IP_filters():
  '''After a test is done, make sure receivers are visible to everyone.
  During a regression test, the receivers will block packets from all IP
  addresses but this server.
  '''
  try:
    print("Disable IP filtering on all receivers...")
    cfg = get_config_xml()
    for dev in cfg.devices:
      rx.DisableIpFilter( dev.ip, dev.user, dev.pw )
  except:
    print("Couldn't disable IP filtering on all receivers...")
    pass


def kill_all(proc_pid):
  '''os.kill() and proc.terminate() don't handle child processes.
  Use psutils to kill everything spawned...
  '''
  process = psutil.Process(proc_pid)
  for proc in process.children(recursive=True):
    proc.kill()
  process.kill()


def kill_background_procs(proc_q):
  '''Input: proc_q = Queue() - list of background processes started
  Kill any background processes
  '''
  while not proc_q.empty():
    pid = proc_q.get()
    if pid != multiprocessing.current_process().pid:
      print('kill',pid)
      try:
        kill_all(pid)
      except:
        # If the process is already finished, don't worry
        pass


def email_error_result():
  '''Email error reports for the normal regression run.'''

  sender = 'RFregression@fermion.eng.trimble.com' # It is ok to use RFReplay's email sender for the time being

  msg_text = ""
  try:
    with open('regression_diag.txt') as f:
      msg_text = f.read()
  except:
    msg_text = "Diagnostic file not found"

  cfg = get_config_xml()
  if len(cfg.email_recipients) > 0:
    msg = email.mime.text.MIMEText(msg_text)
    msg['Subject'] = 'Anti-jamming regression failure'
    msg['From'] = sender
    msg['To'] = ', '.join(cfg.email_recipients)
    s = smtplib.SMTP( cfg.email_host )
    s.sendmail(sender, cfg.email_recipients, msg.as_string())
    s.quit()


def get_regression_status():
  '''Loads "regression_status.json" file as a dictionary'''
  with open("regression_status.json") as f:
    return json.load( f )


def set_regression_status(proc_q,status):
  '''Input: proc_q = Queue() - list of background processes started
            status = Running, Idle, Idle (Error), Killed
  What is running?  Mainly to inform the web GUI if we're actively doing something.
  Also kills any leftover background processes if the user cancels an operation.
  '''
  d = {}
  d['status'] = status
  if status == "Running":
    d['active'] = 1
  else:
    d['active'] = 0

  if status != "Running":
    disable_all_IP_filters()

  if d['active'] == 1:
    # Return an error if we're already running and try to start again
    curr_d = get_regression_status()
    if curr_d['active'] == 1:
      return False
  elif proc_q is not None:
    kill_background_procs(proc_q)

  with open("regression_status.json","w") as f:
    print("Set Regression Status",d)
    json.dump( d, f )
  return True


def get_regression_stop_status():
  '''Loads "regression_stop_status.json" file as a dictionary'''
  with open("regression_stop_status.json") as f:
    return json.load( f )


def set_regression_stop_status(status='Stop'):
  '''Input: status = Unknown (initial value), Enabled (after run starts for 60sec), Stop (requested to stop)
  What is running?  Mainly to inform the web GUI if we're actively doing something.
  Also kills any leftover background processes if the user cancels an operation.
  '''
  d = {}
  d['stop_status'] = status
  with open("regression_stop_status.json","w") as f:
    json.dump( d, f )

def is_jammer_enabled(jammer_id, cfg):
  '''Input: jammer_id = string, eg. HP8648C_GPIB_16, HP8648C_GPIB_18, Chirp_1, HackRF_1
  '''
  return any([(jamdev.id==jammer_id and jamdev.enable==1) for jamdev in cfg.jamming_devices])


def is_build_needed(loginfo):
  '''Input: loginfo - LogInfo()
  Returns: True/False - is a firmware build needed?
           fw_time - the start/check time of the firmware
  '''
  fw_time = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
  try:
    install_age_secs = time.time() - os.path.getmtime('Builds/install.txt')
    if install_age_secs < 15*60:
      loginfo.print_("Skipping builds because they are recent")
      return False, fw_time
  except:
    # Builds/install.txt probably doesn't exist, so continue on
    pass
  return True, fw_time


def build_install_firmware(loginfo, proc_q, cfg, selected_devices=[], use_customized_timg_and_skip_build=False):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             proc_q = Queue() - list of background processes started
             cfg = ConfigXML = info from config.xml
             selected_devices = list of selected device.n as int
             use_customized_timg_and_skip_build = True/False, if True, cfg.device[i].timg is used for installation.
  Builds and installs firmware on all units.
  '''
  if(os.path.isdir("Builds") == False):
    os.mkdir("Builds") # Make sure the directory exists

  if verbose: loginfo.print_(fn(),"selected devices =", selected_devices)

  proc_list = []
  devices = cfg.devices
  devices.sort(key=lambda x: 0 if x.ag==None else 1, reverse=False)
  for device in devices:
    if device.n in selected_devices:
      if device.septentrio is not None:
        loginfo.print_(fn()+"This is Septentrio receiver #%d. Skip." % device.n)
      elif device.novatel is not None:
        loginfo.print_(fn()+"This is Novatel receiver #%d. Skip." % device.n)
      else:
        loginfo.print_(fn()+"This is Trimble receiver #%d" % device.n)
        build_dir_name = "RX" + str(device.n) + "-coreBuild"
        tag_cmd = ""
        if device.tag:
          if device.tag.startswith("-"):
            tag_cmd = " " + device.tag
          else:
            tag_cmd = " -r" + device.tag

        if use_customized_timg_and_skip_build:
          loginfo.print_(fn()+"Use customized timg. Skip CVS checkout Build #%d" % device.n)
        else:
          loginfo.print_(fn()+"CVS checkout Build #%d" % device.n)
          if cfg.clean_builds or not os.path.isdir("Builds/"+build_dir_name):
            check_runq(loginfo,proc_q, 'rm -rf Builds/' + build_dir_name)
            check_runq(loginfo,proc_q, 'cd Builds ; cvs -q co -d ' + build_dir_name + tag_cmd + ' coreBuild',log=True)
          else:
            check_runq(loginfo,proc_q, "cd Builds/%s; cvs -qn up | egrep '^[AMC] ' | cut -d' ' -f2 | xargs -t -i mv {} {}.orig" % build_dir_name)
            check_runq(loginfo,proc_q, 'cd Builds/%s; cvs -q up' % build_dir_name)

          if device.patchFile:
            check_runq(loginfo,proc_q, 'export BASE_DIR=$PWD; cd Builds/%s; patch -p0 < $BASE_DIR/%s' % (build_dir_name,device.patchFile))

          frozen = ""
          if device.frozen is not None:
            frozen = "DATEFROZEN=1"
          
          check_runq(loginfo,proc_q, 'cd Builds/' + build_dir_name + ' ; ./config -f ' + device.build + ' ; ' + frozen + ' make -j8',log=True)

        # Install firmware
        timg_path = ''
        if use_customized_timg_and_skip_build: 
          timg_path = device.timg
        else:
          timg_path = 'Builds/' + build_dir_name + '/' + device.build + '_output/' + device.build + '.timg'

        if not os.path.exists(timg_path):
          raise RuntimeError("timg file does not exist:", timg_path)

        if cfg.allow_firmware_install:
          loginfo.print_(fn()+"Install Build #%d with %s" %(device.n,timg_path))
          args = (device.ip,
                  device.user,
                  device.pw,
                  # Firmware file
                  timg_path,
                  True) # use the failsafe install
          kwargs = {'wait':True}
          proc = multiprocessing.Process( target=rx.upgradeFW, name=device, args=args, kwargs=kwargs )
          proc.start()
          proc_list.append( proc )

  for proc in proc_list:
    proc.join()
    if proc.exitcode != 0:
      loginfo.print_(fn()+"Bad install for {}".format(proc.name.ip))
      remove_one_bad_device( loginfo, cfg, proc.name, "CVS or build problem" )

  if len(cfg.devices) == 0:
    raise RuntimeError(fn()+"build_install_firmware() removed everything!")

  # Touch only for cvs install
  if not use_customized_timg_and_skip_build:
    check_runq(loginfo,proc_q, "touch Builds/install.txt")
    check_runq(loginfo,proc_q, "chmod a+w Builds/install.txt")

  loginfo.print_(fn()+"Allow time for reboot to fully finish.")
  time.sleep(120) # allow time for reboot to fully finish and mount filesystem
  return


def wrap_download( device, cfg_GlobalRunNum ):
  '''
  Download files from trimble receiver
  '''
  try:
    rx.GetLoggedFilesViaHttpNoDisable( device.ip,
                                      device.user,
                                      device.pw,
                                      "DataDir/RX%d-%d"% (device.n,cfg_GlobalRunNum),
                                      False, # don't clear existing data
                                      checkLocal=True )
    return True
  except:
    return False

def reset_simulator(loginfo, cfg):

  if not use_simulator:
    if loginfo is not None:
      loginfo.print_("Skip reset_simulator.")
    return

  r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
  data = r.json()
  if loginfo is not None:
    loginfo.print_("getStatus() returns", data)
  
  # Stop running if needed
  if data['msg']['status'] == str(5): # Running
    if loginfo is not None:
      loginfo.print_("Stop Running Spirent")
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/stopScenario/', None, user['username'], user['password'])
    time.sleep(5)
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
    loginfo.print_("request response: ", r)
    data = r.json()
    

  # Close scenario if needed
  if data['msg']['status'] == str(7) or data['msg']['status'] == str(2): # Ended or Ready
    if loginfo is not None:
      loginfo.print_("Close Scenario")
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/closeScenario/', None, user['username'], user['password'])
    time.sleep(5)
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
    data = r.json()
    
# def turn_off_spirent_simulator(loginfo, cfg):
#   '''
#     Send stop and close sceanrio http request to server controlling simulator on cfg.spirent_simulator_host_ip
#     Inputs: 
#       loginfo = LogInfo() - where to log diagnostics
#       cfg = ConfigXML = info from config.xml
#   '''
#   # Stop Spirent simulator
#   if loginfo is not None:
#     loginfo.print_("Stop Spirent and close scenario.")
#   if use_simulator: r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/stopScenario/', None, user['username'], user['password'])
#   if use_simulator: r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/closeScenario/', None, user['username'], user['password'])


def run_spirent_and_download_files( loginfo, proc_q, cfg, info, selected_devices_list, run_para, devices_filter_type_dict):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             proc_q = Queue() - list of background processes started
             cfg = ConfigXML = info from config.xml
             info = ScenarioInfo = info from current scenario XML
             selected_devices_list = list of integer numbers defined by <n>?</n> in config.xml file
             run_para = string or run parameter in json format
             devices_filter_type_dict = dictionary int rx.n -> string anti-jam filter type
     Returns: proc = multiprocessing.Process if started something, None otherwise
              try_again = True if RF replay stopped because of a drive glitch.  False normally
              run_freq_type =  "Unknown", "Constant", "Stepped", "Customized"
              run_list = list of steps using jammer
  Starts Spirent Simulator and waits until the test is done.
  Also downloads T0x files once an hour for regression tests. FIXME
  '''
  
  if verbose:
    loginfo.print_(fn(),"run para dictionary:",str(run_para))
  
  jammer_list = run_para['jammer_list']
  t_start_offset = int(run_para['start_offset_sec']) if 'start_offset_sec' in run_para else 60 # Optinal feature when it is needed
  t_end_quiescence = int(run_para['additional_quiescence_sec']) if 'additional_quiescence_sec' in run_para else 0

  proc = None
  try_again = False
  run_freq_type = "Unknown"
  
  dt_play = 0
  run_list = []

  # freq_type can not be mixed.
  # freq_type == Customized can be extened to hackrf&chirp to combine different jammers 
  # Stepped frequency type cannot be used with Customized. If needed, use Customized to define individual step
  if any([device_id in jammer_list and jammer_list[device_id]['freq_type'] == 'Customized' for device_id in ['gpib18', 'gpib16', 'hackrf_1', 'chirp_1']]):
    run_freq_type = "Customized"
  elif ( 'gpib18' in jammer_list and jammer_list['gpib18']['freq_type'] == 'Stepped' ) or ( 'gpib16' in jammer_list and jammer_list['gpib16']['freq_type'] == 'Stepped' ) or ( 'hackrf_1' in jammer_list and jammer_list['hackrf_1']['freq_type'] == 'Stepped' ):
    run_freq_type = "Stepped"
  else:
    run_freq_type = "Constant"

  if verbose: loginfo.print_(fn(),"run_freq_type:",str(run_freq_type))

  ######################
  # Constant freq test #
  ######################
  for gpib_id in ['gpib18', 'gpib16']:
    if gpib_id in jammer_list and jammer_list[gpib_id]['freq_type'] == 'Constant':
      loginfo.print_(fn()+"This test includes constant freq at "+gpib_id)
      # run_freq_type = "Constant" if (run_freq_type == "Unknown" or run_freq_type == "Constant") else run_freq_type
      def default_run_list_cw(gpib_id, jammer_list, t_start_offset):
        """For example, jammer_list = run_para["jammer_list"] contains,
              "gpib18": {
                "freq_type": "Constant",
                "pwr": "-18.8",
                "freq": "1575.42",
                "dwell_time": "300",
                "quiescence": "300"
              }"""
        run_list = []
        gpib_addr = gpib_id[-2:]
        freq = jammer_list[gpib_id]['freq']
        pwr = jammer_list[gpib_id]['pwr']
        dwell_time = jammer_list[gpib_id]['dwell_time']
        quiescence = jammer_list[gpib_id]['quiescence']
        run_list.append( {'t':t_start_offset+0, 'device_name':'cw', 'gpib_addr':int(gpib_addr), 'freq':1575.42, 'pwr':-999, 'msg':'off'} )
        if float(dwell_time) != 0:
          run_list.append( {'t':t_start_offset+float(quiescence), 'device_name':'cw', 'gpib_addr':int(gpib_addr), 'freq':float(freq), 'pwr':float(pwr), 'msg':'on'} )
          run_list.append( {'t':t_start_offset+float(quiescence)+float(dwell_time), 'device_name':'cw', 'gpib_addr':int(gpib_addr), 'freq':1575.42, 'pwr':-999, 'msg':'off'} )
        run_list.append( {'t':t_start_offset+float(quiescence)+float(dwell_time)+float(quiescence), 'device_name':'cw', 'gpib_addr':int(gpib_addr), 'freq':1575.42, 'pwr':-999, 'msg':'end'} )
        return run_list
      
      run_list += default_run_list_cw(gpib_id, jammer_list, t_start_offset)
      try:
        run_list.sort(key=lambda x: (x['t'], x['gpib_addr']))
      except:
        run_list.sort(key=lambda x: (x['t'], ))
      dt_play = max(dt_play, run_list[-1]['t'])
      
  if 'chirp_1' in jammer_list and jammer_list['chirp_1']['freq_type'] == 'Constant':
    loginfo.print_(fn()+"This test includes constant frequency by chirp")

    def default_run_list_chirp(jammer_list,t_start_offset, chirp_com_port):
      """For example, jammer_list = run_para["jammer_list"] contains,
            "chirp_1": {
              "pwr": "1.2",
              "dwell_time": "300",
              "quiescence": "300"
            }
      """
      run_list = []
      quiescence_period = float(jammer_list['chirp_1']['quiescence'])
      dwell_time = float(jammer_list['chirp_1']['dwell_time'])
      run_list.append( {'t':t_start_offset+0, 'device_name':'chirp_1', 'how_long':0, 'msg':'start, do nothing'} )
      run_list.append( {'t':t_start_offset+quiescence_period, 'device_name':'chirp_1','com_port':int(chirp_com_port), 'pwr':float(jammer_list['chirp_1']['pwr']), 'how_long':int(dwell_time), 'msg':'on'} )
      run_list.append( {'t':t_start_offset+quiescence_period+dwell_time, 'device_name':'chirp_1','com_port':int(chirp_com_port), 'pwr':float(jammer_list['chirp_1']['pwr']), 'how_long':int(0), 'msg':'off'} ) # redundant msg only for postprocessing to find jammer on and off timestamp
      run_list.append( {'t':t_start_offset+quiescence_period+dwell_time+quiescence_period, 'device_name':'chirp_1', 'how_long':0, 'msg':'end, do nothing'} )
      return run_list

    chirp_com_port = 21
    for jammer in cfg.jamming_devices:
      if jammer.type == 'Chirp':
        chirp_com_port = jammer.com_port

    run_list += default_run_list_chirp(jammer_list,t_start_offset, chirp_com_port)
    run_list.sort(key=lambda x: (x['t'],))
    dt_play = max(dt_play, run_list[-1]['t'])

  if 'hackrf_1' in jammer_list and jammer_list['hackrf_1']['freq_type'] == 'Constant':
    loginfo.print_(fn()+"This test includes constant freqency by hackrf_1")
    
    def default_run_list_hackrf(jammer_list,t_start_offset):
      """For example, jammer_list = run_para["jammer_list"] contains,
            "hackrf_1": {
              "freq_type": "Constant",
              "pwr": "1.2",
              "interference_type": "GAUSSIAN_5MHz",
              "freq": "1595.42",
              "dwell_time": "300",
              "quiescence": "300"
            }
      """
      run_list = []
      quiescence_period = float(jammer_list['hackrf_1']['quiescence'])
      dwell_time = float(jammer_list['hackrf_1']['dwell_time'])
      run_list.append( {'t':t_start_offset+0, 'device_name':'hackrf_1', 'how_long':0, 'msg':'start, do nothing'} )
      run_list.append( {'t':t_start_offset+quiescence_period, 'device_name':'hackrf_1', 'freq':float(jammer_list['hackrf_1']['freq']), 'pwr':float(jammer_list['hackrf_1']['pwr']), 'type':str(jammer_list['hackrf_1']['interference_type']), 'how_long':int(dwell_time), 'msg':'on'} )
      # redundant msg only for postprocessing to find jammer on and off timestamp
      run_list.append( {'t':t_start_offset+quiescence_period+dwell_time, 'device_name':'hackrf_1', 'freq':float(jammer_list['hackrf_1']['freq']), 'pwr':float(jammer_list['hackrf_1']['pwr']), 'type':str(jammer_list['hackrf_1']['interference_type']), 'how_long':int(0), 'msg':'off'} )
      run_list.append( {'t':t_start_offset+quiescence_period+dwell_time+quiescence_period, 'device_name':'hackrf_1', 'how_long':0, 'msg':'end, do nothing'} )
      return run_list

    run_list += default_run_list_hackrf(jammer_list,t_start_offset)
    run_list.sort(key=lambda x: (x['t'], ))
    dt_play = max(dt_play, run_list[-1]['t'])

  ########################
  # Stepwise   freq test #
  # Customized freq test #
  ########################
  for gpib_id in ['gpib18', 'gpib16']:
    if gpib_id in jammer_list and jammer_list[gpib_id]['freq_type'] == 'Stepped':
      loginfo.print_(fn()+"This test includes stepwise freq change at " + gpib_id)
      
      def stepped_run_list_cw(gpib_id,jammer_list,t_start_offset):
        """For example jammer_list = run_para["jammer_list"] contains:
              "gpib18": {
                "freq_type": "Stepped",
                "pwr": "1.2",
                "start_freq": "1560",
                "stop_freq": "1620",
                "freq_step": "0.3",
                "dwell_time": "30",
                "recovery_time": "15",
                "quiescence": "300"
                }
        """
        curr_run_list = []
        pwr = float(jammer_list[gpib_id]['pwr'])
        quiescence_period = float(jammer_list[gpib_id]['quiescence']) # 5*60 [sec]
        dwell_time = float(jammer_list[gpib_id]['dwell_time']) # 30 [sec]
        recovery_time = float(jammer_list[gpib_id]['recovery_time']) # 15 [sec]
        start_freq = float(jammer_list[gpib_id]['start_freq'])
        stop_freq = float(jammer_list[gpib_id]['stop_freq'])
        freq_step = float(jammer_list[gpib_id]['freq_step'])

        def get_t_freq_list(dwell_time, recovery_time, f0, f1, df, t_offset, pwr, addr, msg):
          lst = []
          d_freq = f1 - f0
          n_freq = int(d_freq/df)
          for j in range(n_freq+1):
            freq_j = f0 + j*df
            t_j = t_offset + j*(dwell_time+recovery_time)
            lst.append({'t':t_j, 'device_name':'cw', 'gpib_addr':addr, 'freq':freq_j, 'pwr':pwr, 'msg': 'on'})
            t_j2 = t_j+dwell_time
            lst.append({'t':t_j2, 'device_name':'cw', 'gpib_addr':addr, 'freq':freq_j, 'pwr':-999.0, 'msg': 'off'})
          return lst

        l0 = [{'t':t_start_offset+0, 'device_name':'cw', 'gpib_addr':int(gpib_id[-2:]), 'freq':1600.0, 'pwr':-999, 'msg':'off'}]
        l1 = get_t_freq_list(dwell_time, recovery_time, start_freq, stop_freq,  freq_step, l0[-1]['t'] + quiescence_period, pwr, int(gpib_id[-2:]), 'on_off')
        l2 = get_t_freq_list(dwell_time, recovery_time, stop_freq, start_freq, -freq_step, l1[-1]['t'] + quiescence_period, pwr, int(gpib_id[-2:]), 'on_off')
        l2_end = [{'t':l2[-1]['t']+quiescence_period, 'device_name':'cw', 'gpib_addr':int(gpib_id[-2:]), 'freq':1600.0, 'pwr':-999, 'msg':'off_end'}]
        curr_run_list = l0 + l1 + l2 + l2_end
        return curr_run_list
        
      run_list += stepped_run_list_cw(gpib_id,jammer_list,t_start_offset)
      try:
        run_list.sort(key=lambda x: (x['t'], x['gpib_addr']))
      except:
        run_list.sort(key=lambda x: (x['t'], ))
      dt_play = max(dt_play, run_list[-1]['t'])

  for device_id in ['gpib18', 'gpib16', 'hackrf_1', 'chirp_1']:
    
    if device_id in jammer_list and jammer_list[device_id]['freq_type'] == 'Customized':

      def customized_run_list(device_id,jammer_list,t_start_offset):
        
        curr_run_list = []

        def get_t_freq_list_from_arr_cw(t_offset, addr, arr):
          lst = []
          for j in range(len(arr)):
            t_j = float(arr[j]['t']) + t_offset
            pwr = float(arr[j]['pwr'])
            freq_j = float(arr[j]['freq'])
            lst.append({'t':t_j, 'device_name':'cw', 'gpib_addr':addr, 'freq':freq_j, 'pwr':pwr, 'msg': 'on'})
            if 'dwell_time' in arr[j]:
              lst.append({'t':t_j+float(arr[j]['dwell_time']), 'device_name':'cw', 'gpib_addr':addr, 'freq':freq_j, 'pwr':-999.0, 'msg': 'off'})
          return lst
        
        def get_t_freq_list_from_arr_chirp(t_offset, chirp_com_port, arr):
          lst = []
          for j in range(len(arr)):
            t_j = float(arr[j]['t']) + t_offset
            pwr = float(arr[j]['pwr'])
            dwell_time = int(arr[j]['dwell_time'])
            device_id = 'chirp_1'
            lst.append({'t':t_j, 'device_name':device_id, 'com_port':int(chirp_com_port), 'pwr':pwr, 'how_long':dwell_time, 'msg': 'on'})
            lst.append({'t':t_j+dwell_time, 'device_name':device_id, 'com_port':int(chirp_com_port), 'pwr':pwr, 'how_long':0, 'msg': 'off'})
          return lst

        def get_t_freq_list_from_arr_hackrf(t_offset, arr):
          lst = []
          for j in range(len(arr)):
            t_j = float(arr[j]['t']) + t_offset
            pwr = float(arr[j]['pwr'])
            freq_j = float(arr[j]['freq'])
            interference_type = arr[j]['interference_type']
            dwell_time = int(arr[j]['dwell_time'])
            device_id = 'hackrf_1'
            lst.append({'t':t_j, 'device_name':device_id, 'freq':freq_j, 'pwr':pwr,  'type':interference_type, 'how_long':dwell_time, 'msg': 'on'})
            lst.append({'t':t_j+dwell_time, 'device_name':device_id, 'freq':freq_j, 'pwr':pwr,  'type':interference_type, 'how_long':0, 'msg': 'off'})
          return lst

        if device_id in ['gpib18', 'gpib16']:
          curr_run_list = get_t_freq_list_from_arr_cw(t_start_offset, int(device_id[-2:]), jammer_list[device_id]['schedule'], )
        elif device_id == 'hackrf_1':
          curr_run_list = get_t_freq_list_from_arr_hackrf(t_start_offset, jammer_list[device_id]['schedule'], )
        elif device_id == 'chirp_1':          
          chirp_com_port = 21
          for jammer in cfg.jamming_devices:
            if jammer.type == 'Chirp':
              chirp_com_port = jammer.com_port
          curr_run_list = get_t_freq_list_from_arr_chirp(t_start_offset, int(chirp_com_port), jammer_list[device_id]['schedule'], )

        return curr_run_list

      run_list += customized_run_list(device_id,jammer_list,t_start_offset)
      run_list.sort(key=lambda x: (x['t'], ))
      dt_play = max(dt_play, run_list[-1]['t'])

  if 'hackrf_1' in jammer_list and jammer_list['hackrf_1']['freq_type'] == 'Stepped':
    loginfo.print_(fn()+"This test includes stepwise freq change at " + "hackrf_1")

    def stepped_run_list_hackrf(jammer_list, t_start_offset):
      """For example, jammer_list = run_para["jammer_list"] contains:
            "hackrf_1": {
              "freq_type": "Constant",
              "pwr": "1.2",
              "interference_type": "GAUSSIAN_5MHz",
              "freq": "1595.42",
              "dwell_time": "300",
              "quiescence": "300"
            }
      """
      device_id = 'hackrf_1'
      curr_run_list = []
      pwr = float(jammer_list[device_id]['pwr'])
      quiescence_period = float(jammer_list[device_id]['quiescence']) # 5*60 [sec]
      try:
        dwell_time = int(jammer_list[device_id]['dwell_time']) # 30 [sec] #FIXME Currently it only supports integer number of seconds to sleep
      except RuntimeError:
        raiseExceptions("dwell_time needs to be integer for stepped hackrf test")
      recovery_time = float(jammer_list[device_id]['recovery_time']) # 15 [sec]
      start_freq = float(jammer_list[device_id]['start_freq'])
      stop_freq = float(jammer_list[device_id]['stop_freq'])
      freq_step = float(jammer_list[device_id]['freq_step'])
      interference_type = str(jammer_list[device_id]['interference_type'])

      def get_t_freq_list(dwell_time, recovery_time, f0, f1, df, t_offset, pwr, interference_type, device_id, msg):
        lst = []
        d_freq = f1 - f0
        n_freq = int(d_freq/df)
        for j in range(n_freq+1):
          freq_j = f0 + j*df
          t_j = t_offset + j*(dwell_time+recovery_time)
          lst.append({'t':t_j, 'device_name':device_id, 'freq':freq_j, 'pwr':pwr,  'type':interference_type, 'how_long':dwell_time, 'msg': 'on'})
          # msg for postprocessing to find interval
          t_j = t_offset + j*(dwell_time+recovery_time) + dwell_time
          lst.append({'t':t_j, 'device_name':device_id, 'freq':freq_j, 'pwr':pwr,  'type':interference_type, 'how_long':0, 'msg': 'off'})          
        return lst

      l0 = [{'t':t_start_offset+0, 'device_name':'hackrf_1', 'how_long':0, 'msg':'start, do nothing'}]
      l1 = get_t_freq_list(dwell_time, recovery_time, start_freq, stop_freq,  freq_step, l0[-1]['t'] + quiescence_period, pwr, interference_type, device_id, 'on_off')
      l2 = get_t_freq_list(dwell_time, recovery_time, stop_freq, start_freq, -freq_step, l1[-1]['t'] + recovery_time + quiescence_period, pwr, interference_type, device_id, 'on_off')
      l2_end = [{'t':l2[-1]['t'] + recovery_time + quiescence_period, 'device_name':'hackrf_1', 'how_long':0, 'msg':'end, do nothing'}]
      curr_run_list = l0 + l1 + l2 + l2_end
      return curr_run_list

    run_list += stepped_run_list_hackrf(jammer_list, t_start_offset)
    run_list.sort(key=lambda x: (x['t'],))
    dt_play = max(dt_play, run_list[-1]['t'])

  dt_play = dt_play + t_start_offset + t_end_quiescence

  #############################################
  # Show run list partially to get a snapshot #
  #############################################
  loginfo.print_(fn(),"Steps:")
  if verbose:
    for i in range( len(run_list) ):
      loginfo.print_("step",i,":",run_list[i])
  else:
    if len(run_list) > 5:
      for i in range(5):
        loginfo.print_("step",i,":",run_list[i])
      loginfo.print_("...")
      loginfo.print_("...")
      for i in range( len(run_list)-3, len(run_list) ):
        loginfo.print_("step",i,":",run_list[i])
    else:
      for i in range(len(run_list)):
        loginfo.print_("step",i,":",run_list[i])
  
  #########################
  # Check jamming devices #
  #########################
  # FIXME 
  for jammer in cfg.jamming_devices:
    api = ""
    host_ip, host_port = cfg.jamming_host_ip, cfg.jamming_host_port
    if jammer.type == 'HPSignalGen':
      api = '/api/v1/hp8648c/getStatus/'+jammer.gpib_address
    if jammer.type == 'Attenuator':
      api = '/api/v1/dat64l/getStatus/'+jammer.com_port
    if jammer.type == 'Chirp':
      api = '/api/v1/chirp/getStatus'
    if jammer.type == 'HackRF':
      host_ip, host_port = cfg.hackrf_host_ip, cfg.hackrf_host_port
      api = '/api/v1/getHackRFStatus'

    try:
      # if verbose: loginfo.print_("Send api:", api, "to", host_ip, host_port, "for Jamming device", jammer.type)
      r = send_request(host_ip, host_port, api)
      if r.status_code != 200:
        raise RuntimeError('Bad jammer device status by API:', api, '. device:', jammer)
    except:
      raise RuntimeError('Failed to get jammer device status. '+str((host_ip, host_port, api)))

    loginfo.print_("Jamming device", jammer.desc, "status:", r.text[:-1])

  ##############################
  # Turn off CW and attenuator #
  ##############################
  if is_jammer_enabled('HP8648C_GPIB_18', cfg): 
    loginfo.print_("Turn off HP8468c at gpib_18 and Dat64L at the beginning of test.")
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':18, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])
  if is_jammer_enabled('HP8648C_GPIB_16', cfg): 
    loginfo.print_("Turn off HP8468c at gpib_16 and Dat64L at the beginning of test.")
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':16, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])

  #############################
  # Turn off HackRF and Chirp #
  ############################# 
  if is_jammer_enabled('Chirp_1', cfg):
    loginfo.print_("Turn off Chirp at the beginning of test.") 
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOffChirp', {}, user['username'], user['password'])
  if is_jammer_enabled('HackRF_1', cfg):
    loginfo.print_("Turn off HackRF at the beginning of test.") 
    r = send_request(cfg.hackrf_host_ip, cfg.hackrf_host_port, '/api/v1/turnHackRFOff', {}, user['username'], user['password'])

  #####################################################
  # Load, arm and start scenario on Spirent Simulator #
  #####################################################
  def start_spirent_simulator(cfg, loginfo):
    
    if not use_simulator:
      loginfo.print_('Skip using simulator.', use_simulator)
      return 

    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
    data = r.json()
    loginfo.print_("getStatus() returns", data)
    
    # Stop running if needed
    if data['msg']['status'] == str(5): # Running
      loginfo.print_("Stop Running Spirent")
      r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/stopScenario/', None, user['username'], user['password'])
      time.sleep(5)
      r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
      data = r.json()

    # Close scenario if needed
    if data['msg']['status'] == str(7) or data['msg']['status'] == str(2): # Ended or Ready
      loginfo.print_("Close Scenario")
      r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/closeScenario/', None, user['username'], user['password'])
      time.sleep(5)
      r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
      data = r.json()

    # Load scenario
    if data['msg']['status'] == str(0): # No scenario specified
      loginfo.print_("No scenario specified. Load scenario file.")
      if 'spirent_scn' in run_para and run_para['spirent_scn'] == 'reduced': # if scenario 1b
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/loadScenario/', {'scenario_file':'reduced_signal'}, user['username'], user['password'])
      elif 'spirent_scn' in run_para: # if using another scenario file
        loginfo.print_("scenario file loading:", run_para['spirent_scn'])
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/loadScenario/', {'scenario_file': run_para['spirent_scn']}, user['username'], user['password'])
      else: # if anything else
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/loadScenario/', None, user['username'], user['password'])

      time.sleep(5)

    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
    data = r.json()
    loginfo.print_("Finally, getStatus() returns", data)
    
    standpoint = False
    # Start scenario
    if 'standpoint' in run_para:
        standpoint = run_para['standpoint']

    if not standpoint:
        loginfo.print_("Start Spirent")
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/startScenarioWithoutStatus/', None, user['username'], user['password'])
    else:
        loginfo.print_("Start Standpoint")
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/startStandpoint/', None, user['username'], user['password'])
        time.sleep(160)
  
  sim_start_offset = 0
  if 'sim_start_offset' in run_para:
    sim_start_offset = run_para['sim_start_offset']

  if sim_start_offset == 0:
    start_spirent_simulator(cfg, loginfo)
    loginfo.print_("Wait for Spirent simulator to finish(in", dt_play, "sec)...")
 
  start_time = time.time()
  
  curr = 0
  spirent_checked = False
  logging_has_started_after_60sec = False
  per5sec_ctr = 0
  
  standpointStopped = False
  while True:


    ###########################################
    # Check if Standpoint needs to be stopped #
    ###########################################
    standpoint = False
    sim_start_offset = 0
    sim_started = False
    if 'standpoint' in run_para:
        standpoint = True
    if 'sim_start_offset' in run_para:
        sim_start_offset = run_para['sim_start_offset']
    
    if sim_start_offset > 0 and time.time() - start_time > sim_start_offset and not sim_started:
        loginfo.print_("Starting simulator after start offset")
        start_spirent_simulator(cfg,loginfo)
        sim_started = True
    
    if 'standpoint_duration' in run_para and not standpointStopped:
        if time.time() - start_time > run_para['standpoint_duration']:
            loginfo.print_("Stopping Standpoint")
            standpointStopped = True
            #r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/stopStandpoint/', None, user['username'], user['password'])
            r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/stopStandpoint/', None, user['username'], user['password'])


    ######################
    # Check when to stop #
    ######################
    if time.time() - start_time > dt_play:

      # Turn off jammers at the end
      loginfo.print_("Turn off HP8468c and Dat64L.")
      if is_jammer_enabled('HP8648C_GPIB_18', cfg): 
        r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':18, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])
      if is_jammer_enabled('HP8648C_GPIB_16', cfg): 
        r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':16, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])
      if is_jammer_enabled('Chirp_1', cfg):
        r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOffChirp', {}, user['username'], user['password'])
      if is_jammer_enabled('HackRF_1', cfg):
        r = send_request(cfg.hackrf_host_ip, cfg.hackrf_host_port, '/api/v1/turnHackRFOff', {}, user['username'], user['password'])
      break

    #############
    # Check t_i #
    #############
    
    if curr < len(run_list) and time.time() - start_time > run_list[curr]['t']:
      
      if verbose and use_simulator and sim_start_offset == 0:
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getTOW/')
        data = r.json()
        loginfo.print_("spirentTool/getTOW returns", data)
      loginfo.print_("Step ",curr,":",run_list[curr])
      
      if 'gpib_addr' in run_list[curr].keys():
        # Adjust CW
        # eg. curl -u username:password -X GET -i "http://10.1.141.71:10002/api/v1/turnOnCW?gpib_addr=18&pwr=-25.0&freq=1575.42"
        # eg. curl -u username:password -X GET -i "http://10.1.141.71:10002/api/v1/turnOnCW?gpib_addr=18&pwr=1.2&freq=1575.42"
        # eg. curl -u username:password -X GET -i "http://10.1.141.71:10002/api/v1/turnOffAllCWs"
        if (is_jammer_enabled('HP8648C_GPIB_18', cfg) and run_list[curr]['gpib_addr'] == 18) or (is_jammer_enabled('HP8648C_GPIB_16', cfg) and run_list[curr]['gpib_addr'] == 16):
          settings = (cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':run_list[curr]['gpib_addr'], 'pwr':run_list[curr]['pwr'], 'freq':run_list[curr]['freq'] })
          loginfo.print_('Send settings:', settings)
          r = send_request(*settings, user['username'], user['password'])
        curr += 1
      elif 'device_name' in run_list[curr].keys() and run_list[curr]['device_name'] == 'chirp_1':
        # Adjust Chirp
        if is_jammer_enabled('Chirp_1', cfg):
          loginfo.print_("Turn on Chirp in background for",run_list[curr]['how_long'],"sec")
          if run_list[curr]['how_long'] > 0:
            settings = (cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnChirp', {'com_port':run_list[curr]['com_port'], 'desired_pwr_dBm':run_list[curr]['pwr'], 'sec':run_list[curr]['how_long'] })
            loginfo.print_("Send cmd:", settings)
            proc = multiprocessing.Process(target=send_request, args=[*settings, user['username'], user['password'], run_list[curr]['how_long']+1] )
            proc.start()
            proc_q.put(proc) # After implementing turnOffChirp, this is not needed maybe.
        curr += 1
      elif 'device_name' in run_list[curr].keys() and run_list[curr]['device_name'] == 'hackrf_1':
        # Adjust HackRF
        if is_jammer_enabled('HackRF_1', cfg):
          loginfo.print_("Turn on HackRF in background for",run_list[curr]['how_long'],"sec")
          if run_list[curr]['how_long'] > 0:
            # New method: Func call to get response immediately.
            settings = (cfg.hackrf_host_ip, cfg.hackrf_host_port, '/api/v1/turnHackRFOn', {'interference_type':run_list[curr]['type'], 'center_freq_MHz':run_list[curr]['freq'], 'desired_pwr_dBm':run_list[curr]['pwr'], 'sec':run_list[curr]['how_long'] })
            loginfo.print_('Send settings:', settings)
            r = send_request(*settings, user['username'], user['password'])
            if verbose: loginfo.print_("send_request() returns", r, r.json())
            if r.status_code != 200:
              raise RuntimeError("TurnHackRFOn failed.")
            else:
              loginfo.print_("Turn on HackRF.")

        curr += 1

    time.sleep(0.1)

    # #######################################
    # # Check Spirent Simulator status once #
    # # Wait 10 sec to check Spirent Status #
    # # Set Alloy streaming RTCM if do RTK  #
    # #######################################

    if (not standpoint and time.time() - start_time > 10) or (standpoint and time.time() - start_time > 70)  and spirent_checked == False:
      # Check spirent status
      if use_simulator and sim_start_offset == 0: 
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/')
        data = r.json()
        if data['msg']['status'] != str(5): # Running
          loginfo.print_("Spirent simulator status:", data['status description'])
          loginfo.print_("Bad Spirent Status:", data['msg']['status'], data['status description'], 'Abort.')
          raise RuntimeError("Bad Spirent status")
        else:
          loginfo.print_("Spirent simulator status:", data['status description'])
      else:
        loginfo.print_("Skip check spirent simulator status.")
      spirent_checked = True

      # Enable alloy RTK if do_RTK
      if 'do_rtk' in run_para:
        loginfo.print_('Find do_rtk in run_para.')
        rtk_source_ip_address = run_para['do_rtk']['rtk_source_ip_address']
        rtk_source_port = int(run_para['do_rtk']['rtk_source_port'])
        loginfo.print_('RTK correction from', rtk_source_ip_address,':',rtk_source_port)

        # 1 Spirent Started
        # 2 Softreset alloy
        alloy_ip_address = '10.1.140.124'
        print("rtk_source_ip_address value")
        print(rtk_source_ip_address)
        if rtk_source_ip_address == alloy_ip_address:
          loginfo.print_("RTK source from alloy: "+alloy_ip_address+". Reset Alloy.")
          loginfo.print_("Reboot "+ alloy_ip_address)
          rx.SoftReset(alloy_ip_address, 'admin', 'password')
          loginfo.print_("Make sure to set alloy's Antenna measurement method = Antenna Phase Center")
        else:
          raise RuntimeError("Testing cube setup only supports Alloy currently.")
        
        def set_reference_station_llh(loginfo, IPAddr, username, password, data):

          # Set Reference Station LLH
          loginfo.print_('Set reference station LLH at ', IPAddr)
          use_what = 'DirectInput' # 'Here', 'Avg'
          if use_what == 'Here':
            # Use Here
            lat = str(data['lat'])
            lon = str(data['lon'])
            hgt = str(float(data['hgt'])-float(data['refHgtCorr']))
          elif use_what == 'Avg':
            # Use Avg
            lat = str(data['avgLat'])
            lon = str(data['avgLon'])
            hgt = str(float(data['avgHt'])-float(data['refHgtCorr']))
          elif use_what == 'DirectInput':
            # Use Input
            lat = str(data['lat'])
            lon = str(data['lon'])
            hgt = str(data['hgt'])

          cmd = '/cgi-bin/refPosPage.xml?CMRID=0&RTCMID=0&RTCM3ID=0&RTXSTNID=0&StationName='+str(data['StationName'])+'&StationCode=&coordSystem='+'Geographical'+'&refLatDeg=0&refLatMin=0&refLatSec=0&refLatSign=0&refLonDeg=0&refLonMin=0&refLonSec=0&refLonSign=0&refHgt='+str(hgt)+'&refX=0&refY=0&refZ=0&herePositionUsed=1&hereLat=0&hereLon=&hereHgt=0&refLat='+str(lat)+'&refLon='+str(lon)+'&hereX=0&hereY=0&hereZ=0&AutoAvgTime=120'
          
          raw = rx.SendHttpGet(IPAddr, cmd, username, password)
          loginfo.print_('Set reference station at', IPAddr,' returns', raw)

        if info.fileName == 'AugerTesting.scn':

          loginfo.print_("Sleep 75 sec before setting reference LLH.")
          time.sleep(75)

          # For AugerTesting.scn only
          llh_data = {'StationName':'CREF0001',
                      'lat':37.38333333333333,
                      'lon':-122.0,
                      'hgt':-3.999
                      }

          # 4 Set Auger TCPIP to receive RTCM
          for device in cfg.devices:
            if device.n in selected_devices_list and device.septentrio is None and device.novatel is None:

              # 4.1 set reference station LLH
              loginfo.print_("After 75 sec, set RX-"+str(device.n)+"'s reference station LLH.")
              set_reference_station_llh(loginfo, device.ip, device.user, device.pw, llh_data)
              
              # 4.2 set IO to receive RTCM message
              port = '28001' #'5017'
              remote_address = alloy_ip_address #'10.1.140.124'
              remote_port = '5017'
              cmd = '/cgi-bin/io.xml?port=22&LocalPort='+port+'&type=TCP/IP&Client=on&OutputOnly=off&OutputOnlyRadio=on&timeout=60&udpTxBroadcastPort='+port+'&fakePassword=0&auth_pass=&auth_pass2=&RemoteAddr='+remote_address+'&RemotePort='+remote_port+'&datatype=cmr&CMR=0&CMRDelay=0&BWLimit=0&OK=1'
              raw = rx.SendHttpGet(device.ip, cmd, device.user, device.pw)
              loginfo.print_('Set RX-'+str(device.n)+' TCPIP to receive RTCM: ',raw)

              # 4.3 set RTK Correction Controls
              raw = rx.SendHttpGet(device.ip,'/xml/dynamic/corrMgr.xml?ruleSet=RTK', device.user, device.pw)
              loginfo.print_('Set RX-'+str(device.n)+' RTK to Any Channel under Correction Controls. Return',raw)

              # 4.4 set DGNSS Correction Controls
              raw = rx.SendHttpGet(device.ip,'/xml/dynamic/corrMgr.xml?ruleSet=DGNSS', device.user, device.pw)
              loginfo.print_('Set RX-'+str(device.n)+' DGNSS to Any Channel under Correction Controls. Return',raw)

          # 5 Toggle septentrio 'Connection Input'
          for device in cfg.devices:
            if device.n in selected_devices_list and device.septentrio is not None:
              
              # reset Alloy (RTK source) then toggle septentrio input stream setting IPR1:auto/RTCMv3
              loginfo.print_("Toggle Septentrio input stream setting.")
              if time.time() - start_time > 35:
                loginfo.print_("Skip sleep.")
              else:
                loginfo.print_("Sleep 25 sec.")
                time.sleep(25)
              loginfo.print_("This is Septentrio receiver %s %s"%(device.ip, device.port))
              Sept = sept_rx.Septentrio_Socket(device.ip, int(device.port))
              rt = Sept.set_data_in_out('IPR1','none','none')
              loginfo.print_("run_setup "+"set_data_in_out():","cmd sent", rt.split('\n')[0])
              loginfo.print_("run_setup "+"set_data_in_out():","returns", rt.split('\n')[1])
              loginfo.print_("Sleep 5 sec")
              time.sleep(5)
              rt = Sept.set_data_in_out('IPR1','auto','none')
              loginfo.print_("run_setup "+"set_data_in_out():","cmd sent", rt.split('\n')[0])
              loginfo.print_("run_setup "+"set_data_in_out():","returns", rt.split('\n')[1])

              # Debug
              loginfo.print_("run_setup "+"set_data_in_out() ASCIIDisplay for debugging.")
              # rt = Sept.set_data_in_out('IPR2','none','ASCIIDisplay')
              rt = Sept.set_data_in_out('IP10','none','ASCIIDisplay') # IP10
              loginfo.print_("run_setup "+"set_data_in_out():","cmd sent", rt.split('\n')[0])
              loginfo.print_("run_setup "+"set_data_in_out():","returns", rt.split('\n')[1])
              # Debug end
              del Sept

          # Novatel does RTK automatically 

    ############################
    # Check Stop Button Status #
    ############################
    # Check every 5 sec by reading status json file from diskspace
    if time.time() - start_time > t_start_offset and (time.time() - start_time)/5 > per5sec_ctr:
      per5sec_ctr += 1
      # Enable buttom to stop
      try:
        stop_status = get_regression_stop_status()
        # loginfo.print_("stop_status:", stop_status)
        if stop_status['stop_status'] == 'Unknown':
          loginfo.print_("Enable stop button.")
          set_regression_stop_status('Enabled')
        # Check if stop
        if stop_status['stop_status'] == 'Stop':
          loginfo.print_("Stop the run by setting dt_play = 0.")
          dt_play = 0 # Stop the run by changing dt_play
      except:
        loginfo.print_("Call get_regression_stop_status() failed.")

    # ###############################################################################################
    # # Temporal solution: restart logging after t_start_offset(60 sec) until receiver acquire TOW  #
    # ###############################################################################################
    if time.time() - start_time > t_start_offset and logging_has_started_after_60sec == False:
      # Restart logging
      total_devices = copy.copy(cfg.devices)
      do_devices = [x for x in total_devices if x.n in selected_devices_list]
      do_devices_id = [x.n for x in total_devices if x.n in selected_devices_list]
      loginfo.print_("Restart logging after", t_start_offset, "sec for devices #",do_devices_id)
      
      enableLogging = True
      if 'standpoint' in run_para:
          enableLogging = False

      with multiprocessing.pool.ThreadPool(len(do_devices)) as p:
        def wrap_start_logging( device ):
          try:
            return start_logging( proc_q, loginfo, device, info, reset_clean=False, enable_logging=enableLogging, set_bbsampling_mode=False, selected_devices_bbsampling_mode_afterIM={}, devices_filter_type_dict=devices_filter_type_dict)
          except:
            return 'error'
        results = p.map( wrap_start_logging, do_devices )

      logging_has_started_after_60sec = True

  set_regression_stop_status('Unknown') # FIXME
  
  return None, try_again, run_freq_type, run_list


def check_runq(loginfo, proc_q, cmd, log=None, exit_on_err=True, run_in_bg=False):
  """Inputs: loginfo = LogInfo() - where to log diagnostics
             proc_q = Queue() - list of background processes started
             cmd = shell command to run
             log = if True, output to Builds/log.txt
             exit_on_err = if True, stop on error.
             run_in_bg = if True, don't block
  Run the shell command 'cmd'.  Put processes in 'proc_q' in case the
  user sends a 'Kill' signal.
  """
  loginfo.print_("Running shell cmd: %s" % cmd)
  if log is not None:
    fout = open('Builds/log.txt', 'w')
    p = subprocess.Popen(['bash','-c',cmd],stdout=fout,stderr=fout)
  else:
    p = subprocess.Popen(['bash','-c',cmd])
  proc_q.put(p.pid)
  if run_in_bg:
    ret = 0
  else:
    ret = p.wait()
  if log is not None:
    fout.close()
  if ret != 0 and exit_on_err:
    raise RuntimeError('Error in cmd',cmd)


def stream_log(ip='10.1.xxx.xxx', port=3004, output_filename = 'data_stream.txt'):
  ''' This is designed for novatel receiver.
  Novatel receiver does not have diskspace to save/log data.
  This will capture the data streamed out from novatel receiver's port which needs to be configured by user.
  '''
  cmd = 'nc '+ip+' '+str(port)+' > '+ output_filename
  if verbose: print('cmd:',cmd)
  os.system(cmd)


def run_setup(loginfo, device, info, cfg, proc_list, skip_reboot=False, devices_filter_type_dict={}, run_para=None):
  ''' Inputs: loginfo = LogInfo() - where to log diagnostics
              device = Device() - from config.xml
              info = ScenarioInfo - info from current scenario XML
              cfg = ConfigXML = info from config.xml
              proc_list = list of processes started by this function
              skip_reboot = True/False
              devices_filter_type_dict = dictionary int rx.n -> string anti-jam filter type
             
  Set up a receiver before the RF playback starts.
  1. Delete T0x and almanac/eph data
  2. Configure receiver settings
  3. Install scenario almanac data
  4. Reboot the receiver
  '''

  # Check if do RTK
  do_rtk = False
  rtk_source_ip_address = None
  rtk_source_port = None
  
  if 'do_rtk' in run_para:
    loginfo.print_('Find do_rtk in run_para')
    do_rtk = True
    rtk_source_ip_address = run_para['do_rtk']['rtk_source_ip_address']
    rtk_source_port = int(run_para['do_rtk']['rtk_source_port'])

  if device.septentrio is not None:
    IPAddr = device.ip
    loginfo.print_("This is Septentrio receiver %s %s"%(device.ip, device.port))
    Sept = sept_rx.Septentrio_Socket(device.ip, int(device.port))
    # Disable logging
    loginfo.print_("Disable logging",device.ip, device.port)
    Sept.disable_logging()

    # Remove log files
    flist = Sept.list_ftp_files()
    loginfo.print_("Clean old log files",IPAddr,flist)
    for fname in flist:
      Sept.remove_file(fname)

    # Enable anti jamming notch filter
    if device.n in devices_filter_type_dict:
      filter_type_str = devices_filter_type_dict[device.n]
      if filter_type_str == 'None':
        loginfo.print_("run_setup "+"set_notch_filter(off), set_WBI_mitigation(off)")
        Sept.set_notch_filter('off')
        Sept.set_WBI_mitigation('off')
      elif filter_type_str == 'Notch':
        loginfo.print_("run_setup "+"set_notch_filter(auto), set_WBI_mitigation(off)")
        Sept.set_notch_filter('auto')
        Sept.set_WBI_mitigation('off')
      elif filter_type_str == 'WideBand':
        loginfo.print_("run_setup "+"set_notch_filter(off), set_WBI_mitigation(on)")
        Sept.set_notch_filter('off')
        Sept.set_WBI_mitigation('on')
      elif filter_type_str == 'Notch + WideBand':
        loginfo.print_("run_setup "+"set_notch_filter(auto), set_WBI_mitigation(on)")
        Sept.set_notch_filter('auto')
        Sept.set_WBI_mitigation('on')
      else:
        loginfo.print_("run_setup: "+'Skip setting anti jam filter. filter_type_str =',str(filter_type_str))
      
      rt = Sept.get_notch_filter()
      loginfo.print_("run_setup "+"get_notch_filter():", "cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"get_notch_filter():", "returns", rt.split('\n')[1])
      rt = Sept.get_WBI_mitigation()
      loginfo.print_("run_setup "+"get_WBI_mitigation():", "cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"get_WBI_mitigation():", "returns", rt.split('\n')[1])

    # Reset Position Mode
    rt = Sept.set_pvt_mode(mode='Static',rover_mode='RTK',static_pos='auto')
    loginfo.print_("run_setup "+"set_pvt_mode():", "cmd sent", rt.split('\n')[0])
    loginfo.print_("run_setup "+"set_pvt_mode():", "returns", rt.split('\n')[1])

    # Reset IPR1,2,3 data in/out
    for port_id in range(1,4):
      rt = Sept.set_data_in_out('IPR'+str(port_id),'auto','none')
      loginfo.print_("run_setup "+"set_data_in_out():", "cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"set_data_in_out():", "returns", rt.split('\n')[1])
      rt = Sept.set_ip_receive(ip_receve='IPR'+str(port_id), port=0, tcp_addr='')
      loginfo.print_("run_setup "+"set_ip_receive():", "cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"set_ip_receive():", "returns", rt.split('\n')[1])
    
    if do_rtk:
      # Steps for Septentrio to do RTK
      # Order matters: 
      # 0. Assuming simulator has been turned off already.
      # 1. Config Septentrio ethernet port(IPR1) to receiver message from RTK Source.
      #    Sept.set_ip_receive(ip_receve='IPR1', port=rtk_source_port, tcp_addr=rtk_source_ip_address)
      # 2. Config Septentrio into RTK mode.
      #    Sept.set_pvt_mode(mode='Rover',rover_mode='RTK',static_pos='auto')
      # 3. Start Spirent.
      # 4. SoftReset Alloy after spirent start. (IMPORTANT!)
      # 5. Toggle septentrio input/output setting on port IPR1 to re-detect and re-read input after Spirent started and Alloy reset. (IMPORTANT!)
      #    Sept.set_data_in_out('IPR1','none','none')
      #    Sept.set_data_in_out('IPR1','auto','none')

      loginfo.print_("run_setup "+"Turn on RTK")
      loginfo.print_("When using Alloy as source of RTK, do setup after spirent starts")
      
      rt = Sept.set_ip_receive(ip_receve='IPR1', port=rtk_source_port, tcp_addr=rtk_source_ip_address)
      loginfo.print_("run_setup "+"set_ip_receive():","cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"set_ip_receive():","returns", rt.split('\n')[1])

      rt = Sept.set_pvt_mode(mode='Rover',rover_mode='RTK',static_pos='auto')
      loginfo.print_("run_setup "+"set_pvt_mode():","cmd sent", rt.split('\n')[0])
      loginfo.print_("run_setup "+"set_pvt_mode():","returns", rt.split('\n')[1])

    rt = Sept.get_BBSampling_mode()
    loginfo.print_("run_setup get BBSampling Mode cmd sent",device.ip,":", rt.split('\n')[0])
    loginfo.print_("run_setup get BBSampling Mode returns",device.ip,":", rt.split('\n')[1])

    # Septentrio receiver must be reset.
    # Without reset, zero mearsurment will be converted to T04 by sbf2t0x for repeated run.
    loginfo.print_("run_setup-reboot "+device.ip+" Reset Septentrio is needed.")
    Sept.soft_reset()
    loginfo.print_("run_setup sleep 30 sec after reboot "+device.ip)
    time.sleep(30)
    loginfo.print_("run_setup sleep 30 sec done "+device.ip)
    
    del Sept
  elif device.novatel is not None:
    IPAddr = device.ip
    loginfo.print_("This is Novatel receiver %s %s"%(device.ip, device.port))
    
    # 1) Reset the receiver
    # Novatel receiver must be reset.
    Nov = novt_rx.Novatel_Socket(device.ip, int(device.port), output_format='abbreviated_ascii' )
    loginfo.print_("run_setup factory reset "+device.ip)
    Nov.factory_reset()
    del(Nov)
    loginfo.print_("run_setup sleep 30 sec after reset "+device.ip)
    time.sleep(30)

    # 2) Set channel configuration
    # Receiver will reboot if channel configuration is changed
    Nov = novt_rx.Novatel_Socket(device.ip, int(device.port), output_format='abbreviated_ascii' )
    Nov.set_channel_config_mode(4)
    del(Nov)
    loginfo.print_("run_setup sleep 30 sec after channel configuration "+device.ip)
    time.sleep(30)

    # 3) Configure receiver
    Nov = novt_rx.Novatel_Socket(device.ip, int(device.port), output_format='abbreviated_ascii' )

    # Check channel configuration
    rt = Nov.get_channel_config_mode()
    # loginfo.print_('\n'.join(rt.split('\r\n')[:4]))
    loginfo.print_('channel configuration:' + rt.split('\r\n')[3])

    # Set to 100 - default CC setting
    all_sigs="""GPSL1CA GPSL2Y GPSL2P GPSL2C GPSL5 
              GLOL1CA GLOL2CA GLOL2P
              GALE1 GALE5A GALE5B GALALTBOC
              SBASL1 SBASL5
              BDSB1D1 BDSB1D2 BDSB2D1 BDSB2D2 BDSB3D1 BDSB3D2 BDSB1C BDSB2A BDSB2BI
              QZSSL1CA QZSSL2CM QZSSL5
              """

    for sig in all_sigs.replace('\n','').split(" "):
      if len(sig) > 0:
        Nov.set_dlltime(sig, 100)

    # "true" should force it to also unlog the RXSTATUSEVENTA messages
    Nov.unlogall()

    # Stream out observables and positions on ICOM4 (TCP/IP port 3004) to save data
    port = 'ICOM4'
    interval = '0.2'
    for meas in ['RANGEA', 'BESTPOSA', 'BESTVELA', 'GPGGALONG']:
      Nov.passthrough_log(port, meas, interval)

    # Stream out NMEA on ICOM2 (TCP/IP port 3002) to let python script find synced RTCM message to feed back for doing RTK
    port = 'ICOM2'
    interval = '0.2'
    for meas in ['GPZDA', 'GPGST', 'GPGSV', 'GPVTG', 'GPGSA']:
      Nov.passthrough_log(port, meas, interval)

    if do_rtk:
      # Stream in RTCM on port ICOM3 (port 3003) to do RTK
      Nov.set_interface_mode('ICOM3','RTCMV3','NOVATEL', 'OFF')

    # Disable the PDP filter.
    Nov.set_pdpfilter(False)

    # Set to zero degrees to match our testing
    Nov.set_elev_mask(0)

    # Set anti jamming filter
    if device.n in devices_filter_type_dict:
      filter_type_str = devices_filter_type_dict[device.n]
      if filter_type_str == 'None':
        loginfo.print_("run_setup "+"set filter off.")
        Nov.set_it_prog_config(False)
      elif filter_type_str == 'Notch Auto':
        loginfo.print_("run_setup "+"set notch filter on."+" Not implemented yet. SKIP.")
        # FIXME 
        # This could be done with feedback loop control
      elif filter_type_str.split('_')[0] == 'Notch Manual':
        # <          GPSL1 5
        #            mode        min_lower max_lower min_upper max_upper frequency_step notch_width
        #                        frequency
        #                        cutoff
        #                        (Notch Filter can not be set at 1574.42 MHz)
        #                        (BandPass Filter cut-off freq. specified as input is mirrored by 1575 MHz)
        #                        (No multiple filters)
        # <          NOTCHFILTER 1563.0000 1574.0000 1576.0000 1587.0000 0.05 0.15
        # <          NOTCHFILTER 1563.7500 1573.6000 1576.4000 1586.2500 0.05 0.50
        # <          NOTCHFILTER 1564.0500 1573.3000 1576.7000 1585.9500 0.05 1.00
        # <          NOTCHFILTER 1565.7500 1571.7000 1578.3000 1584.2500 0.05 2.50
        # <          BANDPASSFILTER 1563.7500 1572.5000 1577.5000 1586.2500 0.05 0.00
        arr = filter_type_str.split('_')
        freq = arr[1]
        width = arr[2]
        loginfo.print_("run_setup "+"set notch filter on, center frequency(MHz)="+str(freq)+", bandwidth(MHz)="+str(width))
        Nov.set_it_prog_config(switch_on=True, filter_mode='notchfilter', cut_off_freq=freq, notch_width=width)
      elif filter_type_str.split == 'BandPass':
        arr = filter_type_str.split('_')
        freq = arr[1]
        loginfo.print_("run_setup "+"set notch filter on, cut-off frequency(MHz)="+str(freq))
        Nov.set_it_prog_config(switch_on=True, filter_mode='bandpassfilter', cut_off_freq=freq)
      else:
        loginfo.print_("run_setup: "+'Skip setting anti jam filter. filter_type_str='+str(filter_type_str))

    rt = Nov.get_it_prog_config()
    loginfo.print_("run_setup "+"set filter settings:", "returns", rt.split('\n')[1])

    # Show config so far for debugging
    nov_rx_config = Nov.get_rxconfig()

    # Save rxconfig
    fname = 'RX'+str(device.n)+"_rxconfig.txt"
    with open(fname, 'w') as f:
      f.write(nov_rx_config)

    out_dir = "DataDir/RX%d-%d/"% (device.n,cfg.GlobalRunNum)
    if verbose: loginfo.print_("Make directory %s" % out_dir)
    os.mkdir(out_dir)
    
    if os.path.isfile(fname):
      shutil.move(fname, out_dir)

    # Novatel OEM729 has no internal storage, does not support usb stick, and has no option to run FTP_server.
    # logging is achieved by dump data from ICOM4 to txt file
    # nc 10.1.xxx.xxx 3004 > com4.log
    log_filename = 'RX'+str(device.n)+"_obs.txt"
    proc = multiprocessing.Process(target=stream_log, kwargs={"ip":device.ip,"port":3004, "output_filename":log_filename})
    proc.start()
    loginfo.print_("Add stream_log() PID="+str(proc.pid)+" into proc_list")
    proc_list.append(proc)

    loginfo.print_("run_setup done "+device.ip)

  else:
    # rx.sendDColAntennaOffCmd() is not available in
    # released firmware, so we can't rely on it.
    # The Spirent simulator should be connected and the RF signal
    # off, so we shouldn't get more GNSS data.

    # Disable T0X logging
    loginfo.print_("run_setup "+device.ip)
    rx.DisableDefaultLogging(device.ip,device.user,device.pw)
    time.sleep(5)

    # Some commands below only work in test mode
    rx.EnableTestMode(device.ip,device.user,device.pw)
    loginfo.print_("run_setup-delete files "+device.ip)
    # Delete the GNSS alm/eph etc
    rx.DeleteGNSSData(device.ip,device.user,device.pw)
    # Delete all the files in /Internal
    rx.getFileListAndDelete(device.ip,device.user,device.pw,'/Internal/','T02')
    rx.getFileListAndDelete(device.ip,device.user,device.pw,'/Internal/','T04')
    # Install clone file - receiver settings only

    skip_clone_install = False
    if ('skip_clone_install' in run_para['receiver_list'][str(device.n)]):
        if (run_para['receiver_list'][str(device.n)]['skip_clone_install'] == True):
            skip_clone_install = True

    if (skip_reboot == False and skip_clone_install == False):
      loginfo.print_("run_setup-install clone "+device.ip)
      rx.UploadClone(device.ip,device.user,device.pw,device.settingsClone, n_retries=5)
      rx.InstallClone(device.ip,device.user,device.pw,device.settingsClone,clear=True)
    else:
      loginfo.print_("Skip clone install")

    # Anti Jamming filters' setting will be overwritten after reboot. 
    # It will be set after reboot after Spirent started.
    loginfo.print_('run_setup Anti-Jamming filters will be set after reboot when Spirent is running.')

    ##################################################
    # Reboot the receiver
    ##################################################
    if skip_reboot == False:
      loginfo.print_("run_setup-reboot "+device.ip)
      rx.SoftReset(device.ip,device.user,device.pw)
      # Note: cleaning GNSS data will cause cycle slips when SV generation changes
      # 5-15 minutes into the simulation. Use with caution!
      # cleanSVDataCmd = "/prog/reset?gnssData"
      # rx.SendHttpPostRetry(device.ip,cleanSVDataCmd,device.user,device.pw)
      time.sleep(90)
      loginfo.print_("run_setup-reboot Done."+device.ip)
    else:
      loginfo.print_("Skip reboot")

    if device.septentrio is None and device.novatel is None:
      try:
        rx.sendDColRefWeek(device.ip, 5017, 2020)
      except:
        loginfo.print_("failed to set ref week")


def download_files_at_end_of_test( loginfo, proc_q, cfg, selected_devices_list=[] ):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             proc_q = Queue() - list of background processes started
             cfg = ConfigXML = info from config.xml
             selected_devices_list = list of integer numbers defined by <n>?</n> in config.xml file
  Download T0x files at end of a single RF playback
  '''
  if verbose: loginfo.print_(fn(),"Runtime selected_devices_list: %s"%str(selected_devices_list))
  for device in cfg.devices:
    if device.n in selected_devices_list:

      loginfo.print_("download files.. #",device.n)
      cfg.current_config_device = device

      if device.septentrio is not None:
        loginfo.print_("This is Septentrio receiver %s" % device.ip)
        Sept = sept_rx.Septentrio_Socket(device.ip, int(device.port))
        # Turn off logging
        if verbose: loginfo.print_("Turn off logging. %s" % device.ip)
        Sept.disable_logging()
        # Download file
        sbf_filename = "log_rx"+str(device.n)+".sbf"
        if verbose: loginfo.print_("Download file from ftp. %s %s" % (device.septentrio.ftp_hostname, device.septentrio.ftp_path))

        try_cnt = 2
        while try_cnt > 0:
          try:
            rt = Sept.download_file_ftp(filename = sbf_filename, ftp_hostname = device.septentrio.ftp_hostname, path = device.septentrio.ftp_path)
            try_cnt = 0
          except:
            loginfo.print_("Final download error! Sleep 10s. Try again.")
            try_cnt -= 1
            if try_cnt == 0:
              raise RuntimeError("Cannot download files at end of test")
            time.sleep(10)
            
        if rt == True:
          out_dir = "DataDir/RX%d-%d/"% (device.n,cfg.GlobalRunNum)
          if verbose: loginfo.print_("Make directory %s" % out_dir)
          if os.path.exists(out_dir):
            loginfo.print_("Directory %s exists." % out_dir)
          else:
            if verbose: loginfo.print_("Make directory %s" % out_dir)
            os.mkdir(out_dir)
          if verbose: loginfo.print_("Move files %s" % sbf_filename)
          dest = shutil.move(sbf_filename, out_dir)
        else:
          loginfo.print_("Downloading files failed #",device.n)
          raise RuntimeError("Cannot download files at end of test")

        # Reset Position Mode, data in/out, 'IP Reveive Settings'
        loginfo.print_("After run is finished, reset septentrio Position Mode, data in/out, 'IP Reveive Settings'",device.ip)
        rt = Sept.set_pvt_mode(mode='Static',rover_mode='RTK',static_pos='auto')
        loginfo.print_("run_setup "+"set_pvt_mode():", "cmd sent", rt.split('\n')[0])
        loginfo.print_("run_setup "+"set_pvt_mode():", "returns", rt.split('\n')[1])
        for port_id in range(1,4):
          rt = Sept.set_data_in_out('IPR'+str(port_id),'auto','none')
          loginfo.print_("run_setup "+"set_data_in_out():", "cmd sent", rt.split('\n')[0])
          loginfo.print_("run_setup "+"set_data_in_out():", "returns", rt.split('\n')[1])
          rt = Sept.set_ip_receive(ip_receve='IPR'+str(port_id), port=0, tcp_addr='')
          loginfo.print_("run_setup "+"set_ip_receive():", "cmd sent", rt.split('\n')[0])
          loginfo.print_("run_setup "+"set_ip_receive():", "returns", rt.split('\n')[1])
        del Sept
      elif device.novatel is not None:
        loginfo.print_("This is Novatel receiver %s" % device.ip)
        loginfo.print_("Turn off logging.")
        Nov = novt_rx.Novatel_Socket(device.ip, int(device.port), output_format='abbreviated_ascii' )
        Nov.unlogall()
        time.sleep(0.5)
        log_filename = 'RX'+str(device.n)+"_obs.txt"
        out_dir = "DataDir/RX%d-%d/"% (device.n,cfg.GlobalRunNum)
        if os.path.exists(out_dir):
          loginfo.print_("Directory %s exists." % out_dir)
        else:
          if verbose: loginfo.print_("Make directory %s" % out_dir)
          os.mkdir(out_dir)
        loginfo.print_("Move file "+log_filename+" to "+out_dir)
        if os.path.isfile(log_filename):
          shutil.move(log_filename, out_dir)
          loginfo.print_("Move file "+log_filename+" to "+out_dir+". Done.")
      else:
        try_cnt = 2
        while try_cnt > 0:
          try:
            out_dir = "DataDir/RX%d-%d"% (device.n,cfg.GlobalRunNum)
            rx.DisableDefaultLogging(device.ip,device.user,device.pw)
            rx.GetLoggedFilesViaHttpNoDisable( device.ip, # IP Addr
                                              device.user, # User name
                                              device.pw, # Password
                                              out_dir,
                                              False, # don't remove existing files
                                              checkLocal=True )

            try:
              rx.DownloadSystemErrlog(device.ip,device.user,device.pw,
                                      out_dir+'/SysLog.bin')
              rx.CloneGNSSConfig(device.ip,device.user,device.pw,'ALM.xml')
              rx.DownloadClone(device.ip,device.user,device.pw,out_dir+'/ALM.xml')
            except:
              # this info is not critical.  Ignore if error...
              loginfo.print_("Couldn't get ALM.xml/Syslog.bin for {}".format(device.ip))
              time.sleep(90)

            try_cnt = 0
          except:
            loginfo.print_("Final download error! Sleep 70s - time for reboot...")
            try_cnt -= 1
            if try_cnt == 0:
              raise RuntimeError("Cannot download files at end of test")
            time.sleep(90) # give time for crash recovery

      cfg.current_config_device = None


def startup_receivers( proc_q, loginfo, cfg, info, proc_list, selected_devices_list, devices_filter_type_dict, skip_reboot, run_para=None, selected_devices_bbsampling_mode_afterIM={}):
  '''Inputs: proc_q = Queue() - list of background processes started
             loginfo = LogInfo() - where to log diagnostics
             cfg = ConfigXML = info from config.xml
             info = ScenarioInfo = info from current scenario XML
             proc_list = list of processes started by this function
             selected_devices_list = list of integer numbers defined by <n>?</n> in config.xml file
             devices_filter_type_dict = dictionary int rx.n -> string anti-jam filter type
             skip_reboot = True/False
             selected_devices_bbsampling_mode_afterIM =
  Start receivers for testing:
   - configure receivers
   - start logging
  '''
  total_devices = copy.copy(cfg.devices)
  do_devices = [x for x in total_devices if x.n in selected_devices_list]
  if verbose: loginfo.print_("startup_receivers::do_devices =", do_devices)      

  for n_attempt in range(2):
    
    def wrap_run_setup( device ):
      try:
        run_setup( loginfo, device, info, cfg, proc_list, skip_reboot, devices_filter_type_dict, run_para)
        return True
      except:
        if verbose: loginfo.print_("run_setup() failed. "+device.ip)
        return False

    if run_setup_in_parallel:
      loginfo.print_("Run setup in parallel.")
      with multiprocessing.pool.ThreadPool(len(do_devices)) as p:
        results = p.map( wrap_run_setup, do_devices )
    else:
      loginfo.print_("Run setup in serial.")
      results = []
      for dev in do_devices:
        rt = wrap_run_setup(dev)
        results.append( rt )

    if skip_reboot:
      loginfo.print_("Skip sleep 90s.")  
    else:
      loginfo.print_("Sleep 90s.")
      time.sleep(90) # give time for crash recovery

    tmp = []
    for i in range(len(do_devices)):
      if verbose: loginfo.print_("#",do_devices[i].n,"run_setup() returns", results[i])
      if results[i] == False:
        tmp.append(do_devices[i])
    do_devices = tmp

    if len(do_devices) == 0:
      break
    loginfo.print_("Need to retry run_setup for {}".format(do_devices))
    loginfo.print_("Sleep 15 sec before retry.")
    time.sleep(15)
  

  # Anything left undone is bad:
  for device in do_devices:
    remove_one_bad_device( loginfo, cfg, device, "run_setup error" )

  total_devices = copy.copy(cfg.devices)
  do_devices = [x for x in total_devices if x.n in selected_devices_list]

  enableLogging = False
  if 'standpoint' in run_para:
      loginfo.print_("standpoint exists so enable logging at startup")
      enableLogging = True

  for n_attempt in range(2):
    with multiprocessing.pool.ThreadPool(len(do_devices)) as p:
      def wrap_start_logging( device ):
        try:
          return start_logging( proc_q, loginfo, device, info, reset_clean=True, enable_logging=enableLogging, set_bbsampling_mode=True, selected_devices_bbsampling_mode_afterIM=selected_devices_bbsampling_mode_afterIM, devices_filter_type_dict=devices_filter_type_dict)
        except:
          return 'error'
      results = p.map( wrap_start_logging, do_devices )

    # Get devices that failed:
    bad_devices = []
    for dev,proc in zip(do_devices,results):
      if proc == 'error':
        # Get devices that failed:
        bad_devices.append( dev )
      else:
        # Update proc_list with list of processes started
        proc_list.extend( proc )
    do_devices = bad_devices  # retry bad devices

    if len(do_devices) == 0:
      break
    loginfo.print_("Sleep 90s.  Need to retry start_logging for {}".format(do_devices))
    time.sleep(90) # time for crash recovery

  # Anything left undone is bad:
  for device in do_devices:
    remove_one_bad_device( loginfo, cfg, device, "start_logging error" )

  if len(cfg.devices) == 0:
    raise RuntimeError("startup_receivers() removed everything!")

  if verbose: loginfo.print_("startup_receivers. Done.")
  # Total time 20 + 70 + 15 + 70 + 20

def start_logging( proc_q, loginfo, device, info, reset_clean=True, enable_logging=True, set_bbsampling_mode=False, selected_devices_bbsampling_mode_afterIM={}, devices_filter_type_dict={}):
  '''Inputs: proc_q = Queue() - list of background processes started
             loginfo = ST.SpirentTools.LogInfo = where to log diagnostics
             device = Device object - info on current device
             info = ScenarioInfo object - info on current scenario
             reset_clean = True/False - do clean and reset 
             enable_logging = True/False - do EnableDefaultLogging() or not
             set_bbsampling_mode = True/False
             selected_devices_bbsampling_mode_afterIM = {}, int rx.n -> True/False
             devices_filter_type_dict = dictionary int rx.n -> string anti-jam filter type
  Start logging activities:
    - make sure no T04 files are present
    - set GPS ref week
    - playback any CMR/RTCM data
    - clear error log
    - start logging
  '''

  IPAddr = device.ip
  user = device.user
  password = device.pw
  user_pass = '%s:%s' % (user,password)
  proc_list = []
  loginfo.print_(fn()+"start_logging for %s" % IPAddr)

  if device.septentrio is not None:
    loginfo.print_(fn()+"This is Septentrio receiver %s" % IPAddr)
    Sept = sept_rx.Septentrio_Socket(device.ip, int(device.port))
    if reset_clean:
      # Disable logging
      Sept.disable_logging()

      # Remove log files
      flist = Sept.list_ftp_files()
      loginfo.print_(fn()+"Clean old log files",flist,IPAddr)
      for fname in flist:
        Sept.remove_file(fname)
    if enable_logging:
      # Enable logging
      rt = Sept.enable_logging()

      loginfo.print_(fn()+"enable logging: cmd sent", rt.split('\n')[0])
      loginfo.print_(fn()+"enable logging: return", rt.split('\n')[1])

      Sept.configure_output()
      filename = "log_rx"+str(device.n)+".sbf"
      loginfo.print_(fn()+"Set log file name",filename,IPAddr)
      Sept.set_logging_filename( filename.split('.')[0] )
    
    if set_bbsampling_mode and int(device.n) in selected_devices_bbsampling_mode_afterIM:
      # Set BB Sampling mode
      loginfo.print_(fn()+"selected_devices_bbsampling_mode_afterIM[",device.n,"] =", selected_devices_bbsampling_mode_afterIM[int(device.n)])
      rt = None
      if selected_devices_bbsampling_mode_afterIM[int(device.n)]: #int(device.septentrio.afterIM) == 1:
        loginfo.print_("run_setup set BBSampling Mode AfterIM",device.ip)
        rt = Sept.set_BBSampling_mode('AfterIM')
      else:
        loginfo.print_("run_setup set BBSampling Mode BeforeIM",device.ip)
        rt = Sept.set_BBSampling_mode('BeforeIM')
      time.sleep(1)
      loginfo.print_("run_setup set BBSampling Mode cmd sent",device.ip,":", rt.split('\n')[0])
      loginfo.print_("run_setup set BBSampling Mode returns",device.ip,":", rt.split('\n')[1])

    del Sept
  elif device.novatel is not None:
    loginfo.print_(fn()+"This is Novatel receiver %s" % IPAddr)
    loginfo.print_(fn()+"Logging has been enabled in run_setup().")
  else:
    loginfo.print_(fn()+"This is Trimble receiver %s" % IPAddr)

    # Enable anti jamming filter
    loginfo.print_(fn()+"devices_filter_type_dict:", devices_filter_type_dict)
    loginfo.print_(fn()+"device.n:", device.n)

    if device.n in devices_filter_type_dict:
      filter_dict = {"FIR":0,"FFT":0}
      filter_type_str = devices_filter_type_dict[device.n]
      if verbose: 
        loginfo.print_("run_setup "+'filter_type_str =', filter_type_str)
      if filter_type_str == 'None':
        filter_dict = {"FIR":0,"FFT":0}
      elif filter_type_str == 'FIR':
        filter_dict['FIR'] = 1
      elif filter_type_str == 'FFT':
        filter_dict['FFT'] = 1
      elif filter_type_str == 'FIR + FFT': # FIXME or filter_type_str == 'FIR   FFT':
        filter_dict['FIR'] = 1
        filter_dict['FFT'] = 1
      else:
        loginfo.print_("run_setup "+ "Wrong input filter type:" + filter_type_str)
    
      loginfo.print_('run_setup-set anti jamming filter on device #'+str(device.n)+'. filter_type_str =',filter_type_str,' -> filter_dict =',str(filter_dict))
      try:
        for filter_type, on_off in filter_dict.items():
          rx.SetAntiJamFilter(device.ip,device.user,device.pw, rf_band='L1', filter_type=filter_type, enable=on_off)
        loginfo.print_('run_setup check anti jamming filters setting.')
        d_anti_jam_filters = rx.GetAntiJamFilter(device.ip, device.user, device.pw, rf_band='L1')
        loginfo.print_('run_setup anti jamming filters setting =',d_anti_jam_filters)
      except:
        loginfo.print_('run_setup Setting anti jamming filters failed. Skip.')

    if reset_clean:
      rx.DisableDefaultLogging(device.ip,device.user,device.pw)
      loginfo.print_(fn()+"DisableDefaultLogging for %s" % IPAddr)
      rx.getFileListAndDelete(device.ip,device.user,device.pw,'/Internal/','T04')
      loginfo.print_(fn()+"getFileListAndDelete for %s" % IPAddr)

      # Clear the error/warning log
      rx.ClearErrorLog(IPAddr,user,password)
      loginfo.print_(fn()+"ClearErrorLog for %s" % IPAddr)

    if enable_logging:
      # Enable logging
      rx.EnableDefaultLogging(IPAddr,user,password)
      loginfo.print_(fn()+"EnableDefaultLogging for %s" % IPAddr)

  loginfo.print_(fn()+"Start logging finished.")

  return proc_list


def run_single_test(loginfo, proc_q, cfg, info, fw_time, custom_fw, n_loops, selected_devices_list, devices_filter_type_dict, run_para, skip_reboot, skip_post, selected_devices_bbsampling_mode_afterIM):
  '''Inputs: loginfo = LogInfo() - where to log diagnostics
             proc_q = Queue() - list of background processes started
             cfg = ConfigXML = info from config.xml
             info = ScenarioInfo = info from current scenario XML
             fw_time = text string of when firmware was built
             custom_fw = user has custom firmware?
             n_loops = # of times to run?
             selected_devices_list = list of integer numbers defined by <n>?</n> in config.xml file
             devices_filter_type_dict = dictionary int rx.n -> string anti-jam filter type
             run_para = string or run parameter in json format
             skip_reboot = True/False
             skip_post = True/False
             selected_devices_bbsampling_mode_afterIM = 
  Run spirent simulator:
   - starts playing AugerTesting.scn
   - at end, downloads T0x files and writes out cfg.ResultsQueue summary file
  '''

  loginfo.info['NumReceivers'] = str(len(selected_devices_list))+'/'+str(len(cfg.devices))
  loginfo.info['TestFilename'] = info.fileName
  loginfo.info['Notes'] = run_para['notes'] if 'notes' in run_para else 'N/A'
  if verbose: loginfo.print_(fn(), "run parameters =", run_para)
  
  if 'skip_post_processing' in run_para and run_para['skip_post_processing'] == True:
    if verbose: loginfo.print_(fn(), "skip_post_processing is defined in run parameter. Overwrite skip_post = True.")
    skip_post = True

  scn_start_time = "None"
  if use_simulator:
    loginfo.print_("Read Scenario info.")
    try:
      if 'spirent_scn' in run_para:
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getScenario/',{'scenario_file':run_para['spirent_scn']},user['username'],user['password'])
      else:
        r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getScenario/', None, None, None)
      data = r.json()
      #loginfo.print_("json data: ", data)
      scn_start_time = data['simgen_scenario']['start_time']
      if verbose: loginfo.print_("scn_start_time =", str(scn_start_time))
    except:
      raise RuntimeError('Failed to obtain start_time from scenario .scn file')
  else:
    # Begin live sky workaround
    loginfo.print_("Simulator is not defined in config file.")
    try:
      with open("AugerTesting.scn") as f:
        data = f.read()
        d = xmltodict.parse(data)
        scn_start_time = d['simgen_scenario']['start_time']
        if verbose: loginfo.print_("scn_start_time =", str(scn_start_time))
    except:
      raise RuntimeError('Failed to obtain start_time from scenario .scn file')
    # End live sky workaround
  
  # Reset Simulator
  # Reset receivers without turn off simulator will not reset GPS time in receiver
  loginfo.print_("Reset Spirent Simulator.")
  reset_simulator(loginfo, cfg)

  # Run this set of samples multiple times
  n_errors = 0
  fileRunNum = 0

  while True:

    if fileRunNum >= n_loops:
      break
    loginfo.info['runs'] = '%d / %d' % (fileRunNum+1, n_loops)
    loginfo.print_("run num = %d / %d" % (fileRunNum+1, n_loops))
    start_time_YmdHMS = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

    # While we're waiting we might as well do some useful work in the background:
    # - process any old results
    check_runq(loginfo, proc_q, "/opt/anaconda/bin/python ProcessResults.py",
               exit_on_err=False,
               run_in_bg=True)

    # Clear any output directory
    for device in cfg.devices:
      out_dir = "DataDir/RX%d-%d"% (device.n,cfg.GlobalRunNum)
      loginfo.print_("os.path.exists(",out_dir,"):",os.path.exists(out_dir))
      if os.path.exists(out_dir):
        loginfo.print_("Clear dir %s for: %s"%(out_dir,device.ip))
        shutil.rmtree( out_dir )

    # Samples are available now set up the receiver by clearing from the last run
    loginfo.print_("Clear, reset and configure receivers...")
    proc_list = []
    startup_receivers( proc_q, loginfo, cfg, info, proc_list, selected_devices_list, devices_filter_type_dict, skip_reboot, run_para, selected_devices_bbsampling_mode_afterIM)

    # #####################################################
    proc,try_again,run_freq_type,run_steps = run_spirent_and_download_files( loginfo, proc_q, cfg, info, selected_devices_list, run_para, devices_filter_type_dict)
    proc_list.append(proc)
    if verbose: loginfo.print_("run_spirent_and_download_files(). Done.")

    for proc in proc_list:
      if proc is not None:
        loginfo.print_("Stop PID {}".format(proc.pid))
        proc.terminate()
        proc.join()

    if try_again:
      n_errors += 1
      if n_errors >= 2:
        raise RuntimeError("Too many errors.  Giving up")
      loginfo.print_("Some problem with Spirent.  Try one more time...")
      continue

    # The test has completed
    #  - Download and archive the data
    #  - Write an XML file with details
    #  - kick off in a seperate process analysis

    # If a receiver is crashed at the end of a test, we may not be able
    # to download files.  Try once, but if there is a problem then
    # remove any bad devices and move on.
    loginfo.print_("Download files at end of test...")
    try:
      download_files_at_end_of_test( loginfo, proc_q, cfg, selected_devices_list)
    except:
      remove_bad_devices( loginfo, cfg )
      download_files_at_end_of_test( loginfo, proc_q, cfg, selected_devices_list )

    # Turn off simulator after turnning off logging on receivers based on request from David
    loginfo.print_("Turn off simulator after turnning off all receivers.")
    reset_simulator(loginfo, cfg)

    for device in cfg.devices:
      if device.n in selected_devices_list:
        out_dir = "DataDir/RX%d-%d"% (device.n,cfg.GlobalRunNum)

        # Note if this is custom firmware
        if custom_fw:
          open(out_dir+"/custom_fw.txt", 'a').close()

        # save timg path if install_timg 
        if 'timg_full_path' in run_para:
          fid = open( out_dir + '/install_timg.txt','w')
          fid.write(run_para['timg_full_path'] + '\n')
          fid.close()

        # Requied by David during the test we swith Anti Jamming filter type by using different timg file
        # This could to be disable in the future
        checksum = None
        if device.septentrio is None and device.novatel is None:
          # get CheckSum from fw
          loginfo.print_(fn()+"Get CheckSum from T04.")
          cmd = 'viewdat -d12 ' + out_dir + '/' + './*.T04' 
          cmd_arr = cmd.split(' ')
          loginfo.print_("Run cmd:", cmd_arr)

          p1 = subprocess.Popen(cmd_arr, stdout=subprocess.PIPE)
          output = p1.communicate()[0].decode("utf-8") 
          output_arr = output.split('\n')
          for ele in output_arr:
            if "Firmware checksum" in ele:
              if verbose: loginfo.print_(ele)
              checksum = ele.replace(' ','').split(':')[1]
              open(out_dir+"/"+ele.replace(' ','').replace(':','_')+".txt", 'a').close()
              break

          try:
            # get firmware CRC
            t04 = out_dir + '/*.T04'
            for file in os.listdir(out_dir):
              if file.endswith(".T04"):
                t04 = os.path.join(out_dir, file)
                break
            loginfo.print_(fn()+"Read FW CRC from T04 ", t04)
            fw_crc = mutils.get_kv_info(t04, 'AdlerCRC')
            loginfo.print_(fn()+"Get FW CRC from T04 ",fw_crc.decode("utf-8") )
            open(out_dir+"/"+"Firmare_CRC_"+fw_crc.decode("utf-8")+".txt", 'a').close()
          except:
            open(out_dir+"/"+"Firmare_CRC_unknown.txt", 'a').close()
            
        elif device.septentrio is not None:
          sbf2t0x = '../../../sbf2t0x'
          if os.path.isfile("../../t01/build64/sbf2t0x/sbf2t0x"):
              sbf2t0x = "../../t01/build64/sbf2t0x/sbf2t0x"

          data_dir = out_dir
          # rx_name = re.match('RX([0-9]+)',os.path.basename(data_dir))[0]
          rx_name = "RX"+str(device.n)
          loginfo.print_(fn()+"This is septentrio data.", rx_name)
          # tmp_all_T04 = data_dir + '/rcvr.T04'
          # Run sbf2t0x
          cmd = "%s %s/log_rx%s.sbf %s/rcvr.T04" % (sbf2t0x,
                                        os.getcwd() + '/' + data_dir,
                                        rx_name.replace("RX",""),
                                        os.getcwd() + '/' + data_dir)
          try:
            loginfo.print_(fn()+" Run cmd: "+cmd)
            subprocess.check_call(cmd, shell=True)
          except:
            loginfo.print_(fn()+" Run cmd: "+cmd+ " failed.")

        # Save jammer time history for postprocessing
        f = open( out_dir + '/steps.txt','w')
        for step in run_steps:
          f.write(str(step)+'\n')
        f.close()

        end_time = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
        # Now write a results file that point to the results and truth
        # information
        if(os.path.isdir(cfg.ResultsQueue) == False):
          os.makedirs( cfg.ResultsQueue ) # Make sure the directory exists
        fid = open(cfg.ResultsQueue + '/RX' + str(device.n) + '-' + str(cfg.GlobalRunNum) + '.xml','w')
        fid.write("<data>")
        fid.write("<IP>" + device.ip + "</IP>")
        fid.write("<desc>" + device.desc + "</desc>")
        fid.write("<ScriptDir>%s</ScriptDir>" % cfg.ScriptRoot)
        fid.write("<SampleConfig>" + info.fileName + "</SampleConfig>")
        fid.write("<Data>DataDir/RX" + str(device.n) + "-" + str(cfg.GlobalRunNum) + "</Data>")
        fid.write("<BuildTime>" + fw_time + "</BuildTime>")
        fid.write("<StartTime>" + start_time_YmdHMS + "</StartTime>")
        fid.write("<EndTime>" + end_time + "</EndTime>")
        fid.write("<ScnStartTime>" + str(scn_start_time) + "</ScnStartTime>")
        if device.settingsClone:
          fid.write("<settingsClone>" + device.settingsClone + "</settingsClone>")
        if skip_reboot:
          fid.write("<skip_install_clone_reboot>" + str(1) + "</skip_install_clone_reboot>")
        if 'timg_full_path' in run_para:
          fid.write("<install_local_timg>" + str(1) + "</install_local_timg>")
          fid.write("<timg_path>" + str(run_para['timg_full_path']) + "</timg_path>")
        if device.patchFile:
          fid.write("<patchFile>" + device.patchFile + "</patchFile>")
        if device.tag:
          fid.write("<tag>" + device.tag + "</tag>")
        if device.frozen:
          fid.write("<frozen>" + device.frozen + "</frozen>")
        if device.do_RTK:
          fid.write("<do_RTK>" + device.do_RTK + "</do_RTK>")
        if device.septentrio != None:
          fid.write("<septentrio>" + '1' + '</septentrio>')
          # Log BBSampling mode to xml
          Sept = sept_rx.Septentrio_Socket(device.ip, int(device.port))
          rt = Sept.get_BBSampling_mode()
          rt_msg = rt.split('\n')[1]
          loginfo.print_("Get BBSampling Mode returns",device.ip,":", rt_msg)
          if 'AfterIM' in rt_msg:
            fid.write("<AfterIM>" + '1' + '</AfterIM>')
          elif 'BeforeIM' in rt_msg:
            fid.write("<AfterIM>" + '0' + '</AfterIM>')
          else:
            loginfo.print_("Unknown BBSampling Mode returns",device.ip,".")
            fid.write("<AfterIM>" + '-1' + '</AfterIM>')
          del Sept
        elif device.novatel != None:
          fid.write("<novatel>" + '1' + '</novatel>')
        else: #Trimble GNSS receiver
          fid.write("<trimble>" + '1' + '</trimble>')
          d_anti_jam_filters = rx.GetAntiJamFilter(device.ip, device.user, device.pw, rf_band='L1')
          if "FFT" in d_anti_jam_filters:
            fid.write("<FFT_filter>" + str(d_anti_jam_filters['FFT']) + '</FFT_filter>')
          if "FIR" in d_anti_jam_filters:
            fid.write("<FIR_filter>" + str(d_anti_jam_filters['FIR']) + '</FIR_filter>')
        if skip_post == True:
          fid.write("<skipPost>" + str(1) + "</skipPost>")
        current_filter_type = devices_filter_type_dict[device.n]
        fid.write("<filter_type>" + str(current_filter_type) + "</filter_type>")
        fid.write("<checksum>" + str(checksum) + '</checksum>')
        fid.write("<notes>" + run_para['notes'] + "</notes>")
        fid.write("<freq_type>" + run_freq_type + "</freq_type>")
        fid.write("<run_para>" + json.dumps(run_para) + "</run_para>")
        
        # ToDo - any other meta data we need for the analysis?
        fid.write("</data>")
        fid.close()

        # Make a copy of regression history
        src = './regression_diag.txt'
        shutil.copy(src, out_dir)

    loginfo.print_("Current Run #"+str(cfg.GlobalRunNum)+" finished:")
    cfg.GlobalRunNum = cfg.GlobalRunNum + 1
    fid = open('RunNum.txt','w')
    fid.write(str(cfg.GlobalRunNum))
    fid.close()
    loginfo.info['GlobalRunNum'] = cfg.GlobalRunNum
    fileRunNum += 1


def Run(loginfo, proc_q, custom_fw, n_loops, run_list=[], skip_reboot=False, skip_post=False):
  """Input: proc_q = Queue() - list of background processes started
            custom_fw = if True, don't do a fresh CVS checkout and don't install firmware
            n_loops = # of times to run the scenarios
            run_list = list of run parameters dictionary from json
            skip_reboot = True/False to skip reboot
            skip_post = True/False to skip postprocessing
  Run spirent simulator with selected jamming source on selected receivers
  """

  # 1) Get the test system config file
  cfg = get_config_xml()

  loginfo.print_("Use simulator =", use_simulator)

  jamdev_name_list = ['HP8648C_GPIB_18', 'HP8648C_GPIB_16', 'Chirp_1', 'HackRF_1']
  for jamdev_name in jamdev_name_list:
    loginfo.print_("Use", jamdev_name, any([(jamdev.id==jamdev_name and jamdev.enable==1) for jamdev in cfg.jamming_devices]))

  loginfo.print_("Run list length =", len(run_list))
  loginfo.print_("Run list:", run_list)
  loginfo.info['NumReceivers'] = '?/'+str(len(cfg.devices)) # Each run can have different selected receivers. This will be updated later
  loginfo.info['GlobalRunNum'] = cfg.GlobalRunNum
  loginfo.info['TestFilename'] = ''
  
  runScenario = ''
  for run_i in run_list:
    if 'spirent_scn' in run_i:
        runScenario = run_i['spirent_scn']
    jammer_list = run_i['jammer_list']
    device_id = 'chirp_1'
    if device_id in jammer_list and 'freq_type' not in jammer_list[device_id]:
      jammer_list[device_id]['freq_type'] = 'Constant'
      loginfo.print_(fn()+"Add default frequency type(Constant) in run list for chirp to be backward compatible.")

  # 2) Estimate Time
  t_total = 0

  need_build, _ = is_build_needed(loginfo)
  if need_build and not custom_fw:
    t_total += 5*60

  for run_i in run_list:
    
    def estimate_run_time_sec(run):

      dt_play = 0
      dt_install = 0
      n_rvcr = 0

      for k,v in run['receiver_list'].items():
        n_rvcr += 1
        if 'install_timg' in v and v['install_timg'] == True:
          dt_install = 150 # timg is installed in parallel for each run

      jammer_list = run['jammer_list']

      # frequency_type == 'Constant' or jammer is Chirp
      for device_id in ['gpib18', 'gpib16','hackrf_1','chirp_1']:
        if device_id in jammer_list and jammer_list[device_id]['freq_type'] == 'Constant':
          quiescence_period = float(jammer_list[device_id]['quiescence']) # eg. 300 [sec]
          dwell_time = 0
          try:
            dwell_time = float(jammer_list[device_id]['dwell_time']) # eg. 100 [sec]
          except:
            dwell_time = 0
          dt_play_tmp = quiescence_period + dwell_time + quiescence_period  #eg. 300 + 100 + 300
          dt_play = max(dt_play, dt_play_tmp)

      # frequency_type == 'Stepped'
      for device_id in ['gpib18', 'gpib16', 'hackrf_1']:
        if device_id in jammer_list and jammer_list[device_id]['freq_type'] == 'Stepped':
          quiescence_period = float(jammer_list[device_id]['quiescence']) # 300 [sec]
          dwell_time = float(jammer_list[device_id]['dwell_time']) # 30 [sec]
          recovery_time = float(jammer_list[device_id]['recovery_time']) # 15 [sec]
          start_freq = float(jammer_list[device_id]['start_freq'])
          stop_freq = float(jammer_list[device_id]['stop_freq'])
          freq_step = float(jammer_list[device_id]['freq_step'])
          d_freq = stop_freq - start_freq
          n_freq = int(d_freq/freq_step)
          dt_play_tmp = quiescence_period + n_freq*(dwell_time+recovery_time) + quiescence_period + n_freq*(dwell_time+recovery_time) + quiescence_period
          dt_play = max(dt_play, dt_play_tmp)

      # frequency_type == 'Customized'
      for device_id in ['gpib18', 'gpib16','hackrf_1','chirp_1']:
        if device_id in jammer_list and jammer_list[device_id]['freq_type'] == 'Customized':
          schedule = jammer_list[device_id]['schedule']
          dt_play_tmp = None
          try:
            dt_play_tmp = float(schedule[-1]['t']) + float(schedule[-1]['dwell_time'])
          except:
            raise RuntimeError("No time stamp or dwell_time in schedule")
          dt_play = max(dt_play, dt_play_tmp)

      t_start_offset = int(run['start_offset_sec']) if 'start_offset_sec' in run else 60
      t_end_additional_quiescence = int(run['additional_quiescence_sec']) if 'additional_quiescence_sec' in run else 0
      
      setup_time = (20 + 70 + 15 + 70) + 20 #sec
      dt_setup = setup_time if run_setup_in_parallel else n_rvcr*setup_time
      
      return dt_install + dt_setup + t_start_offset + dt_play + t_end_additional_quiescence

    t_total += estimate_run_time_sec(run_i)

  end_time_str = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(time.time()+t_total))
  loginfo.info['ExpectedEndTime'] = end_time_str
  loginfo.print_("Expected end time: %s" % end_time_str)

  # 3) Check install
  #####################################################
  # Build the firmware - we do this before each
  # time we run through all the samples. That way we
  # get the latest firmware
  # need to call .join() before config receivers
  #####################################################

  # proc = None
  if cfg.allow_builds and not custom_fw:
    loginfo.print_(fn()+"Run build_install_firmware().")
    selected_devices_list = [rx.n for rx in cfg.devices]
    build_install_firmware(loginfo, proc_q, cfg, selected_devices_list)

  _, fw_time = is_build_needed(loginfo)

  # 4) launch each test
  for run in run_list:

    # Build dictionary of selected receivers with if install timg, AJ filter type, BBSampling mode(Septentrio only)
    # FIXME This could be simplified by rewrite build_install_firmware() and run_single_test()
    selected_devices_list = []
    selected_devices_list_and_install_customized_timg = []
    devices_filter_type_dict = {} #{rx.n->FFT, ...}
    selected_devices_bbsampling_mode_afterIM = {} #{rx.n->True/False, ...}

    for k,v in run['receiver_list'].items():
      selected_devices_list.append(int(k))
      if 'install_timg' in v and v['install_timg'] == True:
        selected_devices_list_and_install_customized_timg.append(int(k))
        for i in range(len(cfg.devices)):
          if cfg.devices[i].n == int(k):
            loginfo.print_(fn()+"Overwrite devices #"+str(cfg.devices[i].n)+" .timg", v['timg_path'])
            cfg.devices[i].timg = v['timg_path']
      devices_filter_type_dict[int(k)] = v['filter_type']
      if 'bbsample_mode_afterIM' in v:
        selected_devices_bbsampling_mode_afterIM[int(k)] = v['bbsample_mode_afterIM']

    loginfo.print_(fn()+"Install customized timg for RX-"+str(selected_devices_list_and_install_customized_timg)+".")
    if len(selected_devices_list_and_install_customized_timg) > 0:
      build_install_firmware(loginfo, proc_q, cfg, selected_devices_list_and_install_customized_timg, use_customized_timg_and_skip_build=True)

    loginfo.print_("New selected_devices_list", str(selected_devices_list))
    loginfo.print_("New selected_devices_list_and_install_customized_timg", str(selected_devices_list_and_install_customized_timg))
    loginfo.print_("New devices_filter_type_dict", str(devices_filter_type_dict))
    loginfo.print_("New devices_BBSample_mode_dict", str(selected_devices_bbsampling_mode_afterIM))

    # Run a test
    info = ScenarioInfo()
    loginfo.print_("run scenario: ", runScenario)
    if runScenario != '':
        info.fileName = runScenario
    else:
        info.fileName = 'AugerTesting.scn'
    run_single_test(loginfo, proc_q, cfg, info, fw_time, custom_fw, n_loops, selected_devices_list, devices_filter_type_dict, run, skip_reboot, skip_post, selected_devices_bbsampling_mode_afterIM)


def WrapRun(proc_q, custom_fw, n_loops, msg1_run_list, skip_reboot=False, skip_post=False):
  """Input: proc_q = Queue() - list of background processes started
            custom_fw = if True, don't do a fresh CVS checkout and don't install firmware
            n_loops = # of times to run the scenarios
            msg1_run_list = list of run parameters dictionary from json
            skip_reboot = True/False to skip reboot
            skip_post = True/False to skip postprocessing
  WrapRun spirent simulator with selected jamming source on selected receivers
  """
  proc_q.put(multiprocessing.current_process().pid)
  got_error = False

  loginfo = LogInfo(True)
  try:
    loginfo.print_("Load run list.")
    run_list = [] if msg1_run_list=='' else json.loads(msg1_run_list)['run_list']

    loginfo.print_("Number of loops {}.  Skip FW {}. Skip Post-processing {}.".format(n_loops,custom_fw,skip_post))
    
    Run(loginfo, proc_q, custom_fw, n_loops, run_list, skip_reboot, skip_post)
    
    # ProcessResults.py runs in the background while the Anti-Jam 
    # system is running.  If ProcessResults.py takes too long, it
    # could still be running here.  Use the --wait option to make sure
    # we wait and process the latest results here. Use the --force
    # option to avoid skipping post-processing when OutputResults is 
    # newer than ResultsQueue
    loginfo.print_("Run postprocessing.")
    # FIXME
    check_runq(loginfo, proc_q, "/opt/anaconda/bin/python ProcessResults.py --force --wait")
    loginfo.print_("Finished regression run.")
  except:
    got_error = True
    print("Unexpected error: {}".format(sys.exc_info()[0]))
    loginfo.print_("Unexpected error: {}".format(sys.exc_info()[0]))
    loginfo.print_(sys.exc_info()[1])
    print(sys.exc_info()[1])
    traceback.print_tb( sys.exc_info()[2], file=loginfo.diag_f )
    traceback.print_tb( sys.exc_info()[2] )
  finally:
    loginfo.close()
    if got_error:
      email_error_result()
      set_regression_status(proc_q,'Idle (Error)')
    else:
      set_regression_status(proc_q,'Idle')

def get_freq(t):
  ''' Input t = string, eg. GPS L1 + 5HMz
      Get frequency from string 
  '''
  arr = t.split('MHz')
  freq1,freq2 = None, None
  for x in arr:
    if len(x) > len('GPS L') and x[:len('GPS L')] == 'GPS L':
      f = 0
      if x.rfind('+') != -1:
        res = x.split('+')
        f = 1176.45 if res[0] == 'GPS L5' else 1575.42
        try:
          f = f + float(res[1])
        except: # eg. GPS L1 + varying
          f = -1
      elif x.rfind('-') != -1:
        res = x.split('-')
        f = 1176.45 if res[0] == 'GPS L5' else 1575.42
        f = f - float(res[1])
      else:
        f = 1176.45 if x == 'GPS L5' else 1575.42
      if freq1 == None:
        freq1 = f
      elif freq2 == None:
        freq2 = f
  return freq1,freq2

def get_jamming_scenario(csv_filename="jamming_scenarios.csv", row_start=1,row_end=30,col_start=1,col_end=12):
  ''' Read csv_filename and extract scenario run list info
      Return dictionary info
  '''
  line_list = None
  with open(csv_filename, 'r') as csvfile:
    line = csv.reader(csvfile)
    line_list = list(line)

  rst = []
  for l in line_list[row_start:row_end]:
    sub_l = l[col_start:col_end]
    rst.append(sub_l)

  keys = []
  info = []
  info_dict = []
  for j in range(len(rst[0])):
    keys.append( str(rst[0][j]) )
  for j in range(len(rst[1])):
    if rst[1][j] is not '':
      keys[j] = keys[j] + '\n' + rst[1][j]
  for i in range(2,len(rst)):
    # as dict
    curr = {}
    curr['num'] = str(rst[i][0]).strip().replace('\n','')
    curr['desc'] = str(rst[i][1]).strip().replace('\n','')
    curr['freq'] = str(rst[i][2]).strip().replace('\n','')
    curr['pwr'] = str(rst[i][3]).strip().replace('\n','')
    curr['type'] = str(rst[i][5]).strip().replace('\n','')
    # get freq
    freq1,freq2 = get_freq(curr['type'])
    curr['freq1'] = freq1
    curr['freq2'] = freq2
    curr['wb_text'] = str(rst[i][7]).strip().replace('\n','')
    # curr['purpose'] = str(rst[i][9]).strip().replace('\n','')
    curr['run_hour'] = 0.5 # FIXME to be removed
    
    info_dict.append(curr)
    # as list
    curr = []
    for j in range(len(keys)):
      curr.append(rst[i][j])
    info.append(curr)

  all_info = {}
  all_info['keys'] = keys
  all_info['tests'] = info
  all_info['tests_dict'] = info_dict
  all_info['regression'] = get_regression_status()
  
  return all_info

def list_datadir(ResultsQueue):
  ''' Read run folder info from diskspace. This is abandoned after using db.
      Input: ResultsQueue = string, full path of ResultsQueue, eg. cfg.ResultsQueue
      Get list of folder name in DataDir/
      Return dict eg. {'folders':['RX3-0, RX3-1, ...']}
  '''
  data_dir = []
  fs = os.listdir('./DataDir/')
  for f in fs:
    if len(f)>3 and f[0:2] == 'RX':
      # print(fn(), f)
      data_dir.append(f)

  def f_sort(x):
    # Sort by run num then RX num
    rx,run = x.split('-')
    rx = int(rx[2:])
    run = int(run)
    return (run,rx)

  data_dir = sorted(data_dir, key=f_sort)
  data_dir = data_dir[::-1]

  all_runs = []

  for rx_run in data_dir:

    run_num = int(rx_run.split('-')[1])
    
    # Reading run_num =[0,190] from path/json
    if run_num >= 191:
      continue

    curr = {}
    curr['start_time'] = None
    curr['data_dir'] = rx_run
    curr['scn_id'] = None
    curr['freq_type'] = None
    curr['notes'] = None
    curr['desc'] = None
    curr['results_queue'] = None
    curr['output_results'] = None

    # result_xml_file = './ResultsQueue/' + rx_run + '.xml'
    result_xml_file = ResultsQueue + '/' + rx_run + '.xml'
    output_folder = './OutputResults/' + rx_run

    if os.path.isfile(result_xml_file):
      tree = ET.ElementTree(file=result_xml_file)
      curr['start_time'] = tree.find('./StartTime').text
      curr['freq_type'] = 'N/A' if tree.find('./freq_type') is None else tree.find('./freq_type').text
      curr['scn_id'] = 'N/A'
      if tree.find('./run_para') is not None:
        txt = txt = tree.find('./run_para').text
        try:
          d = json.loads(txt)
          if 'scenario' in d:
            curr['scn_id'] = d['scenario']
        except:
          print('Load json string failed.'+result_xml_file)

      curr['notes'] = tree.find('./notes').text if tree.find('./notes') is not None else None
      curr['desc'] = tree.find('./desc').text
      curr['results_queue'] = result_xml_file
    if os.path.isdir(output_folder): # Is this faster than check RX3-?.json exist?
      curr['output_results'] = output_folder
    
    all_runs.append(curr)

  all_info = {}
  all_info['all_runs'] = all_runs
  
  return all_info


def handle_req(proc_q, conn):
  """Input: proc_q = Queue() - list of background processes started
            conn = socket for communication with app.py
     Handle requests from app.py and send a response back over the 'conn' socket."""
  msg = conn.recv()
  if verbose: print("Got msg:", msg)
  if msg[0] == 'kill':
    set_regression_status(proc_q, 'Killed')
    cfg = get_config_xml()
    # Send request to stop CWs, spirent simulator
    # HackRF, chirp can be stopped once its proc in proc_q is killed by set_regression_status()
    if is_jammer_enabled('HP8648C_GPIB_18', cfg):
      r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':18, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])
    if is_jammer_enabled('HP8648C_GPIB_16', cfg): 
      r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOnCW', {'gpib_addr':16, 'pwr':-999, 'freq':1575.42 }, user['username'], user['password'])
    if is_jammer_enabled('Chirp_1', cfg):
      r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/turnOffChirp', {}, user['username'], user['password'])
    if is_jammer_enabled('HackRF_1', cfg):
      r = send_request(cfg.hackrf_host_ip, cfg.hackrf_host_port, '/api/v1/turnHackRFOff', {}, user['username'], user['password'])
    if use_simulator:
      reset_simulator(None, cfg)
  elif msg[0] == 'userInfo':
    with open("user_status.json","w") as f:
      json.dump( msg[1], f )
  elif msg[0] == 'getReceiverInfo':
    conn.send( get_receiver_info() )
  elif msg[0] == 'getJammingDeviceInfo':
    conn.send( get_jamming_device_info() )
  elif msg[0] == 'getUserInfo':
    with open("user_status.json") as f:
      conn.send(json.load( f ))
  elif msg[0] == 'getRegressionInfo':
    conn.send(get_regression_status())
  elif msg[0] == 'getRegressionProgress':
    with open("regression_progress.json") as f:
      content = json.load( f )
      conn.send(content)
  elif msg[0] == 'getRegressionDiag':
    with open("regression_diag.txt") as f:
      conn.send("".join(f.readlines()))
  elif msg[0] == 'getTestFileInfoAll':
    rst = get_jamming_scenario()
    conn.send(rst)
  elif msg[0] == 'getRegressionStopStatus':
    conn.send(get_regression_stop_status())
  elif msg[0] == 'listDataDir':
    # Not used anymore after using db instead of .json
    cfg = get_config_xml()
    rst = list_datadir(cfg.ResultsQueue)
    conn.send(rst)
  elif msg[0] == 'getConfigInfo':
    cfg = get_config_xml()
    # conn.send() will send empty dict if cfg.__dict__ is passed in due to it contains none str value.
    # Solution : dump dict to string, convert true, false, null into 'true', 'false', 'null', 
    #            then convert string back to dict, otherwise json.loads(str) is needed in app.py
    tmp = json.dumps(cfg.__dict__, default=vars)
    tmp = tmp.replace('false', '\"false\"').replace('true', '\"true\"').replace('null', '\"null\"')
    d = json.loads(tmp)
    conn.send( d )
  elif msg[0] == 'getSpirentSimulatorStatus':
    cfg = get_config_xml()
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getStatus/', None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getSpirentSimulationTOW':
    cfg = get_config_xml()
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getTOW/', None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getSpirentScenario':
    cfg = get_config_xml() #FIXME this might be resource consuming
    r = send_request(cfg.spirent_simulator_host_ip, cfg.spirent_simulator_host_port, '/api/v1/spirentTool/getScenario/', None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      start_time = data['simgen_scenario']['start_time']
      start_time_wn_tow = gpsTimeToWnTow(start_time)
      tow = start_time_wn_tow['tow']
      conn.send(data)
  elif msg[0] == 'getHP8648CStatus':
    if len(msg) != 2:
      conn.send('Missing 2nd parameter: gpib_address')
      return
    gpib_address = int(msg[1])  
    cfg = get_config_xml()
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/hp8648c/getStatus/'+str(gpib_address), None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getDAT64LStatus':
    if len(msg) != 2:
      conn.send('Missing 2nd parameter: com_port')
      return
    com_port = int(msg[1])
    cfg = get_config_xml()
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/dat64l/getStatus/'+str(com_port), None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getChirpStatus':
    cfg = get_config_xml()
    r = send_request(cfg.jamming_host_ip, cfg.jamming_host_port, '/api/v1/chirp/getStatus', None, None, None, timeout=3)
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getHackRFStatus':
    cfg = get_config_xml()
    print(cfg.hackrf_host_ip, cfg.hackrf_host_port)
    r = send_request(cfg.hackrf_host_ip, cfg.hackrf_host_port, '/api/v1/getHackRFStatus', None, None, None, timeout=3) # Send request to hackrf server
    if isinstance(r, str):
      conn.send("Http Request Failed")
    else:
      data = None
      try:
        data = r.json()
      except:
        data = json.dumps(r.text)
      conn.send(data)
  elif msg[0] == 'getRandomFrequencyList':
    if len(msg) < 5:
      conn.send(json.dumps('Not enough parameters'))
    random_seed = int(msg[1])
    start_freq_MHz = int(msg[2])
    stop_freq_MHz = int(msg[3])
    n_points = int(msg[4])
    if verbose: print(random_seed, start_freq_MHz*1e6, stop_freq_MHz*1e6, n_points)
    freq_list = RandomFrequency(random_seed, start_freq_MHz*1e6, stop_freq_MHz*1e6, n_points)
    conn.send({'freq_list': freq_list})
  elif msg[0] == 'stopAndWrapUpRun':
    stop_status = get_regression_stop_status()
    if stop_status['stop_status'] == 'Enabled':
      time.sleep(0.1) # Sleep a while to avoid editting at the same time
      set_regression_stop_status('Stop')
      conn.send({'msg': 'Stop requested.'})
    else:
      conn.send({'msg': 'Stop not enabled yet.'})
  elif msg[0] == 'getAntennaSwitchStatus':
    state_kv = {'0':'Simulator','1':'Sky'}
    HTTP_Result = urlopen('http://10.1.141.66/SP6TA:STATE?', timeout=3)
    PTE_Return = HTTP_Result.read()
    rt = {}
    rt['state'] = PTE_Return.decode("utf-8")
    rt['connect to'] = state_kv[rt['state']]
    if verbose: print('getAntennaSwitchStatus() returns', rt)
    conn.send(rt)
  elif msg[0] == 'switchAntenna':
    if msg[1] == '0':
      # Spirent simulator
      if verbose: print("Using Spirent simulator")
      r = requests.post('http://10.1.141.66/SETA=0', timeout=3, verify=False)
      r.raise_for_status()
      if verbose: print("Off:", r.text)
      return r.text
    else:
      # Live sky
      if verbose: print("Using Live sky")
      r = requests.post('http://10.1.141.66/SETA=1', timeout=3, verify=False)
      r.raise_for_status()
      if verbose: print("On:", r.text)
      return r.text
  elif msg[0] == 'getStandpointSwitchStatus':
    state_kv = {'0': 'Error','1':'Standpoint connected','2':'Standpoint disconnected'}
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        try:
          s.settimeout(5)
          s.connect(('10.1.141.118', 10001))
          query_command = "RFO?\r\n".encode('ascii')
          s.sendall(query_command)
          time.sleep(0.5)
          response_bytes = s.recv(1024)
          response = response_bytes.decode('ascii').strip()
        except Exception as e:
            response = '0'
    rt = {}
    rt['state'] = response
    rt['connect to'] = state_kv[rt['state']]
    if verbose: print('getStandpointSwitchStatus() returns', rt)
    conn.send(rt)
  
  elif msg[0] == 'switchStandpoint':
    command = ''
    if msg[1] == '0':
      command = 'RFO 2'
      if verbose: print("simulator only")
      with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        try:
          s.settimeout(5)
          s.connect(('10.1.141.118',10001))
          command_bytes = (command + '\r\n').encode('ascii')
          s.sendall(command_bytes)
          time.sleep(0.5)
        except Exception as e:
          command = 'Error'
    else:
      command = 'RFO 1'
      if verbose: print("standpoint connected")
      with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        try:
          s.settimeout(5)
          s.connect(('10.1.141.118',10001))
          command_bytes = (command + '\r\n').encode('ascii')
          s.sendall(command_bytes)
          time.sleep(0.5)
        except Exception as e:
          command = 'Error'
    return command

  elif msg[0] == 'checkFileExist':
    if len(msg) != 2:
      conn.send('Missing 2nd parameter: full_file_path')
      return
    full_file_path = msg[1]
    rt = os.path.exists(full_file_path)
    conn.send(rt)
  elif msg[0] == 'playList':
    if set_regression_status(proc_q, 'Running'):
      set_regression_stop_status('Unknown')

      rlist = msg[1]
      n_loops = int(msg[2])
      custom_fw = bool(msg[3])
      skip_reboot = bool(msg[4])
      skip_post = bool(msg[5])

      if verbose:
        print("--- Check input parameters ---") 
        print("rlist", rlist)
        print("n_loops", n_loops)
        print("custom_fw", custom_fw)
        print("skip_reboot", skip_reboot)
        print("skip_post", skip_post)
        print("--- Check input parameters done---") 

      proc = multiprocessing.Process(target=WrapRun, args=(proc_q, custom_fw, n_loops, rlist, skip_reboot, skip_post))
      proc.start()
  else:
    print("Unknown msg:",msg)
  conn.close()

def run_server():
  """
  Start listening on port ? for requests from app.py.  Each request is
  handled in a separate thread so multiple people can access the server.
  """
  config_XML = ET.parse("config.xml")
  control_port = int(config_XML.find('control_port').text)
  proc_q = multiprocessing.Queue()

  addr = ('localhost', control_port)
  print('Listen on {}'.format(addr))
  listener = mp_con.Listener(addr)
  while True:
    conn = listener.accept()
    proc = multiprocessing.Process(target=handle_req, args=(proc_q,conn))
    proc.start()

def init_json():
  '''Initialize all JSON status files at startup.'''
  if not os.path.isfile("user_status.json"):
    with open("user_status.json","w") as f:
      d = {'name':'', 'desc':'', 'end_time': 'Mon, 06 Jan 1980 00:00:00'}
      json.dump( d, f )
  with open("regression_progress.json","w") as f:
    d = {'detail':'Server restarted'}
    json.dump( d, f )
  with open("regression_diag.txt","w") as f:
    f.write('\n')
  set_regression_status(None,'Idle')
  set_regression_stop_status('Unknown') # This is used to stop and preserve the run.

def init_run_num():
  """Create RunNum.txt if it is not present"""
  if os.path.isfile("RunNum.txt"):
    return
  cfg = get_config_xml()
  files = glob.glob( cfg.ResultsQueue + "/*.xml")
  if len(files) == 0:
    with open('RunNum.txt','w') as fid:
      fid.write("1")
      return
  files.sort(key=os.path.getmtime)
  num = int(re.findall(r'RX\d+-(\d+).xml',files[-1])[0]) + 1
  with open('RunNum.txt','w') as fid:
    fid.write(str(num))


if __name__ == "__main__":
  os.environ['no_proxy'] = '*'
  init_json()
  init_run_num()
  run_server()