text: #####################################################################
  #                                                                   #
  # /experiment_queue.py                                                         #
  #                                                                   #
  # Copyright 2013, Monash University                                 #
  #                                                                   #
  # This file is part of the program BLACS, in the labscript suite    #
  # (see http://labscriptsuite.org), and is licensed under the        #
  # Simplified BSD License. See the license.txt file in the root of   #
  # the project for the full license.                                 #
  #                                                                   #
  #####################################################################
  from __future__ import division, unicode_literals, print_function, absolute_import
  from labscript_utils import PY2
  if PY2:
      str = unicode
      import Queue as queue
  else:
      import queue
  
  import logging
  import os
  import platform
  import threading
  import time
  import sys
  import shutil
  from collections import defaultdict
  from tempfile import gettempdir
  from binascii import hexlify
  
  from qtutils.qt.QtCore import *
  from qtutils.qt.QtGui import *
  from qtutils.qt.QtWidgets import *
  
  import zprocess
  from labscript_utils.ls_zprocess import ProcessTree
  process_tree = ProcessTree.instance()
  import labscript_utils.h5_lock, h5py
  
  from qtutils import *
  
  from labscript_utils.qtwidgets.elide_label import elide_label
  from labscript_utils.connections import ConnectionTable
  import labscript_utils.properties
  
  from blacs.tab_base_classes import MODE_MANUAL, MODE_TRANSITION_TO_BUFFERED, MODE_TRANSITION_TO_MANUAL, MODE_BUFFERED  
  import blacs.plugins as plugins
  
  
  def tempfilename(prefix='BLACS-temp-', suffix='.h5'):
      """Return a filepath appropriate for use as a temporary file"""
      random_hex = hexlify(os.urandom(16)).decode()
      return os.path.join(gettempdir(), prefix + random_hex + suffix)
  
  
  FILEPATH_COLUMN = 0
  
  class QueueTreeview(QTreeView):
      def __init__(self,*args,**kwargs):
          QTreeView.__init__(self,*args,**kwargs)
          self.header().setStretchLastSection(True)
          self.setAutoScroll(False)
          self.add_to_queue = None
          self.delete_selection = None
          self._logger = logging.getLogger('BLACS.QueueManager') 
  
      def keyPressEvent(self,event):
          if event.key() == Qt.Key_Delete:
              event.accept()
              if self.delete_selection:
                  self.delete_selection()
          QTreeView.keyPressEvent(self,event)
          
      def dragEnterEvent(self, event):
          if event.mimeData().hasUrls():
              event.accept()
          else:
              event.ignore()
  
      def dragMoveEvent(self, event):
          if event.mimeData().hasUrls():
              event.setDropAction(Qt.CopyAction)
              event.accept()
          else:
              event.ignore()
  
      def dropEvent(self, event):
          if event.mimeData().hasUrls():
              event.setDropAction(Qt.CopyAction)
              event.accept()
              
              for url in event.mimeData().urls():
                  path = str(url.toLocalFile())
                  if path.endswith('.h5') or path.endswith('.hdf5'):
                      self._logger.info('Acceptable file dropped. Path is %s'%path)
                      if self.add_to_queue:
                          self.add_to_queue(str(path))
                      else:
                          self._logger.info('Dropped file not added to queue because there is no access to the neccessary add_to_queue method')
                  else:
                      self._logger.info('Invalid file dropped. Path was %s'%path)
          else:
              event.ignore()
  
  class QueueManager(object):
      
      REPEAT_ALL = 0
      REPEAT_LAST = 1
  
      ICON_REPEAT = ':qtutils/fugue/arrow-repeat'
      ICON_REPEAT_LAST = ':qtutils/fugue/arrow-repeat-once'
  
      def __init__(self, BLACS, ui):
          self._ui = ui
          self.BLACS = BLACS
          self.last_opened_shots_folder = BLACS.exp_config.get('paths', 'experiment_shot_storage')
          self._manager_running = True
          self._manager_paused = False
          self._manager_repeat = False
          self._manager_repeat_mode = self.REPEAT_ALL
          self.master_pseudoclock = self.BLACS.connection_table.master_pseudoclock
          
          self._logger = logging.getLogger('BLACS.QueueManager')   
          
          # Create listview model
          self._model = QStandardItemModel()
          self._create_headers()
          self._ui.treeview.setModel(self._model)
          self._ui.treeview.add_to_queue = self.process_request
          self._ui.treeview.delete_selection = self._delete_selected_items
          
          # set up buttons
          self._ui.queue_pause_button.toggled.connect(self._toggle_pause)
          self._ui.queue_repeat_button.toggled.connect(self._toggle_repeat)
          self._ui.queue_delete_button.clicked.connect(self._delete_selected_items)
          self._ui.queue_clear_button.clicked.connect(self._toggle_clear)
          self._ui.actionAdd_to_queue.triggered.connect(self.on_add_shots_triggered)
          self._ui.queue_add_button.setDefaultAction(self._ui.actionAdd_to_queue)
          self._ui.queue_push_up.clicked.connect(self._move_up)
          self._ui.queue_push_down.clicked.connect(self._move_down)
          self._ui.queue_push_to_top.clicked.connect(self._move_top)
          self._ui.queue_push_to_bottom.clicked.connect(self._move_bottom)
  
          # Set the elision of the status labels:
          elide_label(self._ui.queue_status, self._ui.queue_status_verticalLayout, Qt.ElideRight)
          elide_label(self._ui.running_shot_name, self._ui.queue_status_verticalLayout, Qt.ElideLeft)
          
          # Set up repeat mode button menu:
          self.repeat_mode_menu = QMenu(self._ui)
  
          self.action_repeat_all = QAction(QIcon(self.ICON_REPEAT), 'Repeat all', self._ui)
          self.action_repeat_last = QAction(QIcon(self.ICON_REPEAT_LAST), 'Repeat last', self._ui)
  
          self.action_repeat_all.triggered.connect(lambda *args: setattr(self, 'manager_repeat_mode', self.REPEAT_ALL))
          self.action_repeat_last.triggered.connect(lambda *args: setattr(self, 'manager_repeat_mode', self.REPEAT_LAST))
  
          self.repeat_mode_menu.addAction(self.action_repeat_all)
          self.repeat_mode_menu.addAction(self.action_repeat_last)
  
          self._ui.repeat_mode_select_button.setMenu(self.repeat_mode_menu)
  
          # The button already has an arrow indicating a menu, don't draw another one:
          self._ui.repeat_mode_select_button.setStyleSheet("QToolButton::menu-indicator{width: 0;}")
  
          self.manager = threading.Thread(target = self.manage)
          self.manager.daemon=True
          self.manager.start()
  
      def _create_headers(self):
          self._model.setHorizontalHeaderItem(FILEPATH_COLUMN, QStandardItem('Filepath'))
          
      def get_save_data(self):
          # get list of files in the queue
          file_list = []
          for i in range(self._model.rowCount()):
              file_list.append(self._model.item(i).text())
          # get button states
          return {'manager_paused':self.manager_paused,
                  'manager_repeat':self.manager_repeat,
                  'manager_repeat_mode':self.manager_repeat_mode,
                  'files_queued':file_list,
                  'last_opened_shots_folder': self.last_opened_shots_folder
                 }
      
      def restore_save_data(self,data):
          if 'manager_paused' in data:
              self.manager_paused = data['manager_paused']
          if 'manager_repeat' in data:
              self.manager_repeat = data['manager_repeat']
          if 'manager_repeat_mode' in data:
              self.manager_repeat_mode = data['manager_repeat_mode']
          if 'files_queued' in data:
              file_list = list(data['files_queued'])
              self._model.clear()
              self._create_headers()
              for file in file_list:
                  self.process_request(str(file))
          if 'last_opened_shots_folder' in data:
              self.last_opened_shots_folder = data['last_opened_shots_folder']
          
      @property
      @inmain_decorator(True)
      def manager_running(self):
          return self._manager_running
          
      @manager_running.setter
      @inmain_decorator(True)
      def manager_running(self,value):
          value = bool(value)
          self._manager_running = value
          
      def _toggle_pause(self,checked):    
          self.manager_paused = checked
  
      def _toggle_clear(self):
          self._model.clear()
          self._create_headers()
  
      @property
      @inmain_decorator(True)
      def manager_paused(self):
          return self._manager_paused
      
      @manager_paused.setter
      @inmain_decorator(True)
      def manager_paused(self,value):
          value = bool(value)
          self._manager_paused = value
          if value != self._ui.queue_pause_button.isChecked():
              self._ui.queue_pause_button.setChecked(value)
      
      def _toggle_repeat(self,checked):    
          self.manager_repeat = checked
          
      @property
      @inmain_decorator(True)
      def manager_repeat(self):
          return self._manager_repeat
  
      @manager_repeat.setter
      @inmain_decorator(True)
      def manager_repeat(self,value):
          value = bool(value)
          self._manager_repeat = value
          if value != self._ui.queue_repeat_button.isChecked():
              self._ui.queue_repeat_button.setChecked(value)
  
      @property
      @inmain_decorator(True)
      def manager_repeat_mode(self):
          return self._manager_repeat_mode
  
      @manager_repeat_mode.setter
      @inmain_decorator(True)
      def manager_repeat_mode(self, value):
          assert value in [self.REPEAT_LAST, self.REPEAT_ALL]
          self._manager_repeat_mode = value
          button = self._ui.queue_repeat_button
          if value == self.REPEAT_ALL:
              button.setIcon(QIcon(self.ICON_REPEAT))
          elif value == self.REPEAT_LAST:
              button.setIcon(QIcon(self.ICON_REPEAT_LAST))
  
      def on_add_shots_triggered(self):
          shot_files = QFileDialog.getOpenFileNames(self._ui, 'Select shot files',
                                                    self.last_opened_shots_folder,
                                                    "HDF5 files (*.h5)")
          if isinstance(shot_files, tuple):
              shot_files, _ = shot_files
  
          if not shot_files:
              # User cancelled selection
              return
          # Convert to standard platform specific path, otherwise Qt likes forward slashes:
          shot_files = [os.path.abspath(str(shot_file)) for shot_file in shot_files]
  
          # Save the containing folder for use next time we open the dialog box:
          self.last_opened_shots_folder = os.path.dirname(shot_files[0])
          # Queue the files to be opened:
          for filepath in shot_files:
              if filepath.endswith('.h5'):
                  self.process_request(str(filepath))
  
      def _delete_selected_items(self):
          index_list = self._ui.treeview.selectedIndexes()
          while index_list:
              self._model.takeRow(index_list[0].row())
              index_list = self._ui.treeview.selectedIndexes()
      
      def _move_up(self):        
          # Get the selection model from the treeview
          selection_model = self._ui.treeview.selectionModel()    
          # Create a list of select row indices
          selected_row_list = [index.row() for index in sorted(selection_model.selectedRows())]
          # For each row selected
          for i,row in enumerate(selected_row_list):
              # only move the row if it is not element 0, and the row above it is not selected
              # (note that while a row above may have been initially selected, it should by now, be one row higher
              # since we start moving elements of the list upwards starting from the lowest index)
              if row > 0 and (row-1) not in selected_row_list:
                  # Remove the selected row
                  items = self._model.takeRow(row)
                  # Add the selected row into a position one above
                  self._model.insertRow(row-1,items)
                  # Since it is now a newly inserted row, select it again
                  selection_model.select(self._model.indexFromItem(items[0]),QItemSelectionModel.SelectCurrent)
                  # reupdate the list of selected indices to reflect this change
                  selected_row_list[i] -= 1
         
      def _move_down(self):
          # Get the selection model from the treeview
          selection_model = self._ui.treeview.selectionModel()    
          # Create a list of select row indices
          selected_row_list = [index.row() for index in reversed(sorted(selection_model.selectedRows()))]
          # For each row selected
          for i,row in enumerate(selected_row_list):
              # only move the row if it is not the last element, and the row above it is not selected
              # (note that while a row below may have been initially selected, it should by now, be one row lower
              # since we start moving elements of the list upwards starting from the highest index)
              if row < self._model.rowCount()-1 and (row+1) not in selected_row_list:
                  # Remove the selected row
                  items = self._model.takeRow(row)
                  # Add the selected row into a position one above
                  self._model.insertRow(row+1,items)
                  # Since it is now a newly inserted row, select it again
                  selection_model.select(self._model.indexFromItem(items[0]),QItemSelectionModel.SelectCurrent)
                  # reupdate the list of selected indices to reflect this change
                  selected_row_list[i] += 1
          
      def _move_top(self):
          # Get the selection model from the treeview
          selection_model = self._ui.treeview.selectionModel()    
          # Create a list of select row indices
          selected_row_list = [index.row() for index in sorted(selection_model.selectedRows())]
          # For each row selected
          for i,row in enumerate(selected_row_list):
              # only move the row while it is not element 0, and the row above it is not selected
              # (note that while a row above may have been initially selected, it should by now, be one row higher
              # since we start moving elements of the list upwards starting from the lowest index)
              while row > 0 and (row-1) not in selected_row_list:
                  # Remove the selected row
                  items = self._model.takeRow(row)
                  # Add the selected row into a position one above
                  self._model.insertRow(row-1,items)
                  # Since it is now a newly inserted row, select it again
                  selection_model.select(self._model.indexFromItem(items[0]),QItemSelectionModel.SelectCurrent)
                  # reupdate the list of selected indices to reflect this change
                  selected_row_list[i] -= 1
                  row -= 1
                
      def _move_bottom(self):
          selection_model = self._ui.treeview.selectionModel()    
          # Create a list of select row indices
          selected_row_list = [index.row() for index in reversed(sorted(selection_model.selectedRows()))]
          # For each row selected
          for i,row in enumerate(selected_row_list):
              # only move the row while it is not the last element, and the row above it is not selected
              # (note that while a row below may have been initially selected, it should by now, be one row lower
              # since we start moving elements of the list upwards starting from the highest index)
              while row < self._model.rowCount()-1 and (row+1) not in selected_row_list:
                  # Remove the selected row
                  items = self._model.takeRow(row)
                  # Add the selected row into a position one above
                  self._model.insertRow(row+1,items)
                  # Since it is now a newly inserted row, select it again
                  selection_model.select(self._model.indexFromItem(items[0]),QItemSelectionModel.SelectCurrent)
                  # reupdate the list of selected indices to reflect this change
                  selected_row_list[i] += 1
                  row += 1
      
      @inmain_decorator(True)
      def append(self, h5files):
          for file in h5files:
              item = QStandardItem(file)
              item.setToolTip(file)
              self._model.appendRow(item)
      
      @inmain_decorator(True)
      def prepend(self,h5file):
          if not self.is_in_queue(h5file):
              self._model.insertRow(0,QStandardItem(h5file))
      
      def process_request(self,h5_filepath):
          # check connection table
          try:
              new_conn = ConnectionTable(h5_filepath, logging_prefix='BLACS')
          except Exception:
              return "H5 file not accessible to Control PC\n"
          result,error = inmain(self.BLACS.connection_table.compare_to,new_conn)
          if result:
              # Has this run file been run already?
              with h5py.File(h5_filepath) as h5_file:
                  if 'data' in h5_file['/']:
                      rerun = True
                  else:
                      rerun = False
              if rerun or self.is_in_queue(h5_filepath):
                  self._logger.debug('Run file has already been run! Creating a fresh copy to rerun')
                  new_h5_filepath, repeat_number = self.new_rep_name(h5_filepath)
                  # Keep counting up until we get a filename that isn't in the filesystem:
                  while os.path.exists(new_h5_filepath):
                      new_h5_filepath, repeat_number = self.new_rep_name(new_h5_filepath)
                  success = self.clean_h5_file(h5_filepath, new_h5_filepath, repeat_number=repeat_number)
                  if not success:
                     return 'Cannot create a re run of this experiment. Is it a valid run file?'
                  self.append([new_h5_filepath])
                  message = "Experiment added successfully: experiment to be re-run\n"
              else:
                  self.append([h5_filepath])
                  message = "Experiment added successfully\n"
              if self.manager_paused:
                  message += "Warning: Queue is currently paused\n"
              if not self.manager_running:
                  message = "Error: Queue is not running\n"
              return message
          else:
              # TODO: Parse and display the contents of "error" in a more human readable format for analysis of what is wrong!
              message =  ("Connection table of your file is not a subset of the experimental control apparatus.\n"
                         "You may have:\n"
                         "    Submitted your file to the wrong control PC\n"
                         "    Added new channels to your h5 file, without rewiring the experiment and updating the control PC\n"
                         "    Renamed a channel at the top of your script\n"
                         "    Submitted an old file, and the experiment has since been rewired\n"
                         "\n"
                         "Please verify your experiment script matches the current experiment configuration, and try again\n"
                         "The error was %s\n"%error)
              return message
              
      def new_rep_name(self, h5_filepath):
          basename, ext = os.path.splitext(h5_filepath)
          if '_rep' in basename and ext == '.h5':
              reps = basename.split('_rep')[-1]
              try:
                  reps = int(reps)
              except ValueError:
                  # not a rep
                  pass
              else:
                  return ''.join(basename.split('_rep')[:-1]) + '_rep%05d.h5' % (reps + 1), reps + 1
          return basename + '_rep%05d.h5' % 1, 1
          
      def clean_h5_file(self, h5file, new_h5_file, repeat_number=0):
          try:
              with h5py.File(h5file, 'r') as old_file:
                  with h5py.File(new_h5_file, 'w') as new_file:
                      groups_to_copy = [
                          'devices',
                          'calibrations',
                          'script',
                          'globals',
                          'connection table',
                          'labscriptlib',
                          'waits',
                          'time_markers',
                          'shot_properties',
                      ]
                      for group in groups_to_copy:
                          if group in old_file:
                              new_file.copy(old_file[group], group)
                      for name in old_file.attrs:
                          new_file.attrs[name] = old_file.attrs[name]
                      new_file.attrs['run repeat'] = repeat_number
          except Exception as e:
              # raise
              self._logger.exception('Clean H5 File Error.')
              return False
              
          return True
      
      @inmain_decorator(wait_for_return=True)    
      def is_in_queue(self,path):                
          item = self._model.findItems(path,column=FILEPATH_COLUMN)
          if item:
              return True
          else:
              return False
  
      @inmain_decorator(wait_for_return=True)
      def set_status(self, queue_status, shot_filepath=None):
          self._ui.queue_status.setText(str(queue_status))
          if shot_filepath is not None:
              self._ui.running_shot_name.setText('<b>%s</b>'% str(os.path.basename(shot_filepath)))
          else:
              self._ui.running_shot_name.setText('')
          
      @inmain_decorator(wait_for_return=True)
      def get_status(self):
          return self._ui.queue_status.text()
              
      @inmain_decorator(wait_for_return=True)
      def get_next_file(self):
          return str(self._model.takeRow(0)[0].text())
      
      @inmain_decorator(wait_for_return=True)    
      def transition_device_to_buffered(self, name, transition_list, h5file, restart_receiver):
          tab = self.BLACS.tablist[name]
          if self.get_device_error_state(name,self.BLACS.tablist):
              return False
          tab.connect_restart_receiver(restart_receiver)
          tab.transition_to_buffered(h5file,self.current_queue)
          transition_list[name] = tab
          return True
      
      @inmain_decorator(wait_for_return=True)
      def get_device_error_state(self,name,device_list):
          return device_list[name].error_message
         
       
      def manage(self):
          logger = logging.getLogger('BLACS.queue_manager.thread')  
          process_tree.zlock_client.set_thread_name('queue_manager') 
          # While the program is running!
          logger.info('starting')
          
          # HDF5 prints lots of errors by default, for things that aren't
          # actually errors. These are silenced on a per thread basis,
          # and automatically silenced in the main thread when h5py is
          # imported. So we'll silence them in this thread too:
          h5py._errors.silence_errors()
          
          # This name stores the queue currently being used to
          # communicate with tabs, so that abort signals can be put
          # to it when those tabs never respond and are restarted by
          # the user.
          self.current_queue = queue.Queue()
  
          #TODO: put in general configuration
          timeout_limit = 300 #seconds
          self.set_status("Idle")
          
          while self.manager_running:
              # If the pause button is pushed in, sleep
              if self.manager_paused:
                  if self.get_status() == "Idle":
                      logger.info('Paused')
                      self.set_status("Queue paused") 
                  time.sleep(1)
                  continue
              
              # Get the top file
              try:
                  path = self.get_next_file()
                  self.set_status('Preparing shot...', path)
                  logger.info('Got a file: %s'%path)
              except:
                  # If no files, sleep for 1s,
                  self.set_status("Idle")
                  time.sleep(1)
                  continue
              
              devices_in_use = {}
              transition_list = {}   
              self.current_queue = queue.Queue()
  
              # Function to be run when abort button is clicked
              def abort_function():
                  try:
                      # Set device name to "Queue Manager" which will never be a labscript device name
                      # as it is not a valid python variable name (has a space in it!)
                      self.current_queue.put(['Queue Manager', 'abort'])
                  except Exception:
                      logger.exception('Could not send abort message to the queue manager')
          
              def restart_function(device_name):
                  try:
                      self.current_queue.put([device_name, 'restart'])
                  except Exception:
                      logger.exception('Could not send restart message to the queue manager for device %s'%device_name)
          
              ##########################################################################################################################################
              #                                                       transition to buffered                                                           #
              ########################################################################################################################################## 
              try:  
                  # A Queue for event-based notification when the tabs have
                  # completed transitioning to buffered:        
                  
                  timed_out = False
                  error_condition = False
                  abort = False
                  restarted = False
                  self.set_status("Transitioning to buffered...", path)
                  
                  # Enable abort button, and link in current_queue:
                  inmain(self._ui.queue_abort_button.clicked.connect,abort_function)
                  inmain(self._ui.queue_abort_button.setEnabled,True)
                                  
                  ##########################################################################################################################################
                  #                                                        Plugin callbacks                                                                #
                  ########################################################################################################################################## 
                  for callback in plugins.get_callbacks('pre_transition_to_buffered'):
                      try:
                          callback(path)
                      except Exception:
                          logger.exception("Plugin callback raised an exception")
  
                  start_time = time.time()
                  
                  with h5py.File(path, 'r') as hdf5_file:
                      devices_in_use = {}
                      start_order = {}
                      stop_order = {}
                      for name in  hdf5_file['devices']:
                          device_properties = labscript_utils.properties.get(
                              hdf5_file, name, 'device_properties'
                          )
                          devices_in_use[name] = self.BLACS.tablist[name]
                          start_order[name] = device_properties.get('start_order', None)
                          stop_order[name] = device_properties.get('start_order', None)
  
                  # Sort the devices into groups based on their start_order and stop_order
                  start_groups = defaultdict(set)
                  stop_groups = defaultdict(set)
                  for name in devices_in_use:
                      start_groups[start_order[name]].add(name)
                      stop_groups[stop_order[name]].add(name)
  
                  while (transition_list or start_groups) and not error_condition:
                      if not transition_list:
                          # Ready to transition the next group:
                          for name in start_groups.pop(min(start_groups)):
                              try:
                                  # Connect restart signal from tabs to current_queue and transition the device to buffered mode
                                  success = self.transition_device_to_buffered(name,transition_list,path,restart_function)
                                  if not success:
                                      logger.error('%s has an error condition, aborting run' % name)
                                      error_condition = True
                                      break
                              except Exception as e:
                                  logger.exception('Exception while transitioning %s to buffered mode.'%(name))
                                  error_condition = True
                                  break
                          if error_condition:
                              break
                          
                      try:
                          # Wait for a device to transtition_to_buffered:
                          logger.debug('Waiting for the following devices to finish transitioning to buffered mode: %s'%str(transition_list))
                          device_name, result = self.current_queue.get(timeout=2)
                          
                          #Handle abort button signal
                          if device_name == 'Queue Manager' and result == 'abort':
                              # we should abort the run
                              logger.info('abort signal received from GUI')
                              abort = True
                              break
                              
                          if result == 'fail':
                              logger.info('abort signal received during transition to buffered of %s' % device_name)
                              error_condition = True
                              break
                          elif result == 'restart':
                              logger.info('Device %s was restarted, aborting shot.'%device_name)
                              restarted = True
                              break
                              
                          logger.debug('%s finished transitioning to buffered mode' % device_name)
                          
                          # The tab says it's done, but does it have an error condition?
                          if self.get_device_error_state(device_name,transition_list):
                              logger.error('%s has an error condition, aborting run' % device_name)
                              error_condition = True
                              break
  
                          del transition_list[device_name]
                      except queue.Empty:
                          # It's been 2 seconds without a device finishing
                          # transitioning to buffered. Is there an error?
                          for name in transition_list:
                              if self.get_device_error_state(name,transition_list):
                                  error_condition = True
                                  break
                                  
                          if error_condition:
                              break
                              
                          # Has programming timed out?
                          if time.time() - start_time > timeout_limit:
                              logger.error('Transitioning to buffered mode timed out')
                              timed_out = True
                              break
  
                  # Handle if we broke out of loop due to timeout or error:
                  if timed_out or error_condition or abort or restarted:
                      # Pause the queue, re add the path to the top of the queue, and set a status message!
                      # only if we aren't responding to an abort click
                      if not abort:
                          self.manager_paused = True
                          self.prepend(path)                
                      if timed_out:
                          self.set_status("Programming timed out\nQueue paused")
                      elif abort:
                          self.set_status("Aborted")
                      elif restarted:
                          self.set_status("Device restarted in transition to\nbuffered. Aborted. Queue paused.")
                      else:
                          self.set_status("Device(s) in error state\nQueue Paused")
                          
                      # Abort the run for all devices in use:
                      # need to recreate the queue here because we don't want to hear from devices that are still transitioning to buffered mode
                      self.current_queue = queue.Queue()
                      for tab in devices_in_use.values():                        
                          # We call abort buffered here, because if each tab is either in mode=BUFFERED or transition_to_buffered failed in which case
                          # it should have called abort_transition_to_buffered itself and returned to manual mode
                          # Since abort buffered will only run in mode=BUFFERED, and the state is not queued indefinitely (aka it is deleted if we are not in mode=BUFFERED)
                          # this is the correct method call to make for either case
                          tab.abort_buffered(self.current_queue)
                          # We don't need to check the results of this function call because it will either be successful, or raise a visible error in the tab.
                          
                          # disconnect restart signal from tabs
                          inmain(tab.disconnect_restart_receiver,restart_function)
                          
                      # disconnect abort button and disable
                      inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
                      inmain(self._ui.queue_abort_button.setEnabled,False)
                      
                      # Start a new iteration
                      continue
                  
              
              
                  ##########################################################################################################################################
                  #                                                             SCIENCE!                                                                   #
                  ##########################################################################################################################################
              
                  # Get front panel data, but don't save it to the h5 file until the experiment ends:
                  states,tab_positions,window_data,plugin_data = self.BLACS.front_panel_settings.get_save_data()
                  self.set_status("Running (program time: %.3fs)..."%(time.time() - start_time), path)
                      
                  # A Queue for event-based notification of when the experiment has finished.
                  experiment_finished_queue = queue.Queue()
                  logger.debug('About to start the master pseudoclock')
                  run_time = time.localtime()
  
                  ##########################################################################################################################################
                  #                                                        Plugin callbacks                                                                #
                  ########################################################################################################################################## 
                  for callback in plugins.get_callbacks('science_starting'):
                      try:
                          callback(path)
                      except Exception:
                          logger.exception("Plugin callback raised an exception")
  
                  #TODO: fix potential race condition if BLACS is closing when this line executes?
                  self.BLACS.tablist[self.master_pseudoclock].start_run(experiment_finished_queue)
                  
                                                  
                  # Wait for notification of the end of run:
                  abort = False
                  restarted = False
                  done = False
                  while not (abort or restarted or done):
                      try:
                          done = experiment_finished_queue.get(timeout=0.5) == 'done'
                      except queue.Empty:
                          pass
                      try:
                          # Poll self.current_queue for abort signal from button or device restart
                          device_name, result = self.current_queue.get_nowait()
                          if (device_name == 'Queue Manager' and result == 'abort'):
                              abort = True
                          if result == 'restart':
                              restarted = True
                          # Check for error states in tabs
                          for device_name, tab in devices_in_use.items():
                              if self.get_device_error_state(device_name,devices_in_use):
                                  restarted = True
                      except queue.Empty:
                          pass
                          
                  if abort or restarted:
                      for devicename, tab in devices_in_use.items():
                          if tab.mode == MODE_BUFFERED:
                              tab.abort_buffered(self.current_queue)
                          # disconnect restart signal from tabs 
                          inmain(tab.disconnect_restart_receiver,restart_function)
                                              
                  # Disable abort button
                  inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
                  inmain(self._ui.queue_abort_button.setEnabled,False)
                  
                  if restarted:                    
                      self.manager_paused = True
                      self.prepend(path)  
                      self.set_status("Device restarted during run.\nAborted. Queue paused")
                  elif abort:
                      self.set_status("Aborted")
                      
                  if abort or restarted:
                      # after disabling the abort button, we now start a new iteration
                      continue                
                  
                  logger.info('Run complete')
                  self.set_status("Saving data...", path)
              # End try/except block here
              except Exception:
                  logger.exception("Error in queue manager execution. Queue paused.")
  
                  # Raise the error in a thread for visibility
                  zprocess.raise_exception_in_thread(sys.exc_info())
                  # clean up the h5 file
                  self.manager_paused = True
                  # is this a repeat?
                  try:
                      with h5py.File(path, 'r') as h5_file:
                          repeat_number = h5_file.attrs.get('run repeat', 0)
                  except Exception:
                      repeat_numer = 0
                  # clean the h5 file:
                  temp_path = tempfilename()
                  self.clean_h5_file(path, temp_path, repeat_number=repeat_number)
                  try:
                      shutil.move(temp_path, path)
                  except Exception:
                      msg = ('Couldn\'t delete failed run file %s, ' % path + 
                             'another process may be using it. Using alternate ' 
                             'filename for second attempt.')
                      logger.warning(msg, exc_info=True)
                      shutil.move(temp_path, path.replace('.h5','_retry.h5'))
                      path = path.replace('.h5','_retry.h5')
                  # Put it back at the start of the queue:
                  self.prepend(path)
                  
                  # Need to put devices back in manual mode
                  self.current_queue = queue.Queue()
                  for devicename, tab in devices_in_use.items():
                      if tab.mode == MODE_BUFFERED or tab.mode == MODE_TRANSITION_TO_BUFFERED:
                          tab.abort_buffered(self.current_queue)
                      # disconnect restart signal from tabs 
                      inmain(tab.disconnect_restart_receiver,restart_function)
                  self.set_status("Error in queue manager\nQueue paused")
  
                  # disconnect and disable abort button
                  inmain(self._ui.queue_abort_button.clicked.disconnect,abort_function)
                  inmain(self._ui.queue_abort_button.setEnabled,False)
                  
                  # Start a new iteration
                  continue
                               
              ##########################################################################################################################################
              #                                                           SCIENCE OVER!                                                                #
              ##########################################################################################################################################
              finally:
                  ##########################################################################################################################################
                  #                                                        Plugin callbacks                                                                #
                  ########################################################################################################################################## 
                  for callback in plugins.get_callbacks('science_over'):
                      try:
                          callback(path)
                      except Exception:
                          logger.exception("Plugin callback raised an exception")
  
              
              ##########################################################################################################################################
              #                                                       Transition to manual                                                             #
              ##########################################################################################################################################
              # start new try/except block here                   
              try:
                  with h5py.File(path,'r+') as hdf5_file:
                      self.BLACS.front_panel_settings.store_front_panel_in_h5(hdf5_file,states,tab_positions,window_data,plugin_data,save_conn_table=False, save_queue_data=False)
  
                      data_group = hdf5_file['/'].create_group('data')
                      # stamp with the run time of the experiment
                      hdf5_file.attrs['run time'] = time.strftime('%Y%m%dT%H%M%S',run_time)
          
                  error_condition = False
                  response_list = {}
                  # Keep transitioning tabs to manual mode and waiting on them until they
                  # are all done or have all errored/restarted/failed. If one fails, we
                  # still have to transition the rest to manual mode:
                  while stop_groups:
                      transition_list = {}
                      # Transition the next group to manual mode:
                      for name in stop_groups.pop(min(stop_groups)):
                          tab = devices_in_use[name]
                          try:
                              tab.transition_to_manual(self.current_queue)
                              transition_list[name] = tab
                          except Exception as e:
                              logger.exception('Exception while transitioning %s to manual mode.'%(name))
                              error_condition = True
                      # Wait for their responses:
                      while transition_list:
                          logger.info('Waiting for the following devices to finish transitioning to manual mode: %s'%str(transition_list))
                          try:
                              name, result = self.current_queue.get(2)
                              if name == 'Queue Manager' and result == 'abort':
                                  # Ignore any abort signals left in the queue, it is too
                                  # late to abort in any case:
                                  continue
                          except queue.Empty:
                              # 2 seconds without a device transitioning to manual mode.
                              # Is there an error:
                              for name in transition_list.copy():
                                  if self.get_device_error_state(name, transition_list):
                                      error_condition = True
                                      logger.debug('%s is in an error state' % name)
                                      del transition_list[name]
                              continue
                          response_list[name] = result
                          if result == 'fail':
                              error_condition = True
                              logger.debug('%s failed to transition to manual' % name)
                          elif result == 'restart':
                              error_condition = True
                              logger.debug('%s restarted during transition to manual' % name)
                          elif self.get_device_error_state(name, devices_in_use):
                              error_condition = True
                              logger.debug('%s is in an error state' % name)
                          else:
                              logger.debug('%s finished transitioning to manual mode' % name)
                          # Once device has transitioned_to_manual, disconnect restart
                          # signal:
                          tab = devices_in_use[name]
                          inmain(tab.disconnect_restart_receiver, restart_function)
                          del transition_list[name]
                      
                  if error_condition:                
                      self.set_status("Error in transtion to manual\nQueue Paused")
                                         
              except Exception as e:
                  error_condition = True
                  logger.exception("Error in queue manager execution. Queue paused.")
                  self.set_status("Error in queue manager\nQueue paused")
  
                  # Raise the error in a thread for visibility
                  zprocess.raise_exception_in_thread(sys.exc_info())
                  
              if error_condition:                
                  # clean up the h5 file
                  self.manager_paused = True
                  # is this a repeat?
                  try:
                      with h5py.File(path, 'r') as h5_file:
                          repeat_number = h5_file.attrs.get('run repeat', 0)
                  except:
                      repeat_number = 0
                  # clean the h5 file:
                  temp_path = tempfilename()
                  self.clean_h5_file(path, temp_path, repeat_number=repeat_number)
                  try:
                      shutil.move(temp_path, path)
                  except Exception:
                      msg = ('Couldn\'t delete failed run file %s, ' % path + 
                             'another process may be using it. Using alternate ' 
                             'filename for second attempt.')
                      logger.warning(msg, exc_info=True)
                      shutil.move(temp_path, path.replace('.h5','_retry.h5'))
                      path = path.replace('.h5','_retry.h5')
                  # Put it back at the start of the queue:
                  self.prepend(path)
                  
                  continue
              
              ##########################################################################################################################################
              #                                                        Analysis Submission                                                             #
              ########################################################################################################################################## 
              logger.info('All devices are back in static mode.')  
  
              # check for analysis Filters in Plugins
              send_to_analysis = True
              for callback in plugins.get_callbacks('analysis_cancel_send'):
                  try:
                      if callback(path):
                          send_to_analysis = False
                          break
                  except Exception:
                      logger.exception("Plugin callback raised an exception")
  
              # Submit to the analysis server
              if send_to_analysis:
                  self.BLACS.analysis_submission.get_queue().put(['file', path])
  
              ##########################################################################################################################################
              #                                                        Plugin callbacks                                                                #
              ########################################################################################################################################## 
              for callback in plugins.get_callbacks('shot_complete'):
                  try:
                      callback(path)
                  except Exception:
                      logger.exception("Plugin callback raised an exception")
  
              ##########################################################################################################################################
              #                                                        Repeat Experiment?                                                              #
              ##########################################################################################################################################
              # check for repeat Filters in Plugins
              repeat_shot = self.manager_repeat
              for callback in plugins.get_callbacks('shot_ignore_repeat'):
                  try:
                      if callback(path):
                          repeat_shot = False
                          break
                  except Exception:
                      logger.exception("Plugin callback raised an exception")
  
              if repeat_shot:
                  if ((self.manager_repeat_mode == self.REPEAT_ALL) or
                      (self.manager_repeat_mode == self.REPEAT_LAST and inmain(self._model.rowCount) == 0)):
                      # Resubmit job to the bottom of the queue:
                      try:
                          message = self.process_request(path)
                      except Exception:
                          # TODO: make this error popup for the user
                          self.logger.exception('Failed to copy h5_file (%s) for repeat run'%s)
                      logger.info(message)      
  
              self.set_status("Idle")
          logger.info('Stopping')
  
  