code: #
  # See top-level LICENSE.rst file for Copyright information
  #
  # -*- coding: utf-8 -*-
  """
  desispec.pipeline.db
  ===========================
  
  Pipeline processing database
  """
  
  from __future__ import absolute_import, division, print_function
  
  import os
  
  import re
  from collections import OrderedDict
  
  from contextlib import contextmanager
  
  import numpy as np
  
  from desiutil.log import get_logger
  
  from .. import io
  
  import fitsio
  
  from .defs import (task_states, task_int_to_state, task_state_to_int, task_name_sep)
  
  
  def all_task_types():
      """Get the list of possible task types that are supported.
  
      Returns:
          list: The list of supported task types.
  
      """
      from . import tasks
      from .tasks.base import default_task_chain
      ttypes = ["fibermap", "rawdata"]
      ttypes.extend(tasks.base.default_task_chain)
      # Insert qadata after cframe
      idx = ttypes.index('cframe')
      ttypes.insert(idx+1, 'qadata')
      return ttypes
  
  
  def task_sort(tasks):
      """Sort a list of tasks by type.
  
      This takes a list of arbitrary tasks and sorts them by type.  The result
      is placed in an ordered dictionary of lists in run order.
  
      Args:
          tasks (list): the list of input tasks.
  
      Returns:
          (OrderedDict): ordered dictionary of tasks sorted by type.
  
      """
      from .tasks.base import task_classes, task_type
      sort = dict()
      ttypes = all_task_types()
      for tp in ttypes:
          sort[tp] = list()
  
      for tsk in tasks:
          sort[task_type(tsk)].append(tsk)
  
      ret = OrderedDict()
      for tp in ttypes:
          if len(sort[tp]) > 0:
              ret[tp] = sort[tp]
      return ret
  
  
  def all_tasks(night, nside, expid=None):
      """Get all possible tasks for a single night.
  
      This uses the filesystem to query the raw data for a particular night and
      return a dictionary containing all possible tasks for each task type.  For
      objects which span multiple nights (e.g. spectra, redrock), this returns the
      tasks which are touched by the given night.
  
      Args:
          night (str): The night to scan for tasks.
          nside (int): The HEALPix NSIDE value to use.
          expid (int): Only get tasks for this single exposure.
  
      Returns:
          dict: a dictionary whose keys are the task types and where each value
              is a list of task properties.
  
      """
      import desimodel.footprint
  
      log = get_logger()
  
      log.debug("io.get_exposures night={}".format(night))
  
      expids = io.get_exposures(night, raw=True)
  
      full = dict()
      for t in all_task_types():
          full[t] = list()
  
      healpix_frames = []
  
      if expid is not None:
          if expid not in expids:
              raise RuntimeError("exposure ID {} not valid for night {}"\
                                 .format(expid, night))
          expids = [ expid ]
  
      for ex in sorted(expids):
  
          # get the fibermap for this exposure
          fibermap = io.get_raw_files("fibermap", night, ex)
  
          log.debug("read {}".format(fibermap))
  
          fmdata = io.read_fibermap(fibermap)
          header = fmdata.meta
  
          # fmdata, header = fitsio.read(fibermap, 'FIBERMAP', header=True)
          flavor = header["FLAVOR"].strip().lower()
          if flavor not in ["arc","flat","science"] :
              log.error("Do not know what do to with fibermap flavor '{}' for file '{}".format(flavor,fibermap))
              raise ValueError("Do not know what do to with fibermap flavor '{}' for file '{}".format(flavor,fibermap))
  
          fmpix = dict()
          if (flavor != "arc") and (flavor != "flat"):
              # This will be used to track which healpix pixels are
              # touched by fibers from each spectrograph.
              ra = np.array(fmdata["TARGET_RA"], dtype=np.float64)
              dec = np.array(fmdata["TARGET_DEC"], dtype=np.float64)
  
              # rm NaN (possible depending on versions of fiberassign)
              valid_coordinates  = (np.isnan(ra)==False)&(np.isnan(dec)==False)
  
              for spectro in np.unique( fmdata["SPECTROID"] ) :
                  ii=np.where(fmdata["SPECTROID"][valid_coordinates]==spectro)[0]
                  if ii.size == 0 : continue
                  pixels  = desimodel.footprint.radec2pix(nside, ra[valid_coordinates][ii], dec[valid_coordinates][ii])
                  for pixel in np.unique(pixels) :
                      props = dict()
                      props["night"] = int(night)
                      props["expid"] = int(ex)
                      props["spec"]  = spectro
                      props["nside"] = nside
                      props["pixel"] = pixel
                      props["ntargets"] = np.sum(pixels==pixel)
                      healpix_frames.append(props)
              # all spectro at once
              pixels  = np.unique(desimodel.footprint.radec2pix(nside, ra[valid_coordinates], dec[valid_coordinates]))
              for pixel in pixels :
                  props = dict()
                  props["pixel"] = pixel
                  props["nside"] = nside
                  props["state"]  = "waiting"
                  exists=False
                  for entry in full["spectra"] :
                      if entry["pixel"]==props["pixel"] :
                          exists=True
                          break
                  if not exists : full["spectra"].append(props)
                  exists=False
                  for entry in full["redshift"] :
                      if entry["pixel"]==props["pixel"] :
                          exists=True
                          break
                  if not exists : full["redshift"].append(props)
  
          fmprops = dict()
          fmprops["night"]  = int(night)
          fmprops["expid"]  = int(ex)
          fmprops["flavor"] = flavor
          fmprops["state"]  = "done"
  
          full["fibermap"].append(fmprops)
  
          rdprops = dict()
          rdprops["night"]  = int(night)
          rdprops["expid"]  = int(ex)
          rdprops["flavor"] = flavor
          rdprops["state"]  = "done"
  
          full["rawdata"].append(rdprops)
  
          # Add the preprocessed pixel files
          for band in ['b', 'r', 'z']:
              # need to open the rawdata file to see how many spectros
              # and cameras are there
              for spec in np.unique( fmdata["SPECTROID"] ) :
                  pixprops = dict()
                  pixprops["night"] = int(night)
                  pixprops["band"] = band
                  pixprops["spec"] = spec
                  pixprops["expid"] = int(ex)
                  pixprops["flavor"] = flavor
                  pixprops["state"] = "ready"
                  full["preproc"].append(pixprops)
  
                  if flavor == "arc" :
                      # Add the PSF files
                      props = dict()
                      props["night"] = int(night)
                      props["band"] = band
                      props["spec"] = spec
                      props["expid"] = int(ex)
                      props["state"] = "waiting" # see defs.task_states
                      full["psf"].append(props)
  
                      # Add a PSF night file if does not exist
                      exists=False
                      for entry in full["psfnight"] :
                          if entry["night"]==props["night"] \
                             and entry["band"]==props["band"] \
                             and entry["spec"]==props["spec"] :
                              exists=True
                              break
                      if not exists :
                           props = dict()
                           props["night"] = int(night)
                           props["band"] = band
                           props["spec"] = spec
                           props["state"] = "waiting" # see defs.task_states
                           full["psfnight"].append(props)
  
                  if flavor != "arc" :
                      # Add extractions
                      props = dict()
                      props["night"] = int(night)
                      props["band"] = band
                      props["spec"] = spec
                      props["expid"] = int(ex)
                      props["state"] = "waiting" # see defs.task_states
  
                      # Add traceshift
                      full["traceshift"].append(props)
  
                      # Add extractions
                      full["extract"].append(props)
  
                  if flavor == "flat" :
                      # Add a fiberflat task
                      props = dict()
                      props["night"] = int(night)
                      props["band"] = band
                      props["spec"] = spec
                      props["expid"] = int(ex)
                      props["state"] = "waiting" # see defs.task_states
                      full["fiberflat"].append(props)
                      # Add a fiberflat night file if does not exist
                      exists=False
                      for entry in full["fiberflatnight"] :
                          if entry["night"]==props["night"] \
                             and entry["band"]==props["band"] \
                             and entry["spec"]==props["spec"] :
                              exists=True
                              break
                      if not exists :
                           props = dict()
                           props["night"] = int(night)
                           props["band"] = band
                           props["spec"] = spec
                           props["state"] = "waiting" # see defs.task_states
                           full["fiberflatnight"].append(props)
  
                  if flavor != "arc" and flavor != "flat":
                      # Add sky
                      props = dict()
                      props["night"] = int(night)
                      props["band"] = band
                      props["spec"] = spec
                      props["expid"] = int(ex)
                      props["state"] = "waiting" # see defs.task_states
                      full["sky"].append(props)
                      # Add fluxcalib
                      full["fluxcalib"].append(props)
                      # Add cframe
                      full["cframe"].append(props)
                      # Add QA
                      full["qadata"].append(props)
  
                      # Add starfit if does not exist
                      exists=False
                      for entry in full["starfit"] :
                          if entry["night"]==props["night"] \
                             and entry["expid"]==props["expid"] \
                             and entry["spec"]==props["spec"] :
                              exists=True
                              break
                      if not exists :
                           props = dict()
                           props["night"] = int(night)
                           props["expid"] = int(ex)
                           props["spec"] = spec
                           props["state"] = "waiting" # see defs.task_states
                           full["starfit"].append(props)
  
      log.debug("done")
      return full , healpix_frames
  
  
  def check_tasks(tasklist, db=None, inputs=None):
      """Check a list of tasks and return their state.
  
      If the database is specified, it is used to check the state of the tasks
      and their dependencies.  Otherwise the filesystem is checked.
  
      Args:
          tasklist (list): list of tasks.
          db (pipeline.db.DB): The optional database to use.
          inputs (dict): optional dictionary containing the only input
              dependencies that should be considered.
  
      Returns:
          dict: The current state of all tasks.
  
      """
      from .tasks.base import task_classes, task_type
      states = dict()
  
      if db is None:
          # Check the filesystem to see which tasks are done.  Since we don't
          # have a DB, we can only distinguish between "waiting", "ready", and
          # "done" states.
          for tsk in tasklist:
              tasktype = task_type(tsk)
              st = "waiting"
  
              # Check dependencies
              deps = task_classes[tasktype].deps(tsk, db=db, inputs=inputs)
  
              if len(deps)==0 :
                  # do not set state to ready of tasks with 0 dependencies
                  ready = False
              else :
                  ready = True
                  for k, v in deps.items():
                      if not isinstance(v, list):
                          v = [ v ]
                      for dp in v:
                          deptype = task_type(dp)
                          depfiles = task_classes[deptype].paths(dp)
                          for odep in depfiles:
                              if not os.path.isfile(odep):
                                  ready = False
                                  break
                  if ready:
                      st = "ready"
  
              done = True
              # Check outputs
              outfiles = task_classes[tasktype].paths(tsk)
              for out in outfiles:
                  if not os.path.isfile(out):
                      done = False
                      break
              if done:
                  st = "done"
  
              states[tsk] = st
      else:
          states = db.get_states(tasklist)
  
      return states
  
  
  class DataBase:
      """Class for tracking pipeline processing objects and state.
      """
      def __init__(self):
          self._conn = None
          return
  
  
      def get_states_type(self, tasktype, tasks):
          """Efficiently get the state of many tasks of a single type.
  
          Args:
              tasktype (str): the type of these tasks.
              tasks (list): list of task names.
  
          Returns:
              dict: the state of each task.
  
          """
          states = None
          namelist = ",".join([ "'{}'".format(x) for x in tasks ])
  
          log = get_logger()
          log.debug("opening db")
  
          with self.cursor() as cur:
              log.debug("selecting in db")
              cur.execute(\
                  'select name, state from {} where name in ({})'.format(tasktype,
                  namelist))
              st = cur.fetchall()
              log.debug("done")
              states = { x[0] : task_int_to_state[x[1]] for x in st }
          return states
  
  
      def count_task_states(self, tasktype):
          """Return a dictionary of how many tasks are in each state
  
          Args:
              tasktype (str): the type of these tasks.
  
          Returns:
              dict: keyed by state, values are number of tasks in that state0
          """
          state_count = OrderedDict()
          for state in task_states:
              state_count[state] = 0
  
          with self.cursor() as cur:
              cur.execute( 'select name, state from {}'.format(tasktype))
              for name, intstate in cur.fetchall():
                  state_count[task_int_to_state[intstate]] += 1
  
          return state_count
  
  
      def get_states(self, tasks):
          """Efficiently get the state of many tasks at once.
  
          Args:
              tasks (list): list of task names.
  
          Returns:
              dict: the state of each task.
  
          """
          from .tasks.base import task_classes, task_type
  
          # Sort by type
          taskbytype = task_sort(tasks)
  
          # Get state of each type
          states = dict()
          for t, tlist in taskbytype.items():
              states.update(self.get_states_type(t, tlist))
  
          return states
  
  
      def set_states_type(self, tasktype, tasks, postprocessing=True):
          """Efficiently get the state of many tasks of a single type.
  
          Args:
              tasktype (str): the type of these tasks.
              tasks (list): list of tuples containing the task name and the
                  state to set.
  
          Returns:
              Nothing.
  
          """
          from .tasks.base import task_classes
  
          log = get_logger()
          log.debug("opening db")
  
          with self.cursor() as cur:
              log.debug("updating in db")
              for tsk in tasks:
                  cur.execute("update {} set state = {} where name = '{}'".format(tasktype, task_state_to_int[tsk[1]], tsk[0]))
                  if postprocessing and tsk[1]=="done" :
                      task_classes[tasktype].postprocessing(db=self,name=tsk[0],cur=cur)
              log.debug("done")
          return
  
  
      def set_states(self, tasks):
          """Efficiently set the state of many tasks at once.
  
          Args:
              tasks (list): list of tuples containing the task name and the
                  state to set.
  
          Returns:
              Nothing.
  
          """
          from .tasks.base import task_classes, task_type
          # First find the type of each task.
          ttypes = dict()
          for tsk in tasks:
              ttypes[tsk[0]] = task_type(tsk[0])
  
          # Sort tasks into types
          taskbytype = dict()
          for t in all_task_types():
              taskbytype[t] = list()
          for tsk in tasks:
              taskbytype[ttypes[tsk[0]]].append(tsk)
  
          # Process each type
          for t, tlist in taskbytype.items():
              if len(tlist) > 0:
                  self.set_states_type(t, tlist)
          return
  
  
      def get_submitted(self, tasks):
          """Return the submitted flag for the list of tasks.
  
          Args:
              tasks (list): list of task names.
  
          Returns:
              (dict): the boolean submitted state of each task (True means that
                  the task has been submitted).
  
          """
          from .tasks.base import task_type
          # Sort by type
          taskbytype = task_sort(tasks)
  
          # Process each type
          submitted = dict()
          for t, tlist in taskbytype.items():
              if (t == "spectra") or (t == "redshift"):
                  raise RuntimeError("spectra and redshift tasks do not have submitted flag.")
              namelist = ",".join([ "'{}'".format(x) for x in tlist ])
              with self.cursor() as cur:
                  cur.execute(\
                      'select name, submitted from {} where name in ({})'.format(t, namelist))
                  sb = cur.fetchall()
                  submitted.update({ x[0] : x[1] for x in sb })
          return submitted
  
  
      def set_submitted_type(self, tasktype, tasks, unset=False):
          """Flag a list of tasks of a single type as submitted.
  
          Args:
              tasktype (str): the type of these tasks.
              tasks (list): list of task names.
              unset (bool): if True, invert the behavior and unset the submitted
                  flag for these tasks.
  
          Returns:
              Nothing.
  
          """
          val = 1
          if unset:
              val = 0
          with self.cursor() as cur:
              for tsk in tasks:
                  cur.execute("update {} set submitted = {} where name = '{}'".format(tasktype, val, tsk))
          return
  
  
      def set_submitted(self, tasks, unset=False):
          """Flag a list of tasks as submitted.
  
          Args:
              tasks (list): list of task names.
              unset (bool): if True, invert the behavior and unset the submitted
                  flag for these tasks.
  
          Returns:
              Nothing.
  
          """
          from .tasks.base import task_type
          # Sort by type
          taskbytype = task_sort(tasks)
  
          # Process each type
          for t, tlist in taskbytype.items():
              if (t == "spectra") or (t == "redshift"):
                  raise RuntimeError("spectra and redshift tasks do not have submitted flag.")
              self.set_submitted_type(tlist, unset=unset)
          return
  
  
      def update(self, night, nside, expid=None):
          """Update DB based on raw data.
  
          This will use the usual io.meta functions to find raw exposures.  For
          each exposure, the fibermap and all following objects will be added to
          the DB.
  
          Args:
              night (str): The night to scan for updates.
              nside (int): The current NSIDE value used for pixel grouping.
              expid (int): Only update the DB for this exposure.
  
          """
          from .tasks.base import task_classes, task_type
  
          log = get_logger()
  
          alltasks, healpix_frames = all_tasks(night, nside, expid=expid)
  
          with self.cursor() as cur:
              # insert or ignore all healpix_frames
              log.debug("updating healpix_frame ...")
              for entry in healpix_frames:
                  # see if we already have this entry
                  cmd = "select exists(select 1 from healpix_frame where (expid = {} and spec = {} and nside = {} and pixel = {} ))".format(entry["expid"], entry["spec"], entry["nside"], entry["pixel"])
                  cur.execute(cmd)
                  have_row = cur.fetchone()[0]
  
                  if not have_row:
                      cur.execute("insert into healpix_frame (night,expid,spec,nside,pixel,ntargets,state) values({},{},{},{},{},{},{})".format(entry["night"],entry["expid"],entry["spec"],entry["nside"],entry["pixel"],entry["ntargets"],0))
  
              # read what is already in db
              tasks_in_db = {}
              for tt in all_task_types():
                  cur.execute("select name from {}".format(tt))
                  tasks_in_db[tt] = [ x for (x, ) in cur.fetchall()]
  
              for tt in all_task_types():
                  log.debug("updating {} ...".format(tt))
                  for tsk in alltasks[tt]:
                      tname = task_classes[tt].name_join(tsk)
                      if tname not in tasks_in_db[tt] :
                          log.debug("adding {}".format(tname))
                          task_classes[tt].insert(cur, tsk)
  
          return
  
  
      def sync(self, night, specdone=False):
          """Update states of tasks based on filesystem.
  
          Go through all tasks in the DB for the given night and determine their
          state on the filesystem.  Then update the DB state to match.
  
          Args:
              night (str): The night to scan for updates.
              specdone: If true, set spectra to done if files exist.
          """
          from .tasks.base import task_classes
          log = get_logger()
  
          # Get the list of task types excluding spectra and redshifts,
          # which will be handled separately.
          ttypes = [ t for t in all_task_types() if (t != "spectra") \
              and (t != "redshift") ]
  
          tasks_in_db = None
          # Grab existing nightly tasks
          with self.cursor() as cur:
              tasks_in_db = {}
              for tt in ttypes:
                  cur.execute("select name from {} where night = {}"\
                      .format(tt, night))
                  tasks_in_db[tt] = [ x for (x, ) in cur.fetchall() ]
  
          # For each task type, check status WITHOUT the DB, then set state.
          # Save out the cframe states for later use with the healpix_frame table
          cfstates = None
          for tt in ttypes:
              tstates = check_tasks(tasks_in_db[tt], db=None)
              st = [ (x, tstates[x]) for x in tasks_in_db[tt] ]
              self.set_states_type(tt, st)
              if tt == "cframe":
                  cfstates = tstates.copy()
  
          # Now examine the spectra and redshift files.  If the files exist,
          # we assume they are done and completely up to date.  If the files
          # are not up to date, they must be manually deleted in order for the
          # sync to correctly reconstruct the database state.
  
          pixrows = self.select_healpix_frame({"night" : night})
          # First check the existence of the files touched by this night
          spec_exists = dict()
          red_exists = dict()
          for row in pixrows:
              if row["pixel"] in spec_exists:
                  continue
              spec_name = task_classes["spectra"].name_join(row)
              red_name = task_classes["redshift"].name_join(row)
  
              # Check spectra outputs
              outfiles = task_classes["spectra"].paths(spec_name)
              spec_exists[row["pixel"]] = True
              for out in outfiles:
                  if not os.path.isfile(out):
                      spec_exists[row["pixel"]] = False
                      break
  
              # Check redshift outputs
              outfiles = task_classes["redshift"].paths(red_name)
              red_exists[row["pixel"]] = True
              for out in outfiles:
                  if not os.path.isfile(out):
                      red_exists[row["pixel"]] = False
                      break
  
          # Now use all this info.  Some internal helpers to avoid code
          # duplication
          def set_hpx_frame_0(row, spec, red, cur):
              self.update_healpix_frame_state(row, 0, cur)
              task_classes["spectra"].state_set(
                  self, spec, "waiting", cur)
              task_classes["redshift"].state_set(
                  self, red, "waiting", cur)
              return
  
          def set_hpx_frame_1(row, spec, red, cur):
              self.update_healpix_frame_state(row, 1, cur)
              # getready() will do this for us:
              #task_classes["spectra"].state_set(
              #    self, spec, "ready", cur)
              task_classes["redshift"].state_set(
                  self, red, "waiting", cur)
              return
  
          def set_hpx_frame_2(row, spec, red, cur):
              self.update_healpix_frame_state(row, 2, cur)
              task_classes["spectra"].state_set(
                  self, spec, "done", cur)
              # getready() will do this:
              #task_classes["redshift"].state_set(
              #    self, red, "ready", cur)
              return
  
          def set_hpx_frame_3(row, spec, red, cur):
              self.update_healpix_frame_state(row, 3, cur)
              task_classes["spectra"].state_set(
                  self, spec, "done", cur)
              task_classes["redshift"].state_set(
                  self, red, "done", cur)
              return
  
          with self.cursor() as cur:
              for row in pixrows:
                  cfdone = True
                  cfprops = row.copy()
                  for band in ["b", "r", "z"]:
                      cfprops["band"] = band
                      cf_name = task_classes["cframe"].name_join(cfprops)
                      if cfstates[cf_name] != "done":
                          cfdone = False
  
                  spec_name = task_classes["spectra"].name_join(row)
                  red_name = task_classes["redshift"].name_join(row)
  
                  if (not cfdone) and (not specdone) :
                      # The cframes do not exist, so reset the state of the
                      # spectra and redshift tasks.
                      set_hpx_frame_0(row, spec_name, red_name, cur)
                  else:
                      # The cframe exists...
                      if spec_exists[row["pixel"]]:
                          if red_exists[row["pixel"]]:
                              # We are all done (state 3)
                              set_hpx_frame_3(row, spec_name, red_name, cur)
                          else:
                              # We are only at state 2
                              set_hpx_frame_2(row, spec_name, red_name, cur)
                      else:
                          # We are at just at state 1
                          set_hpx_frame_1(row, spec_name, red_name, cur)
  
          # Update ready state of tasks
          self.getready(night=night)
  
          return
  
  
      def cleanup(self, tasktypes=None, expid=None, cleanfailed=False,
                  cleansubmitted=False):
          """Reset states of tasks.
  
          Any tasks that are marked as "running" will have their
          state reset to "ready".  This can be called if a job dies before
          completing all tasks.
  
          Args:
              tasktypes (list): if not None, clean up only tasks of these types.
              expid (int): if not None, only clean tasks related to this
                  exposure ID.  Note that tasks which are independent of
                  an expid (psfnight, fiberflatnight, spectra, redshift)
                  will be ignored if this option is given.
              cleanfailed (bool): if True, also reset failed tasks to ready.
              cleansubmitted (bool): if True, set submitted flag to False.
  
          """
          tasks_running = None
  
          alltypes = all_task_types()
          ttypes = None
          if tasktypes is None:
              ttypes = alltypes
          else:
              for tt in tasktypes:
                  if tt not in alltypes:
                      raise RuntimeError("Cannot clean invalid task type {}"\
                          .format(tt))
              ttypes = tasktypes
  
          # Grab existing nightly tasks
          with self.cursor() as cur:
              tasks_running = {}
              for tt in ttypes:
                  hasexpid = (tt not in ["psfnight", "fiberflatnight", "spectra",
                              "redshift"])
                  if hasexpid:
                      # This task type has an expid property.
                      cmd = None
                      if expid is not None:
                          # We are cleaning only a single exposure.
                          cmd = "select name from {} where expid = {} and ( state = {}".format(tt, expid, task_state_to_int["running"])
                      else:
                          # We are cleaning all exposures for this task type.
                          cmd = "select name from {} where ( state = {}".format(tt, task_state_to_int["running"])
                      if cleanfailed:
                          cmd = "{} or state = {} )".format(cmd,
                              task_state_to_int["failed"])
                      else:
                          cmd = "{} )".format(cmd)
                      cur.execute(cmd)
                      tasks_running[tt] = [ x for (x, ) in cur.fetchall() ]
                      if cleansubmitted:
                          if expid is not None:
                              cmd = "update {} set submitted = 0 where expid = {}".format(tt, expid)
                          else:
                              cmd = "update {} set submitted = 0".format(tt)
                          cur.execute(cmd)
                  else:
                      # This task type has no concept of an exposure ID
                      if expid is not None:
                          # We specified an exposure ID, which makes no sense
                          # for this task type.  Skip it.
                          tasks_running[tt] = list()
                          continue
                      else:
                          # cleanup this task type.
                          cmd = "select name from {} where ( state = {}".format(tt, task_state_to_int["running"])
                          if cleanfailed:
                              cmd = "{} or state = {} )".format(cmd,
                                  task_state_to_int["failed"])
                          else:
                              cmd = "{} )".format(cmd)
                          cur.execute(cmd)
                          tasks_running[tt] = [ x for (x, ) in cur.fetchall() ]
                          if cleansubmitted:
                              if (tt != "spectra") and (tt != "redshift"):
                                  cmd = "update {} set submitted = 0".format(tt)
                                  cur.execute(cmd)
  
          for tt in ttypes:
              if len(tasks_running[tt]) > 0:
                  st = [ (x, "waiting") for x in tasks_running[tt] ]
                  self.set_states_type(tt, st)
  
          self.getready()
  
          return
  
  
      def getready(self, night=None):
          """Update DB, changing waiting to ready depending on status of dependencies .
  
          Args:
              night (str): The night to process.
  
          """
          from .tasks.base import task_classes, task_type
          log = get_logger()
  
          # Get the list of task types excluding spectra and redshifts,
          # which will be handled separately.
          ttypes = [ t for t in all_task_types() if (t != "spectra") \
              and (t != "redshift") ]
  
          with self.cursor() as cur:
              for tt in ttypes:
                  # for each type of task, get the list of tasks in waiting mode
                  cmd = "select name from {} where state = {}".format(tt, task_state_to_int["waiting"])
                  if night is not None:
                      cmd = "{} and night = {}".format(cmd, night)
                  cur.execute(cmd)
                  tasks = [ x for (x, ) in cur.fetchall()]
                  if len(tasks) > 0:
                      log.debug("checking {} {} tasks ...".format(len(tasks),tt))
                  for tsk in tasks:
                      task_classes[tt].getready(db=self, name=tsk, cur=cur)
  
              for tt in [ "spectra" , "redshift" ]:
                  if tt == "spectra":
                      required_healpix_frame_state = 1
                      # means we have a cframe
                  elif tt == "redshift":
                      required_healpix_frame_state = 2
                      # means we have an updated spectra file
  
                  cur.execute('select nside,pixel from healpix_frame where state = {}'.format(required_healpix_frame_state))
                  entries = cur.fetchall()
                  for entry in entries :
                      log.debug("{} of pixel {} is ready to run".format(tt,entry[1]))
                      cur.execute('update {} set state = {} where nside = {} and pixel = {}'.format(tt,task_state_to_int["ready"],entry[0],entry[1]))
  
                  log.debug("checking waiting {} tasks to see if they are done...".format(tt))
                  cmd = "select pixel from {} where state = {}".format(tt, task_state_to_int["waiting"])
                  cur.execute(cmd)
                  pixels = [ x for (x, ) in cur.fetchall()]
                  if len(pixels) > 0:
                      log.debug("checking {} {} ...".format(len(pixels),tt))
                      if tt == "spectra":
                          required_healpix_frame_state = 2
                      elif tt == "redshift":
                          required_healpix_frame_state = 3
                      for pixel in pixels:
                          cur.execute('select pixel from healpix_frame where pixel = {} and state != {}'.format(pixel,required_healpix_frame_state))
                          entries = cur.fetchall()
                          if len(entries)==0 :
                              log.debug("{} task of pixel {} is done".format(tt,pixel))
                              cur.execute('update {} set state = {} where pixel = {}'.format(tt,task_state_to_int["done"],pixel))
          return
  
  
      def update_healpix_frame_state(self, props, state, cur):
          if "expid" in props :
              # update from a cframe
              cmd = "update healpix_frame set state = {} where expid = {} and spec = {} and state = {}".format(state,props["expid"],props["spec"],props["state"])
          else :
              # update from a spectra or redshift task
              cmd = "update healpix_frame set state = {} where nside = {} and pixel = {} and state = {}".format(state,props["nside"],props["pixel"],props["state"])
  
          if cur is None :
              with self.cursor() as cur:
                  cur.execute(cmd)
          else :
              cur.execute(cmd)
          return
  
  
      def select_healpix_frame(self, props):
          res = []
          with self.cursor() as cur:
              cmd = "select * from healpix_frame where "
              first=True
              for k in props.keys() :
                  if not first : cmd += " and "
                  first=False
                  cmd += "{}={}".format(k,props[k])
              cur.execute(cmd)
              entries = cur.fetchall()
              # convert that to list of dictionaries
              for entry in entries :
                  tmp = dict()
                  for i, k in enumerate(["night", "expid", "spec", "nside",
                                         "pixel", "ntargets", "state"]):
                      tmp[k] = entry[i]
                  res.append(tmp)
          return res
  
  
      def create_healpix_frame_table(self) :
          with self.cursor() as cur:
              cmd = "create table healpix_frame (night integer, expid integer, spec integer, nside integer, pixel integer, ntargets integer, state integer, unique(expid, spec, nside, pixel))"
              cur.execute(cmd)
  
          return
  
  
  class DataBaseSqlite(DataBase):
      """Pipeline database using sqlite3 as the backend.
  
      Args:
          path (str): the filesystem path of the database to open.  If None, then
              a temporary database is created in memory.
          mode (str): if "r", the database is open in read-only mode.  If "w",
              the database is open in read-write mode and created if necessary.
  
      """
      def __init__(self, path, mode):
          super(DataBaseSqlite, self).__init__()
  
          self._path = path
          self._mode = mode
  
          create = True
          if (self._path is not None) and os.path.exists(self._path):
              create = False
  
          if self._mode == 'r' and create:
              raise RuntimeError("cannot open a non-existent DB in read-only "
                  " mode")
  
          self._connstr = None
  
          # This timeout is in seconds
          self._busytime = 1000
  
          # Journaling options
          self._journalmode = "persist"
          self._syncmode = "normal"
  
          if create:
              self.initdb()
          return
  
  
      def _open(self):
          import sqlite3
  
          if self._path is None:
              # We are opening an in-memory DB
              self._conn = sqlite3.connect(":memory:")
          else:
              try:
                  # only python3 supports uri option
                  if self._mode == 'r':
                      self._connstr = 'file:{}?mode=ro'.format(self._path)
                  else:
                      self._connstr = 'file:{}?mode=rwc'.format(self._path)
                  self._conn = sqlite3.connect(self._connstr, uri=True,
                      timeout=self._busytime)
              except:
                  self._conn = sqlite3.connect(self._path, timeout=self._busytime)
          if self._mode == 'w':
              # In read-write mode, set the journaling
              self._conn.execute("pragma journal_mode={}"\
                  .format(self._journalmode))
              self._conn.execute("pragma synchronous={}".format(self._syncmode))
              # Other tuning options
              self._conn.execute("pragma temp_store=memory")
              self._conn.execute("pragma page_size=4096")
              self._conn.execute("pragma cache_size=4000")
          return
  
  
      def _close(self):
          del self._conn
          self._conn = None
          return
  
  
      @contextmanager
      def cursor(self):
          import sqlite3
          self._open()
          cur = self._conn.cursor()
          cur.execute("begin transaction")
          try:
              yield cur
          except sqlite3.DatabaseError as err:
              log = get_logger()
              log.error(err)
              cur.execute("rollback")
              raise err
          else:
              try:
                  cur.execute("commit")
              except sqlite3.OperationalError:
                  #- sqlite3 in py3.5 can't commit a read-only finished transaction
                  pass
          finally:
              del cur
              self._close()
  
  
      def initdb(self):
          """Create DB tables for all tasks if they do not exist.
          """
          # check existing tables
          tables_in_db = None
          with self.cursor() as cur:
              cur.execute("select name FROM sqlite_master WHERE type='table'")
              tables_in_db = [x for (x, ) in cur.fetchall()]
  
          # Create a table for every task type
          from .tasks.base import task_classes, task_type
          for tt, tc in task_classes.items():
              if tt not in tables_in_db:
                  tc.create(self)
  
          if "healpix_frame" not in tables_in_db:
              self.create_healpix_frame_table()
          return
  
  
  class DataBasePostgres(DataBase):
      """Pipeline database using PostgreSQL as the backend.
  
      Args:
          host (str): The database server.
          port (int): The connection port.
          dbname (str): The database to connect.
          user (str): The user name for the connection.  The password should be
              stored in the ~/.pgpass file.
          schema (str): The schema within the database.  If this is specified,
              then the database is assumed to exist.  Otherwise the schema is
              computed from a hash of the production location and will be
              created.
          authorize (str): If creating the schema, this is the list of
              additional roles that should be granted access.
  
      """
      def __init__(self, host, port, dbname, user, schema=None, authorize=None):
          super(DataBasePostgres, self).__init__()
  
          self._schema = schema
          self._user = user
          self._dbname = dbname
          self._host = host
          self._port = port
          self._authorize = authorize
  
          self._proddir = os.path.abspath(io.specprod_root())
  
          create = False
          if self._schema is None:
              create = True
              self._schema = self._compute_schema()
  
          if create:
              self.initdb()
          return
  
  
      def _compute_schema(self):
          import hashlib
          md = hashlib.md5()
          md.update(self._proddir.encode())
          return "pipe_{}".format(md.hexdigest())
  
  
      def _open(self):
          import psycopg2 as pg2
          import time
          import numpy.random
  
          # Open connection.  If psycopg2 raises an exception, then sleep
          # for a random time interval and keep trying.
          maxtry = 10
          ntry = 0
          while True:
              try:
                  self._conn = pg2.connect(host=self._host, port=self._port,
                      user=self._user, dbname=self._dbname)
              except pg2.OperationalError as err:
                  log = get_logger()
                  log.debug("PostgreSQL connection failed with '{}', will sleep and retry".format(err))
                  if ntry > maxtry:
                      log.error(err)
                      break
                  numpy.random.seed(int(time.time()))
                  sec = numpy.random.uniform() * 3.0
                  time.sleep(sec)
                  ntry += 1
              else:
                  break
  
          return
  
  
      def _close(self):
          del self._conn
          self._conn = None
          return
  
  
      @property
      def schema(self):
          return self._schema
  
  
      def _have_schema(self, cur):
          com = "select exists(select 1 from pg_namespace where nspname = '{}')".format(self._schema)
          cur.execute(com)
          return cur.fetchone()[0]
  
  
      @contextmanager
      def cursor(self, skipcheck=False):
          import psycopg2
          self._open()
          cur = self._conn.cursor()
          if not skipcheck:
              have_schema = self._have_schema(cur)
              if not have_schema:
                  raise RuntimeError("Postgres schema for production {} does"
                      " not exist.  Make sure you create the production with"
                      " postgres options and source the top-level setup.sh"
                      " file.".format(self._proddir))
          cur.execute("set search_path to '{}'".format(self._schema))
          cur.execute("begin transaction")
          try:
              yield cur
          except psycopg2.DatabaseError as err:
              log = get_logger()
              log.error(err)
              cur.execute("rollback")
              raise err
          else:
              cur.execute("commit")
          finally:
              del cur
              self._close()
  
  
      def initdb(self):
          """Create DB tables for all tasks if they do not exist.
          """
          log = get_logger()
          # Check existence of the schema.  If we were not passed the schema
          # in the constructor, it means that we are creating a new prod, so any
          # existing schema should be wiped and recreated.
          tables_in_db = None
          with self.cursor(skipcheck=True) as cur:
              # See if our schema already exists...
              have_schema = self._have_schema(cur)
              if have_schema:
                  # We need to wipe it first
                  com = "drop schema {} cascade".format(self._schema)
                  log.debug(com)
                  cur.execute(com)
              com = "create schema {} authorization {}"\
                  .format(self._schema, self._user)
              log.debug(com)
              cur.execute(com)
  
              if self._authorize is not None:
                  com = "grant usage on schema {} to {}"\
                      .format(self._schema, self._authorize)
                  log.debug(com)
                  cur.execute(com)
  
                  com = "alter default privileges in schema {} grant select on tables to {}".format(self._schema, self._authorize)
                  log.debug(com)
                  cur.execute(com)
  
                  com = "alter default privileges in schema {} grant select,usage on sequences to {}".format(self._schema, self._authorize)
                  log.debug(com)
                  cur.execute(com)
  
                  com = "alter default privileges in schema {} grant execute on functions to {}".format(self._schema, self._authorize)
                  log.debug(com)
                  cur.execute(com)
  
                  com = "alter default privileges in schema {} grant usage on types to {}".format(self._schema, self._authorize)
                  log.debug(com)
                  cur.execute(com)
  
              # Create a table of information about this prod
              com = "create table {}.info (key text unique, val text)"\
                  .format(self._schema)
              log.debug(com)
              cur.execute(com)
              com = "insert into {}.info values ('{}', '{}')"\
                  .format(self._schema, "path", self._proddir)
              log.debug(com)
              cur.execute(com)
              if 'USER' in os.environ:
                  com = "insert into {}.info values ('{}', '{}')"\
                      .format(self._schema, "created_by", os.environ['USER'])
                  log.debug(com)
                  cur.execute(com)
  
              # check existing tables
              cur.execute("select tablename from pg_tables where schemaname =  '{}'".format(self.schema))
              tables_in_db = [x for (x, ) in cur.fetchall()]
  
          # Create a table for every task type
          from .tasks.base import task_classes, task_type
          for tt, tc in task_classes.items():
              if tt not in tables_in_db:
                  tc.create(self)
  
          if "healpix_frame" not in tables_in_db:
              self.create_healpix_frame_table()
  
          return
  
  
  def load_db(dbstring, mode="w", user=None):
      """Load a database from a connection string.
  
      This instantiates either an sqlite or postgresql database using a string.
      If this string begins with "postgresql:", then it is taken to be the
      information needed to connect to a postgres server.  Otherwise it is
      assumed to be a filesystem path to use with sqlite.  The mode is only
      meaningful when using sqlite.  Postgres permissions are controlled through
      the user permissions.
  
      Args:
          dbstring (str): either a filesystem path (sqlite) or a colon-separated
              string of connection properties in the form
              "postresql:<host>:<port>:<dbname>:<user>:<schema>".
          mode (str): for sqlite, the mode.
          user (str): for postgresql, an alternate user name for opening the DB.
              This can be used to connect as a user with read-only access.
  
      Returns:
          DataBase: a derived database class of the appropriate type.
  
      """
      if re.search(r"postgresql:", dbstring) is not None:
          props = dbstring.split(":")
          host = props[1]
          port = int(props[2])
          dbname = props[3]
          username = props[4]
          if user is not None:
              username = user
          schema = None
          if len(props) > 5:
              # Our DB string also contains the name of an existing
              # schema.
              schema = props[5]
          return DataBasePostgres(host=host, port=port, dbname=dbname,
              user=username, schema=schema)
      else:
          return DataBaseSqlite(dbstring, mode)
   
  from __future__ import absolute_import, division
  
  import os
  import fitsio
  import argparse
  import numpy as np
  
  from desiutil.log import get_logger
  
  from desispec.io import read_fiberflat,write_fiberflat,findfile,read_frame
  from desispec.io.fiberflat_vs_humidity import get_humidity,read_fiberflat_vs_humidity
  from desispec.calibfinder import CalibFinder
  from desispec.fiberflat_vs_humidity import compute_humidity_corrected_fiberflat
  
  def parse(options=None):
      parser = argparse.ArgumentParser(description="Compute a fiberflat corrected for variations with humidity.")
  
      parser.add_argument('-i','--infile', type = str, default = None, required=True,
                          help = 'path of DESI exposure frame fits file')
      parser.add_argument('--fiberflat', type = str, default = None, required=True,
                          help = 'path of DESI fiberflat fits file')
      parser.add_argument('--use-sky-fibers', action = 'store_true',
                          help = 'use sky fibers to improve the correction')
      parser.add_argument('-o','--outfile', type = str, default = None, required=True,
                          help = 'path of output fiberflar file')
      args = None
      if options is None:
          args = parser.parse_args()
      else:
          args = parser.parse_args(options)
      return args
  
  
  def main(args) :
  
      log = get_logger()
  
      # just read frame header in case we don't need to do anything
      frame_header = fitsio.read_header(args.infile,"FLUX")
  
      if args.use_sky_fibers :
          # need full frame to adjust correction on data
          frame = read_frame(args.infile)
      else :
          frame = None
  
      cfinder = CalibFinder([frame_header])
      if not cfinder.haskey("FIBERFLATVSHUMIDITY"):
          log.info("No information on fiberflat vs humidity for camera {}, simply link the input fiberflat".format(frame_header["CAMERA"]))
          if not os.path.islink(args.outfile) :
              relpath=os.path.relpath(args.fiberflat,os.path.dirname(args.outfile))
              os.symlink(relpath,args.outfile)
          return 0
  
      # read fiberflat
      calib_fiberflat = read_fiberflat(args.fiberflat)
  
      # read mean fiberflat vs humidity
      filename = cfinder.findfile("FIBERFLATVSHUMIDITY")
      log.info(f"reading {filename}")
      mean_fiberflat_vs_humidity , humidity_array, ffh_wave, ffh_header = read_fiberflat_vs_humidity(filename)
      assert(np.allclose(calib_fiberflat.wave,ffh_wave))
  
      # now need to find the humidity for this frame and for this fiberflat
      night=frame_header["NIGHT"]
      camera=frame_header["CAMERA"]
      current_frame_humidity =get_humidity(night=night,expid=frame_header["EXPID"],camera=camera)
      log.info("humidity during current exposure={:.2f}".format(current_frame_humidity))
  
  
  
      # we can compute the correction now that we have everything in hand
      improved_fiberflat = compute_humidity_corrected_fiberflat(calib_fiberflat, mean_fiberflat_vs_humidity , humidity_array, current_frame_humidity, frame = frame)
  
      # add telemetry humidity for the dome flats for the record
      # try to read the night exposure table to get the list of flats
      first_expid = calib_fiberflat.header["EXPID"]
      calib_night = calib_fiberflat.header["NIGHT"]
      calib_humidity=[ get_humidity(calib_night,first_expid,camera) ]
      fiberflat_expid=[ first_expid]
      for expid in range(first_expid+1,first_expid+40) :
          filename=findfile("raw",calib_night,expid)
          if not os.path.isfile(filename): continue
          head=fitsio.read_header(filename,1)
          if not "OBSTYPE" in head.keys() or head["OBSTYPE"]!="FLAT" :
              break
          fiberflat_expid.append(expid)
          calib_humidity.append(get_humidity(calib_night,expid,camera))
      log.debug("calib expids={}".format(fiberflat_expid))
      log.debug("calib humidities={}".format(calib_humidity))
      calib_humidity=np.mean(calib_humidity)
      if np.isnan(calib_humidity) :
          log.warning("missing humidity info for fiber flat, use link to input")
          calib_humidity=0.
      else :
          log.info("mean humidity during calibration exposures={:.2f}".format(calib_humidity))
          fit_humidity = improved_fiberflat.header["CALFHUM"]
          if np.abs(fit_humidity-calib_humidity)>10 :
              message="large difference between best fit humidity during dome flats ({:.1f}) and value from telemetry ({:.1f})".format(fit_humidity,calib_humidity)
              if np.abs(fit_humidity-calib_humidity)>20 :
                  log.error(message)
                  raise RuntimeError(message)
              log.warning(message)
  
      improved_fiberflat.header["CALTHUM"] = (calib_humidity,"dome flat humidity from telemetry")
  
      # write it
      write_fiberflat(args.outfile,improved_fiberflat)
      log.info("wrote humidity corrected flat {}".format(args.outfile))
  
      return 0
   """ 
  Monitoring algorithms for Quicklook pipeline
  """
  
  import os,sys
  import datetime
  import numpy as np
  import scipy.ndimage
  import yaml
  import re
  import astropy.io.fits as fits
  import desispec.qa.qa_plots_ql as plot
  import desispec.quicklook.qlpsf
  import desispec.qa.qa_plots_ql as fig
  from desispec.quicklook.qas import MonitoringAlg, QASeverity
  from desispec.quicklook import qlexceptions
  from desispec.quicklook import qllogger
  from desispec.quicklook.palib import resample_spec
  from astropy.time import Time
  from desispec.qa import qalib
  from desispec.io import qa, read_params
  from desispec.io.meta import findfile
  from desispec.io.sky import read_sky
  from desispec.image import Image as im
  from desispec.frame import Frame as fr
  from desispec.preproc import parse_sec_keyword
  from desispec.util import runcmd
  from desispec.qproc.qframe import QFrame
  from desispec.fluxcalibration import isStdStar
  from desitarget.targetmask import desi_mask
  import astropy
  from astropy.io import fits
  
  qlog=qllogger.QLLogger("QuickLook",0)
  log=qlog.getlog()
  
  def get_inputs(*args,**kwargs):
      '''
      Get inputs required for each QA
      '''
      inputs={}
      inputs["camera"]=kwargs["camera"]
  
      if "paname" not in kwargs: inputs["paname"]=None
      else: inputs["paname"]=kwargs["paname"]
  
      if "ReferenceMetrics" in kwargs: inputs["refmetrics"]=kwargs["ReferenceMetrics"]
      else: inputs["refmetrics"]=None
  
      inputs["amps"]=False
      if "amps" in kwargs: inputs["amps"]=kwargs["amps"]
  
      if "param" in kwargs: inputs["param"]=kwargs["param"]
      else: inputs["param"]=None
  
      inputs["psf"]=None
      if "PSFFile" in kwargs: inputs["psf"]=kwargs["PSFFile"]
  
      inputs["fibermap"]=None
      if "FiberMap" in kwargs: inputs["fibermap"]=kwargs["FiberMap"]
  
      if "Peaks" in kwargs: inputs["Peaks"]=kwargs["Peaks"]
  
      if "qafile" in kwargs: inputs["qafile"] = kwargs["qafile"]
      else: inputs["qafile"]=None
  
      if "qafig" in kwargs: inputs["qafig"]=kwargs["qafig"]
      else: inputs["qafig"]=None
  
      if "plotconf" in kwargs: inputs["plotconf"]=kwargs["plotconf"]
      else: inputs["plotconf"]=None
  
      if "hardplots" in kwargs: inputs["hardplots"]=kwargs["hardplots"]
      else: inputs["hardplots"]=False
  
      return inputs
  
  def get_image(filetype,night,expid,camera,specdir):
      '''
      Make image object from file if in development mode
      '''
      #- Find correct file for QA
      imagefile = findfile(filetype,int(night),int(expid),camera,specprod_dir=specdir)
  
      #- Create necessary input for desispec.image
      image = fits.open(imagefile)
      pix = image['IMAGE'].data
      ivar = image['IVAR'].data
      mask = image['MASK'].data
      readnoise = image['READNOISE'].data
      meta = image['IMAGE'].header
  
      #- Create image object
      imageobj = im(pix,ivar,mask=mask,readnoise=readnoise,camera=camera,meta=meta)
      return imageobj
  
  def get_frame(filetype,night,expid,camera,specdir):
      '''
      Make frame object from file if in development mode
      '''
      #- Find correct file for QA
      framefile = findfile(filetype,int(night),int(expid),camera,specprod_dir=specdir)
  
      #- Create necessary input for desispec.frame
      frame = fits.open(framefile)
      wave = frame['WAVE'].data
      flux = frame['FLUX'].data
      ivar = frame['IVAR'].data
      fibermap = frame['FIBERMAP'].data
      fibers = fibermap['FIBER']
      meta = frame['FLUX'].header
  
      #- Create frame object
      frameobj = QFrame(wave,flux,ivar,fibers=fibers,fibermap=fibermap,meta=meta)
  
      return frameobj
  
  
  class Check_HDUs(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="CHECKHDUS"
          import astropy
          rawtype=astropy.io.fits.hdu.hdulist.HDUList
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "CHECKHDUS"
          status=kwargs['statKey'] if 'statKey' in kwargs else "CHECKHDUS_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          MonitoringAlg.__init__(self,name,rawtype,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
  
          if kwargs["singleqa"] == 'Check_HDUs':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              rawfile = findfile('raw',int(night),int(expid),camera,rawdata_dir=kwargs["rawdir"])
              raw = fits.open(rawfile)
          else: raw=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(raw,inputs)
  
      def run_qa(self,raw,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
  
          rawimage=raw[camera.upper()].data
          header=raw[camera.upper()].header
          
          retval={}
          retval["EXPID"]= '{0:08d}'.format(header["EXPID"])
          retval["CAMERA"] = camera
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["FLAVOR"] = header["FLAVOR"]
          #SE: quicklook to crash when a mismatched config file with the one in fits header
          from desispec.scripts import quicklook
   
          args=quicklook.parse()       
          ad,fl = args.config.split("qlconfig_")
          flvr = fl.split(".yaml")[0]
          #if flvr in ['darksurvey','graysurvey','brightsurvey']: flvr = 'science'
          if header["FLAVOR"] == 'science':   
             flvr = flvr.split("survey")[0]
             if (header["FLAVOR"] == flvr or header["FLAVOR"] == format(flvr.upper()) or flvr == 'test'):
                      log.info("The correct configuration file is being used!")
             else:
                      log.critical("Wrong configuration file is being used!")
                      sys.exit("Wrong configuration file! use the one for "+str(header["FLAVOR"]))
  
          elif (header["FLAVOR"] == flvr or flvr == 'test'): 
                      log.info("The correct configuration file is being used!")
          else: 
                      log.critical("Wrong configuration file is being used!")
                      sys.exit("Wrong configuration file! use the one for "+str(header["FLAVOR"]))
          
  
          if retval["FLAVOR"] == 'science':
              retval["PROGRAM"] = header["PROGRAM"]
          else:
              pass
          retval["NIGHT"] = header["NIGHT"]
          kwargs=self.config['kwargs']
          
  
          HDUstat = "NORMAL" 
          EXPNUMstat = "NORMAL"    
          
          param['EXPTIME'] = header["EXPTIME"]
  
          if camera != header["CAMERA"]:
                  log.critical("The raw FITS file is missing camera "+camera)
                  sys.exit("QuickLook Abort: CHECK THE RAW FITS FILE :"+rawfile)
                  HDUstat = 'ALARM'
          
          if header["EXPID"] != kwargs['expid'] : 
                  log.critical("The raw FITS file is missing camera "+camera)
                  sys.exit("QuickLook Abort: EXPOSURE NUMBER DOES NOT MATCH THE ONE IN THE HEADER")            
                  EXPNUMstat = "ALARM"
          
          
          
          if header["FLAVOR"] != "science" :
              
             retval["METRICS"] = {"CHECKHDUS_STATUS":HDUstat,"EXPNUM_STATUS":EXPNUMstat}
  
          else :
             retval["METRICS"] = {"CHECKHDUS_STATUS":HDUstat,"EXPNUM_STATUS":EXPNUMstat}
             param['SEEING'] = header["SEEING"]
             param['AIRMASS'] = header["AIRMASS"]
             param['PROGRAM'] = header["PROGRAM"]
             
            
          retval["PARAMS"] = param   
          
          if 'INHERIT' in header and header['INHERIT']:
              h0 = raw[0].header
              for key in h0:
                  if key not in header:
                      header[key] = h0[key]
          
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Trace_Shifts(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="XYSHIFTS"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "XYSHIFTS"
          status=kwargs['statKey'] if 'statKey' in kwargs else "XYSHIFTS_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
          if "XYSHIFTS_WARN_RANGE" in parms and "XYSHIFTS_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["XYSHIFTS_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["XYSHIFTS_NORMAL_RANGE"]),QASeverity.NORMAL)]
          MonitoringAlg.__init__(self,name,im,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Trace_Shifts':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              image = get_image('preproc',night,expid,camera,kwargs["specdir"])
          else: image=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(image,inputs)
  
      def run_qa(self,image,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          
          #- qa dictionary 
          retval={}
          retval["PANAME" ]= paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = expid = '{0:08d}'.format(image.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = image.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if image.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
              
          
          retval["NIGHT"] = night = image.meta["NIGHT"]
          
  
          if param is None:
                  log.critical("No parameter is found for this QA")
                  sys.exit("Update the configuration file for the parameters")
  
          # create xytraceset object
          
          from desispec.calibfinder import findcalibfile
          from desispec.xytraceset import XYTraceSet
          #SE: all next lines till the dashed line exist just so that we get the psf name without hardcoding any address -> there must be a better way
          rawfile = findfile('raw',int(night),int(expid),camera,rawdata_dir=os.environ["QL_SPEC_DATA"])
          hdulist=fits.open(rawfile)
          primary_header=hdulist[0].header
          camera_header =hdulist[camera].header
          hdulist.close()
          #--------------------------------------------------------
          psffile=findcalibfile([camera_header,primary_header],"PSF")
          psf=fits.open(psffile)
          xcoef=psf['XTRACE'].data
          ycoef=psf['YTRACE'].data
          wavemin=psf["XTRACE"].header["WAVEMIN"]
          wavemax=psf["XTRACE"].header["WAVEMAX"]
          npix_y=image.meta['NAXIS2']
          psftrace=XYTraceSet(xcoef,ycoef,wavemin,wavemax,npix_y=npix_y)
  
          # compute dx and dy
          from desispec.trace_shifts import compute_dx_from_cross_dispersion_profiles as compute_dx
          from desispec.trace_shifts import compute_dy_using_boxcar_extraction as compute_dy
          fibers=np.arange(500) #RS: setting nfibers to 500 for now
          ox,oy,odx,oex,of,ol=compute_dx(xcoef,ycoef,wavemin,wavemax,image,fibers=fibers)
          x_for_dy,y_for_dy,ody,ey,fiber_for_dy,wave_for_dy=compute_dy(psftrace,image,fibers)
  
          # return average shifts in x and y
          dx=np.mean(odx)
          dy=np.mean(ody)
          xyshift=np.array([dx,dy])
  
          retval["METRICS"]={"XYSHIFTS":xyshift}
          retval["PARAMS"]=param
  
          #get_outputs(qafile,qafig,retval,'plot_traceshifts')
  #        outfile = qa.write_qa_ql(qafile,retval)
  #        log.debug("Output QA data is in {}".format(outfile))
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Bias_From_Overscan(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="BIAS_OVERSCAN"
  
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "BIAS_AMP"
          status=kwargs['statKey'] if 'statKey' in kwargs else "BIAS_AMP_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "BIAS_WARN_RANGE" in parms and "BIAS_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["BIAS_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["BIAS_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe 
  
          MonitoringAlg.__init__(self,name,im,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Bias_From_Overscan':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              image = get_image('preproc',night,expid,camera,kwargs["specdir"])
          else: image=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(image,inputs)
  
      def run_qa(self,image,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          retval={}
          retval["EXPID"] = '{0:08d}'.format(image.meta["EXPID"])
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["CAMERA"] = camera
          retval["NIGHT"] = image.meta["NIGHT"]
          retval["FLAVOR"] = flavor = image.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if image.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          retval["EXPTIME"] = image.meta["EXPTIME"]
          
  
          if retval["FLAVOR"] == 'arc':
              pass
          else:
              retval["FLAVOR"] = image.meta["FLAVOR"]
          retval["NIGHT"] = image.meta["NIGHT"]
          kwargs=self.config['kwargs']
          
          #SE: this would give the desispec version stored in DEPVER07 key of the raw simulated fits file :0.16.0.dev1830
          #RS: don't look for this if not using simulated files, differences in simulated headers vs. data headers cause this to crash
          if flavor == 'science':
              param['FITS_DESISPEC_VERSION'] = image.meta['DEPVER07'] 
              import desispec
              from desispec import quicklook
              param['PROC_DESISPEC_VERSION']= desispec.__version__
              param['PROC_QuickLook_VERSION']= quicklook.__qlversion__
                    
          
          if 'INHERIT' in image.meta and image.meta['INHERIT']:
  
              h0 = image.meta
              #h0 = header
              for key in h0:
                  if key not in image.meta:
                      image.meta[key] = h0[key]
  
          #RS: look for values in simulated data, if not found try finding data values
          try:
              bias_overscan = [image.meta['OVERSCN1'],image.meta['OVERSCN2'],image.meta['OVERSCN3'],image.meta['OVERSCN4']]
          except:
              bias_overscan = [image.meta['OVERSCNA'],image.meta['OVERSCNB'],image.meta['OVERSCNC'],image.meta['OVERSCND']]
  
          bias = np.mean(bias_overscan)
  
          if param is None:
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
                  
  
          retval["PARAMS"] = param
  
          if amps:
              bias_amps=np.array(bias_overscan)
              retval["METRICS"]={'BIAS_AMP':bias_amps}
          else:
              #retval["METRICS"]={'BIAS':bias,"DIFF1SIG":diff1sig,"DIFF2SIG":diff2sig,"DIFF3SIG":diff3sig,"DATA5SIG":data5sig,"BIAS_ROW":mean_row}
              retval["METRICS"]={}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_bias_overscan(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Get_RMS(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="RMS"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "NOISE_AMP" 
          status=kwargs['statKey'] if 'statKey' in kwargs else "NOISE_AMP_STATUS" 
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
                  
          if "NOISE_WARN_RANGE" in parms and "NOISE_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["NOISE_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["NOISE_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe 
          
          MonitoringAlg.__init__(self,name,im,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
                  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Get_RMS':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              image = get_image('preproc',night,expid,camera,kwargs["specdir"])
          else: image=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(image,inputs)
  
      def run_qa(self,image,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          retval={}
          retval["EXPID"] = '{0:08d}'.format(image.meta["EXPID"])
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["CAMERA"] = camera
          retval["FLAVOR"] = flavor = image.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if flavor == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = image.meta["NIGHT"]
  
          # return rms values in rms/sqrt(exptime)
          #rmsccd=qalib.getrms(image.pix/np.sqrt(image.meta["EXPTIME"])) #- should we add dark current and/or readnoise to this as well?
          #rmsccd = np.mean([image.meta['RDNOISE1'],image.meta['RDNOISE2'],image.meta['RDNOISE3'],image.meta['RDNOISE4']]) #--> "NOISE":rmsccd
          
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")            
  
          retval["PARAMS"] = param
  
          #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
          # SE: this section is moved from BIAS_FROM_OVERSCAN to header
  
          data=[]
          row_data_amp1=[]
          row_data_amp2=[]
          row_data_amp3=[]
          row_data_amp4=[]
          bias_patnoise=[]
          #bias_overscan=[]        
          #RS: loop through amps based on header info
          loop_amps = get_amp_ids(image.meta)
          exptime=image.meta["EXPTIME"]
          if exptime == 0.:
              exptime = 1.
          for kk in loop_amps:
              sel=parse_sec_keyword(image.meta['BIASSEC'+kk])
              #- Obtain counts/second in bias region
  #            pixdata=image[sel]/header["EXPTIME"]
              pixdata=image.pix[sel]/exptime
              if kk == '1' or kk == 'A':
                  for i in range(pixdata.shape[0]):
                      row_amp1=pixdata[i]
                      row_data_amp1.append(row_amp1)
              if kk == '2' or kk == 'B':
                  
                  for i in range(pixdata.shape[0]):
                      row_amp2=pixdata[i]
                      row_data_amp2.append(row_amp2)
              if kk == '3' or kk == 'C':
                  
                  for i in range(pixdata.shape[0]):
                      row_amp3=pixdata[i]
                      row_data_amp3.append(row_amp3)
              if kk == '4' or kk == 'D':
                  
                  for i in range(pixdata.shape[0]):
                      row_amp4=pixdata[i]
                      row_data_amp4.append(row_amp4)
              #- Compute statistics of the bias region that only reject
              #  the 0.5% of smallest and largest values. (from sdssproc) 
              isort=np.sort(pixdata.ravel())
              nn=isort.shape[0]
              bias=np.mean(isort[int(0.005*nn) : int(0.995*nn)])
              #bias_overscan.append(bias)
              data.append(isort)
  
          #- Combine data from each row per amp and take average
          # BIAS_ROW = mean_row  
          median_row_amp1=[]
          for i in range(len(row_data_amp1)):
              median=np.median(row_data_amp1[i])
              median_row_amp1.append(median)
          
          rms_median_row_amp1= np.std(median_row_amp1)
          try:
              noise1 = image.meta['RDNOISE1']
          except:
              noise1 = image.meta['OBSRDNA']
          bias_patnoise.append(rms_median_row_amp1/noise1)
          
          median_row_amp2=[]
          for i in range(len(row_data_amp2)):
              median=np.median(row_data_amp2[i])
              median_row_amp2.append(median)
          
          rms_median_row_amp2= np.std(median_row_amp2)
          try:
              noise2 = image.meta['RDNOISE2']
          except:
              noise2 = image.meta['OBSRDNB']
          bias_patnoise.append(rms_median_row_amp2/noise2)
          
          
          median_row_amp3=[]
          for i in range(len(row_data_amp3)):
              median=np.median(row_data_amp3[i])
              median_row_amp3.append(median)
          
          rms_median_row_amp3= np.std(median_row_amp3)
          try:
              noise3 = image.meta['RDNOISE3']
          except:
              noise3 = image.meta['OBSRDNC']
          bias_patnoise.append(rms_median_row_amp3/noise3)
          
          median_row_amp4=[]
          for i in range(len(row_data_amp4)):
              median=np.median(row_data_amp4[i])
              median_row_amp4.append(median)
          
          rms_median_row_amp4= np.std(median_row_amp4)
          try:
              noise4 = image.meta['RDNOISE4']
          except:
              noise4 = image.meta['OBSRDND']
          bias_patnoise.append(rms_median_row_amp4/noise4)
  
  
          #- Calculate upper and lower bounds of 1, 2, and 3 sigma  
          full_data=np.concatenate((data[0],data[1],data[2],data[3])).ravel()
          sig1_lo = np.percentile(full_data,50.-(param['PERCENTILES'][0]/2.))
          sig1_hi = np.percentile(full_data,50.+(param['PERCENTILES'][0]/2.))
          sig2_lo = np.percentile(full_data,50.-(param['PERCENTILES'][1]/2.))
          sig2_hi = np.percentile(full_data,50.+(param['PERCENTILES'][1]/2.))
          sig3_lo = np.percentile(full_data,50.-(param['PERCENTILES'][2]/2.))
          sig3_hi = np.percentile(full_data,50.+(param['PERCENTILES'][2]/2.))
  
          #- Find difference between upper and lower sigma bounds
          # DIFF1SIG: The number of counts separating the 1 sigma percentiles in the noise distribution (from the overscan region)
          diff1sig = sig1_hi - sig1_lo
          # DIFF2SIG: The number of counts separating 2 or 3 sigma in the noise distribution
          diff2sig = sig2_hi - sig2_lo
          diff3sig = sig3_hi - sig3_lo
  
          #-DATA5SIG: number of pixels more than 5 sigma below the bias level
          sig5_value = np.percentile(full_data,3e-5)
          data5sig = len(np.where(full_data <= sig5_value)[0])
         
          #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  
          if amps:
              rms_over_amps = [noise1,noise2,noise3,noise4]
              try:
                  rms_amps = [image.meta['OBSRDN1'],image.meta['OBSRDN2'],image.meta['OBSRDN3'],image.meta['OBSRDN4']]
              except:
                  rms_amps = [image.meta['OBSRDNA'],image.meta['OBSRDNB'],image.meta['OBSRDNC'],image.meta['OBSRDND']]
              retval["METRICS"]={"NOISE_AMP":np.array(rms_amps),"NOISE_OVERSCAN_AMP":np.array(rms_over_amps),"DIFF1SIG":diff1sig,"DIFF2SIG":diff2sig,"DATA5SIG":data5sig,"BIAS_PATNOISE":bias_patnoise}#,"NOISE_ROW":noise_row,"EXPNUM_WARN":expnum,"NOISE_OVER":rmsover
          else:
              retval["METRICS"]={"DIFF1SIG":diff1sig,"DIFF2SIG":diff2sig,"DATA5SIG":data5sig, "BIAS_PATNOISE":bias_patnoise} # Dropping "NOISE_OVER":rmsover,"NOISE_ROW":noise_row,"EXPNUM_WARN":expnum
  
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_RMS(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval    
  
      def get_default_config(self):
          return {}
  
  
  class Calc_XWSigma(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="XWSIGMA"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "XWSIGMA"
          status=kwargs['statKey'] if 'statKey' in kwargs else "XWSIGMA_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "XWSIGMA_WARN_RANGE" in parms and "XWSIGMA_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["XWSIGMA_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["XWSIGMA_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,im,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
                  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Calc_XWSigma':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              image = get_image('preproc',night,expid,camera,kwargs["specdir"])
          else: image=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(image,inputs)
  
      def run_qa(self,image,inputs):
          import desispec.quicklook.qlpsf
          from scipy.optimize import curve_fit
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          psffile=inputs["psf"]
          psf=desispec.quicklook.qlpsf.PSF(psffile)
          amps=inputs["amps"]
          allpeaks=inputs["Peaks"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat() 
          retval["EXPID"] = '{0:08d}'.format(image.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = image.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if image.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=program=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = image.meta["NIGHT"]
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          retval["PARAMS"] = param
          #- Ensure that the QA will run even if 500 spectra aren't present
          if fibermap['FIBER'].shape[0] >= 500:
              fibers = 500
          else:
              fibers = fibermap['FIBER'].shape[0]
  
          #- Define number of pixels to be fit
          dp=param['PIXEL_RANGE']/2
          #- Get wavelength ranges around peaks
          peaks=allpeaks['{}_PEAKS'.format(camera[0].upper())]
          #- Maximum allowed fit sigma value
          maxsigma=param['MAX_SIGMA']
  
          xfails=[]
          wfails=[]
          xsigma=[]
          wsigma=[]
          xsigma_amp1=[]
          wsigma_amp1=[]
          xsigma_amp2=[]
          wsigma_amp2=[]
          xsigma_amp3=[]
          wsigma_amp3=[]
          xsigma_amp4=[]
          wsigma_amp4=[]
          
          for fiber in range(fibers):
              
              xs = -1 # SE: this prevents crash in "XWSIGMA_AMP" for when xs or ws is empty list -> try b9 of 20200515/00000001  
              ws = -1
              xsig=[]
              wsig=[]
              for peak in range(len(peaks)):
                  #- Use psf information to convert wavelength to pixel values
                  xpixel=desispec.quicklook.qlpsf.PSF.x(psf,ispec=fiber,wavelength=peaks[peak])[0][0]
                  ypixel=desispec.quicklook.qlpsf.PSF.y(psf,ispec=fiber,wavelength=peaks[peak])[0][0]
                  #- Find x and y pixel values around sky lines
                  xpix_peak=np.arange(int(xpixel-dp),int(xpixel+dp),1)
                  ypix_peak=np.arange(int(ypixel-dp),int(ypixel+dp),1)
                  #- Fit gaussian to counts in pixels around sky line
                  #- If any values fail, store x/w, wavelength, and fiber
                  try:
                      xpopt,xpcov=curve_fit(qalib.gauss,np.arange(len(xpix_peak)),image.pix[int(ypixel),xpix_peak])
                      xs=np.abs(xpopt[2])
                      if xs <= maxsigma:
                          xsig.append(xs)
                      else:
                          xfail=[fiber,peaks[peak]]
                          xfails.append(xfail)
                  except:
                      xfail=[fiber,peaks[peak]]
                      xfails.append(xfail)
                      pass
                  try:
                      wpopt,wpcov=curve_fit(qalib.gauss,np.arange(len(ypix_peak)),image.pix[ypix_peak,int(xpixel)])
                      ws=np.abs(wpopt[2])
                      if ws <= maxsigma:
                          wsig.append(ws)
                      else:
                          wfail=[fiber,peaks[peak]]
                          wfails.append(wfail)
                  except:
                      wfail=[fiber,peaks[peak]]
                      wfails.append(wfail)
                      pass
  
                  #- Excluding fibers 240-260 in case some fibers overlap amps
                  #- Excluding peaks in the center of image in case peak overlaps two amps
                  #- This shouldn't cause a significant loss of information 
                  
                  if amps:
  
                      if fibermap['FIBER'][fiber]<240:
                          if ypixel < 2000.:
                              xsigma_amp1.append(xs)
                              wsigma_amp1.append(ws)
                          if ypixel > 2100.:
                              xsigma_amp3.append(xs)
                              wsigma_amp3.append(ws)
  
                      if fibermap['FIBER'][fiber]>260:
                          if ypixel < 2000.:
                              xsigma_amp2.append(xs)
                              wsigma_amp2.append(ws)
                          if ypixel > 2100.:
                              xsigma_amp4.append(xs)
                              wsigma_amp4.append(ws)
                      
  
              if len(xsig)!=0:
                  xsigma.append(np.mean(xsig))
              if len(wsig)!=0:
                  wsigma.append(np.mean(wsig))
  
          if fibermap['FIBER'].shape[0]<260:
              xsigma_amp2=[]
              xsigma_amp4=[]
              wsigma_amp2=[]
              wsigma_amp4=[]
  
          #- Calculate desired output metrics 
          xsigma_med=np.median(np.array(xsigma))
          wsigma_med=np.median(np.array(wsigma))
          xsigma_amp=np.array([np.median(xsigma_amp1),np.median(xsigma_amp2),np.median(xsigma_amp3),np.median(xsigma_amp4)])
          wsigma_amp=np.array([np.median(wsigma_amp1),np.median(wsigma_amp2),np.median(wsigma_amp3),np.median(wsigma_amp4)])
          xwfails=np.array([xfails,wfails])
  
  
          #SE: mention the example here when the next lines are ineffective and when they are effective in removing the NaN from XWSIGMA_AMP--> XWSIGMA itself no longer includes any NaN value. As we both know, this is not the way to properly deal with NaNs -->let's see if switching to non-scipy fuction would bring about a better solution
          if len(xsigma)==0:
              xsigma=[param['XWSIGMA_{}_REF'.format(program.upper())][0]]
  
          if len(wsigma)==0:
              wsigma=[param['XWSIGMA_{}_REF'.format(program.upper())][1]]
  
          #- Combine metrics for x and w
          xwsigma_fib=np.array((xsigma,wsigma)) #- (2,nfib)
          xwsigma_med=np.array((xsigma_med,wsigma_med)) #- (2)
          xwsigma_amp=np.array((xsigma_amp,wsigma_amp))
  
          if amps:
              #if len(xsigma_amp1)==0 :
                  #xsigma_amp1 = [param['XWSIGMA_REF'][0]]
              #if len(xsigma_amp2)==0 :
                  #xsigma_amp2 = [param['XWSIGMA_REF'][0]]
              #if len(xsigma_amp3)==0 :
                  #xsigma_amp3 = [param['XWSIGMA_REF'][0]]
              #if len(xsigma_amp4)==0 :
                  #xsigma_amp4 = [param['XWSIGMA_REF'][0]]
  
              #if len(wsigma_amp1)==0 :
                  #wsigma_amp1 = [param['XWSIGMA_REF'][1]]
              #if len(wsigma_amp2)==0 :
                  #wsigma_amp2 = [param['XWSIGMA_REF'][1]]
              #if len(wsigma_amp3)==0 :
                  #wsigma_amp3 = [param['XWSIGMA_REF'][1]]
              #if len(wsigma_amp4)==0 :
                  #wsigma_amp4 = [param['XWSIGMA_REF'][1]]
  
              retval["METRICS"]={"XWSIGMA":xwsigma_med,"XWSIGMA_FIB":xwsigma_fib,"XWSIGMA_AMP":xwsigma_amp}#,"XWSHIFT":xwshift,"XWSHIFT_AMP":xwshift_amp,"XWSIGMA_SHIFT": xwsigma_shift}
          else:
              retval["METRICS"]={"XWSIGMA":xwsigma_med,"XWSIGMA_FIB":xwsigma_fib}#,"XWSHIFT":xwshift,"XWSIGMA_SHIFT": xwsigma_shift}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_XWSigma(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
   
      def get_default_config(self):
          return {}
  
  
  class Count_Pixels(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="COUNTPIX"
          from desispec.image import Image as im
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "LITFRAC_AMP"
          status=kwargs['statKey'] if 'statKey' in kwargs else "LITFRAC_AMP_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
                  
          if "LITFRAC_AMP_WARN_RANGE" in parms and "LITFRAC_AMP_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["LITFRAC_AMP_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["LITFRAC_AMP_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,im,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
                  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Count_Pixels':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              image = get_image('preproc',night,expid,camera,kwargs["specdir"])
          else: image=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(image,inputs)
  
      def run_qa(self,image,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(image.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = image.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if image.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          
          retval["NIGHT"] = image.meta["NIGHT"]
          
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
  
  
          retval["PARAMS"] = param
  
          #- get the counts for each amp
          npix_amps=[]
          litfrac_amps=[]
  
          from desispec.preproc import parse_sec_keyword
          #RS: loop through amps based on header info
          try:
              header_test=parse_sec_keyword(image.meta['CCDSEC1'])
              loop_amps=['1','2','3','4']
          except:
              loop_amps=['A','B','C','D']
          #- get amp boundary in pixels
          for kk in loop_amps:
              ampboundary=parse_sec_keyword(image.meta["CCDSEC"+kk])
              try:
                  rdnoise_thisamp=image.meta["RDNOISE"+kk]
              except:
                  rdnoise_thisamp=image.meta["OBSRDN"+kk]
              npix_thisamp= image.pix[ampboundary][image.pix[ampboundary] > param['CUTPIX'] * rdnoise_thisamp].size #- no of pixels above threshold
              npix_amps.append(npix_thisamp)
              size_thisamp=image.pix[ampboundary].size
              litfrac_thisamp=round(np.float64(npix_thisamp)/size_thisamp,2) #- fraction of pixels getting light above threshold
              litfrac_amps.append(litfrac_thisamp)
  	#        retval["METRICS"]={"NPIX_AMP",npix_amps,'LITFRAC_AMP': litfrac_amps}
          retval["METRICS"]={"LITFRAC_AMP": litfrac_amps}	
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_countpix(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class CountSpectralBins(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="COUNTBINS"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "NGOODFIB"
          status=kwargs['statKey'] if 'statKey' in kwargs else "NGOODFIB_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "NGOODFIB_WARN_RANGE" in parms and "NGOODFIB_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["NGOODFIB_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["NGOODFIB_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe 
  
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'CountSpectralBins':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('frame',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          psf=inputs["psf"]
          qafile=inputs["qafile"]
          qafig=None #inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          #- qa dictionary 
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = frame.meta["NIGHT"]
  
          grid=np.gradient(frame.wave)
          if not np.all(grid[0]==grid[1:]): 
              log.debug("grid_size is NOT UNIFORM")
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
  
          retval["PARAMS"] = param
          #- get the effective readnoise for the fibers 
          #- readnoise per fib = readnoise per pix * sqrt(box car width)* sqrt(no. of bins in the amp) * binsize/pix size scale
          nspec=frame.nspec
          rdnoise_fib=np.zeros(nspec)
          if nspec > 250: #- upto 250 - amp 1 and 3, beyond that 2 and 4
              rdnoise_fib[:250]=[(frame.meta['RDNOISE1']+frame.meta['RDNOISE3'])*np.sqrt(5.)*np.sqrt(frame.flux.shape[1]/2)*frame.meta['WAVESTEP']/0.5]*250
              rdnoise_fib[250:]=[(frame.meta['RDNOISE2']+frame.meta['RDNOISE4'])*np.sqrt(5.)*np.sqrt(frame.flux.shape[1]/2)*frame.meta['WAVESTEP']/0.5]*(nspec-250)
          else:
              rdnoise_fib=[(frame.meta['RDNOISE1']+frame.meta['RDNOISE3'])*np.sqrt(5.)*np.sqrt(frame.flux.shape[1]/2)*frame.meta['WAVESTEP']/0.5]*nspec
          threshold=[param['CUTBINS']*ii for ii in rdnoise_fib]
          #- compare the flux sum to threshold
          
          totcounts=frame.flux.sum(axis=1)
          passfibers=np.where(totcounts>threshold)[0] 
          ngoodfibers=passfibers.shape[0]
          good_fibers=np.array([0]*frame.nspec)
          good_fibers[passfibers]=1 #- assign 1 for good fiber
  
          #- leaving the amps granularity needed for caching as defunct. If needed in future, this needs to be propagated through.
          amps=False
          leftmax=None
          rightmax=None
          bottommax=None
          topmin=None
  
          if amps: #- leaving this for now
              leftmax,rightmin,bottommax,topmin = qalib.fiducialregion(frame,psf)
              retval["LEFT_MAX_FIBER"]=int(leftmax)
              retval["RIGHT_MIN_FIBER"]=int(rightmin)
              retval["BOTTOM_MAX_WAVE_INDEX"]=int(bottommax)
              retval["TOP_MIN_WAVE_INDEX"]=int(topmin)
  
          retval["METRICS"]={"NGOODFIB": ngoodfibers, "GOOD_FIBERS": good_fibers, "TOTCOUNT_FIB": totcounts}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_countspectralbins(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Sky_Continuum(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="SKYCONT"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "SKYCONT"
          status=kwargs['statKey'] if 'statKey' in kwargs else "SKYCONT_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "SKYCONT_WARN_RANGE" in parms and "SKYCONT_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["SKYCONT_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["SKYCONT_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Sky_Continuum':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('fframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          #- qa dictionary 
          retval={}
          retval["PANAME" ]= paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = frame.meta["NIGHT"]
  
          camera=frame.meta["CAMERA"]
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
   
  
          wrange1=param["{}_CONT".format(camera[0].upper())][0]
          wrange2=param["{}_CONT".format(camera[0].upper())][1]
  
          retval["PARAMS"] = param
  
          skyfiber, contfiberlow, contfiberhigh, meancontfiber, skycont = qalib.sky_continuum(
              frame, wrange1, wrange2)
   
                              
          retval["METRICS"]={"SKYFIBERID": skyfiber.tolist(), "SKYCONT":skycont, "SKYCONT_FIBER":meancontfiber}
               
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_sky_continuum(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Sky_Rband(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="SKYRBAND"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "SKYRBAND"
          status=kwargs['statKey'] if 'statKey' in kwargs else "SKYRBAND_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "SKYRBAND_WARN_RANGE" in parms and "SKYRBAND_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["SKYRBAND_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["SKYRBAND_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is found for this QA")
              sys.exit("Update the configuration file for the parameters")
  
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Sky_Rband':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('cframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          #- qa dictionary 
          retval={}
          retval["NIGHT"] = frame.meta["NIGHT"]
          retval["PANAME" ]= paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
  
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=program=fibmap[1].header['PROGRAM']
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
  
          retval["PARAMS"] = param
  
          #- Find sky fibers
          objects=frame.fibermap['OBJTYPE']
          skyfibers=np.where(objects=="SKY")[0]
  
          flux=frame.flux
          wave=frame.wave
          #- Set appropriate filter and zero point
          if camera[0].lower() == 'r':
              responsefilter='decam2014-r'
  
              #- Get filter response information from speclite
              try:
                  from pkg_resources import resource_filename
                  responsefile=resource_filename('speclite','data/filters/{}.ecsv'.format(responsefilter))
                  #- Grab wavelength and response information from file
                  rfile=np.genfromtxt(responsefile)
                  rfile=rfile[1:] # remove wavelength/response labels
                  rwave=np.zeros(rfile.shape[0])
                  response=np.zeros(rfile.shape[0])
                  for i in range(rfile.shape[0]):
                      rwave[i]=10.*rfile[i][0] # convert to angstroms
                      response[i]=rfile[i][1]
              except:
                  log.critical("Could not find filter response file, can't compute spectral magnitudes")
  
              #- Convole flux with response information 
              res=np.zeros(frame.wave.shape)
              for w in range(response.shape[0]):
                  if w >= 1 and w<= response.shape[0]-2:
                      ind=np.abs(frame.wave-rwave[w]).argmin()
                      lo=(rwave[w]-rwave[w-1])/2
                      wlo=rwave[w]-lo
                      indlo=np.abs(frame.wave-wlo).argmin()
                      hi=(rwave[w+1]-rwave[w])/2
                      whi=rwave[w]+hi
                      indhi=np.abs(frame.wave-whi).argmin()
                      res[indlo:indhi]=response[w]
              skyrflux=res*flux[skyfibers]
  
              #- Calculate integrals for sky fibers
              integrals=[]
              for ii in range(len(skyrflux)):
                  integrals.append(qalib.integrate_spec(frame.wave,skyrflux[ii]))
              integrals=np.array(integrals)
  
              #- Convert calibrated flux to fiber magnitude
              specmags=np.zeros(integrals.shape)
              specmags[integrals>0]=21.1-2.5*np.log10(integrals[integrals>0]/frame.meta["EXPTIME"])
              avg_skyrband=np.mean(specmags[specmags>0])
  
              retval["METRICS"]={"SKYRBAND_FIB":specmags,"SKYRBAND":avg_skyrband}
  
          #- If not in r channel, set reference and metrics to zero
          else:
              retval["PARAMS"]["SKYRBAND_{}_REF".format(program.upper())]=[0.]
              zerospec=np.zeros_like(skyfibers)
              zerorband=0.
              retval["METRICS"]={"SKYRBAND_FIB":zerospec,"SKYRBAND":zerorband}
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Sky_Peaks(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="PEAKCOUNT"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "PEAKCOUNT"
          status=kwargs['statKey'] if 'statKey' in kwargs else "PEAKCOUNT_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "PEAKCOUNT_WARN_RANGE" in parms and "PEAKCOUNT_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["PEAKCOUNT_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["PEAKCOUNT_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Sky_Peaks':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('fframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          from desispec.qa.qalib import sky_peaks
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          allpeaks=inputs["Peaks"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = frame.meta["NIGHT"]
  
          # Parameters
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          param['B_PEAKS']=allpeaks['B_PEAKS']
          param['R_PEAKS']=allpeaks['R_PEAKS']
          param['Z_PEAKS']=allpeaks['Z_PEAKS']
  
          #nspec_counts, sky_counts, tgt_counts, tgt_counts_rms = sky_peaks(param, frame)
          nspec_counts, sky_counts, skyfibers, nskyfib= sky_peaks(param, frame)
          rms_nspec = np.std(nspec_counts)#qalib.getrms(nspec_counts)
          rms_skyspec = np.std(sky_counts)#qalib.getrms(sky_counts)  
          
          sumcount_med_sky=np.median(sky_counts)
  
          retval["PARAMS"] = param
  
          fiberid=frame.fibermap['FIBER']
  
          retval["METRICS"]={"FIBERID":fiberid,"PEAKCOUNT":sumcount_med_sky,"PEAKCOUNT_NOISE":rms_skyspec,"PEAKCOUNT_FIB":nspec_counts,"SKYFIBERID":skyfibers, "NSKY_FIB":nskyfib}#,"PEAKCOUNT_TGT":tgt_counts,"PEAKCOUNT_TGT_NOISE":tgt_counts_rms}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_sky_peaks(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Sky_Residual(MonitoringAlg):
      """ 
      Use offline sky_residual function to calculate sky residuals
      """
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="RESIDUAL"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "RESIDNOISE"
          status=kwargs['statKey'] if 'statKey' in kwargs else "RESID_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "RESID_WARN_RANGE" in parms and "RESID_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["RESID_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["RESID_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe 
  
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Sky_Residual':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('sframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
          skymodel=args[1]
  
          return self.run_qa(frame,skymodel,inputs)
  
      def run_qa(self,frame,skymodel,inputs):
          from desispec.sky import qa_skysub
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
              
          if skymodel is None:
              raise IOError("Must have skymodel to find residual. It can't be None")
          #- return values
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=fibmap[1].header['PROGRAM']
         
          retval["NIGHT"] = frame.meta["NIGHT"]
          
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          retval["PARAMS"] = param
  
          qadict=qalib.sky_resid(param,frame,skymodel,quick_look=True)
  
          retval["METRICS"] = {}
          for key in qadict.keys():
              retval["METRICS"][key] = qadict[key]
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_residuals(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  
  class Integrate_Spec(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="INTEG"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "DELTAMAG_TGT"
          status=kwargs['statKey'] if 'statKey' in kwargs else "DELTAMAG_TGT_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "DELTAMAG_WARN_RANGE" in parms and "DELTAMAG_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["DELTAMAG_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["DELTAMAG_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Integrate_Spec':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('cframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          flux=frame.flux
          ivar=frame.ivar
          wave=frame.wave
  
          retval={}
          retval["PANAME" ] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["NIGHT"] = frame.meta["NIGHT"]
          retval["EXPID"] = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=program=fibmap[1].header['PROGRAM']
  
          retval["NIGHT"] = frame.meta["NIGHT"]
  
          flux=frame.flux
          wave=frame.wave
          #- Grab magnitudes for appropriate filter
          if camera[0].lower() == 'b':
              band = 'G'
              responsefilter='decam2014-g'
          elif camera[0].lower() == 'r':
              band = 'R'
              responsefilter='decam2014-r'
          elif camera[0].lower() == 'z':
              band = 'Z'
              responsefilter='decam2014-z'
          else:
              raise ValueError("Camera {} not in b, r, or z channels...".format(camera))
  
          #- Find fibers per target type
          elgfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.ELG) != 0)[0]
          lrgfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.LRG) != 0)[0]
          qsofibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.QSO) != 0)[0]
          bgsfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.BGS_ANY) != 0)[0]
          mwsfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.MWS_ANY) != 0)[0]
          stdfibers = np.where(isStdStar(frame.fibermap))[0]
          skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
  
          #- Setup target fibers per program
          if program == 'dark':
              objfibers = [elgfibers,lrgfibers,qsofibers,stdfibers]
          elif program == 'gray':
              objfibers = [elgfibers,stdfibers]
          elif program == 'bright':
              objfibers = [bgsfibers,mwsfibers,stdfibers]
  
          magnitudes=np.zeros(frame.nspec)
          key = 'FLUX_'+band
          magnitudes = 22.5 - 2.5*np.log10(frame.fibermap[key])
          #- Set objects with zero flux to 30 mag
          zeroflux = np.where(frame.fibermap[key]==0.)[0]
          magnitudes[zeroflux] = 30.
  
          #- Get filter response information from speclite
          try:
              from pkg_resources import resource_filename
              responsefile=resource_filename('speclite','data/filters/{}.ecsv'.format(responsefilter))
              #- Grab wavelength and response information from file
              rfile=np.genfromtxt(responsefile)
              rfile=rfile[1:] # remove wavelength/response labels
              rwave=np.zeros(rfile.shape[0])
              response=np.zeros(rfile.shape[0])
              for i in range(rfile.shape[0]):
                  rwave[i]=10.*rfile[i][0] # convert to angstroms
                  response[i]=rfile[i][1]
          except:
              log.critical("Could not find filter response file, can't compute spectral magnitudes")
  
          #- Convole flux with response information 
          res=np.zeros(frame.wave.shape)
          for w in range(response.shape[0]):
              if w >= 1 and w<= response.shape[0]-2:
                  ind=np.abs(frame.wave-rwave[w]).argmin()
                  lo=(rwave[w]-rwave[w-1])/2
                  wlo=rwave[w]-lo
                  indlo=np.abs(frame.wave-wlo).argmin()
                  hi=(rwave[w+1]-rwave[w])/2
                  whi=rwave[w]+hi
                  indhi=np.abs(frame.wave-whi).argmin()
                  res[indlo:indhi]=response[w]
          rflux=res*flux
  
          #- Calculate integrals for all fibers
          integrals=[]
          for ii in range(len(rflux)):
              integrals.append(qalib.integrate_spec(frame.wave,rflux[ii]))
          integrals=np.array(integrals)
  
          #- Convert calibrated flux to spectral magnitude
          specmags=np.zeros(integrals.shape)
          specmags[integrals>0]=21.1-2.5*np.log10(integrals[integrals>0]/frame.meta["EXPTIME"])
  
          #- Save number of negative flux fibers
          negflux=np.where(specmags==0.)[0]
          num_negflux=len(negflux)
  
          #- Set sky and negative flux fibers to 30 mag
          specmags[skyfibers]=30.
          specmags[negflux]=30.
  
          #- Calculate integrals for each target type
          tgt_specmags=[]
          for T in objfibers:
              if num_negflux != 0:
                  T=np.array(list(set(T) - set(negflux)))
              obj_integ=[]
              for ii in range(len(rflux[T])):
                  obj_integ.append(qalib.integrate_spec(frame.wave,rflux[T][ii]))
              obj_integ = np.array(obj_integ)
  
              #- Convert calibrated flux to spectral magnitude per terget type
              #- Using ST magnitude system because frame flux is in units ergs/s/cm**2/A
              obj_specmags = np.zeros(obj_integ.shape)
              obj_specmags[obj_integ>0] = 21.1-2.5*np.log10(obj_integ[obj_integ>0]/frame.meta["EXPTIME"])
              tgt_specmags.append(obj_specmags)
  
          tgt_specmags = np.array(tgt_specmags)
  
          #- Fiber magnitudes per target type
          tgt_mags=[]
          for obj in objfibers:
              if num_negflux != 0:
                  obj=np.array(list(set(obj) - set(negflux)))
              tgt_mags.append(magnitudes[obj])
  
          tgt_mags = np.array(tgt_mags)
  
          #- Calculate delta mag, remove sky/negative flux fibers first
          remove_fib = np.array(list(set(skyfibers) | set(negflux)))
          nosky_specmags = np.delete(specmags,remove_fib)
          nosky_mags = np.delete(magnitudes,remove_fib)
          deltamag = nosky_specmags - nosky_mags
  
          #- Calculate avg delta mag per target type
          deltamag_tgt = tgt_specmags - tgt_mags        
          deltamag_tgt_avg=[]
          for tgt in range(len(deltamag_tgt)):
              deltamag_tgt_avg.append(np.mean(deltamag_tgt[tgt]))
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
  
          retval["PARAMS"] = param
  
          fiberid=frame.fibermap['FIBER']
  
          #SE: should not have any nan or inf at this point but let's keep it for safety measures here 
          retval["METRICS"]={"FIBERID":fiberid,"NFIBNOTGT":num_negflux,"SPEC_MAGS":specmags, "DELTAMAG":np.nan_to_num(deltamag), "STD_FIBERID":stdfibers, "DELTAMAG_TGT":np.nan_to_num(deltamag_tgt_avg)}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_integral(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval    
  
      def get_default_config(self):
          return {}
   
  class Calculate_SNR(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="SNR"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "FIDSNR_TGT"
          status=kwargs['statKey'] if 'statKey' in kwargs else "FIDSNR_TGT_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "FIDSNR_TGT_WARN_RANGE" in parms and "FIDSNR_TGT_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["FIDSNR_TGT_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["FIDSNR_TGT_NORMAL_RANGE"]),QASeverity.NORMAL)]# sorted by most severe to least severe
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Calculate_SNR':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              frame = get_frame('sframe',night,expid,camera,kwargs["specdir"])
          else: frame=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(frame,inputs)
  
      def run_qa(self,frame,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          if isinstance(frame,QFrame):
              frame = frame.asframe()
  
          #- return values
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = expid = '{0:08d}'.format(frame.meta["EXPID"])
          retval["CAMERA"] = camera
          retval["FLAVOR"] = frame.meta["FLAVOR"]
          kwargs=self.config['kwargs']
          
          if frame.meta["FLAVOR"] == 'science':
              fibmap =fits.open(kwargs['FiberMap'])
              retval["PROGRAM"]=program=fibmap[1].header['PROGRAM']
  
          objlist=[]
          if program == 'dark':
              objlist = ['ELG','LRG','QSO','STAR']
          elif program == 'gray':
              objlist = ['ELG','STAR']
          elif program == 'bright':
              objlist = ['BGS','MWS','STAR']
  
          retval["NIGHT"] = night = frame.meta["NIGHT"]
  
          ra = fibermap["TARGET_RA"]
          dec = fibermap["TARGET_DEC"]
  
          #- select band for mag, using DECAM_R if present
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
  
          fidboundary=None
  
          qadict,fitsnr = qalib.orig_SNRFit(frame,night,camera,expid,param,fidboundary=fidboundary)
  
          #- Check for inf and nans in missing magnitudes for json support of QLF #TODO review this later
  
          for obj in range(len(qadict["SNR_MAG_TGT"])):
              for mag in [qadict["SNR_MAG_TGT"][obj]]:
                  k=np.where(~np.isfinite(mag))[0]
                  if len(k) > 0:
                      log.warning("{} objects have no or unphysical magnitudes".format(len(k)))
              mag=np.array(mag)
              mag[k]=26.  #- Putting 26, so as to make sure within reasonable range for plots.
          retval["METRICS"] = qadict
          retval["PARAMS"] = param
  
          rescut=param["RESIDUAL_CUT"]
          sigmacut=param["SIGMA_CUT"]
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_SNR(retval,qafig,objlist,fitsnr,rescut,sigmacut,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  class Check_Resolution(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="CHECKARC"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "CHECKARC"
          status=kwargs['statKey'] if 'statKey' in kwargs else "CHECKARC_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "CHECKARC_WARN_RANGE" in parms and "CHECKARC_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["CHECKARC_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["CHECKARC_NORMAL_RANGE"]),QASeverity.NORMAL)]
  
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Check_Resolution':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
              #- Finding psf file for QA
              #file_psf = get_psf('psf',night,expid,camera,kwargs["specdir"])
          else: file_psf = args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(file_psf,inputs)
  
      def run_qa(self,file_psf,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          plotconf=inputs["plotconf"]
          hardplots=inputs["hardplots"]
  
          retval={}
          retval['PANAME'] = paname
          kwargs=self.config['kwargs']
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["EXPID"] = '{:08d}'.format(kwargs['expid'])
          retval["CAMERA"] = camera
          retval["PROGRAM"] = 'ARC'
          retval["FLAVOR"] = 'arc'
          retval["NIGHT"] = kwargs['night']
   
  
          # file_psf.ycoeff is not the wsigma_array.
          # FIX later.TEST QA with file_psf.ycoeff
          
          wsigma_array = file_psf.ysig_vs_wave_traceset._coeff
          p0 = wsigma_array[0:, 0:1]
          p1 = wsigma_array[0:, 1:2]
          p2 = wsigma_array[0:, 2:3]
  
          #- Save array of ones and zeros for good/no fits
          nfib = len(p0)
          nofit = np.where(p0 == 0.)[0]
          allfibs=np.ones(nfib)
          allfibs[nofit] = 0.
          #- Total number of fibers fit used as scalar metric
          ngoodfits = len(np.where(allfibs == 1.)[0])
  
          # Medians of Legendre Coeffs to be used as 'Model'
          medlegpolcoef = np.median(wsigma_array,axis = 0)
  
          wsigma_rms = np.sqrt(np.mean((wsigma_array - medlegpolcoef)**2,axis = 0))
  
          # Check how many of each parameter are outside of +- 2 RMS of the median.
          toperror = np.array([medlegpolcoef[val] + 2*wsigma_rms[val] for val in [0,1,2]])
          bottomerror = np.array([medlegpolcoef[val] - 2*wsigma_rms[val] for val in [0,1,2]])
  
          badparamrnum0 = list(np.where(np.logical_or(p0>toperror[0], p0<bottomerror[0]))[0])
          badparamrnum1 = list(np.where(np.logical_or(p1>toperror[1], p1<bottomerror[1]))[0])
          badparamrnum2 = list(np.where(np.logical_or(p2>toperror[2], p2<bottomerror[2]))[0])
          nbadparam = np.array([len(badparamrnum0), len(badparamrnum1), len(badparamrnum2)])
  
          retval["METRICS"]={"CHECKARC":ngoodfits, "GOODPSFS":allfibs, "CHECKPSF":nbadparam}
          retval["DATA"]={"LPolyCoef0":p0, "LPolyCoef1":p1, "LPolyCoef2":p2}
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          retval["PARAMS"] = param
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
  #        if qafile is not None:
  #            outfile=qa.write_qa_ql(qafile,retval)
  #            log.debug("Output QA data is in {}".format(outfile))
          if qafig is not None:
              fig.plot_lpolyhist(retval,qafig,plotconf=plotconf,hardplots=hardplots)
              log.debug("Output QA fig {}".format(qafig))
  
          return retval
  
      def get_default_config(self):
          return {}
  
  class Check_FiberFlat(MonitoringAlg):
      def __init__(self,name,config,logger=None):
          if name is None or name.strip() == "":
              name="CHECKFLAT"
          kwargs=config['kwargs']
          parms=kwargs['param']
          key=kwargs['refKey'] if 'refKey' in kwargs else "CHECKFLAT"
          status=kwargs['statKey'] if 'statKey' in kwargs else "CHECKFLAT_STATUS"
          kwargs["RESULTKEY"]=key
          kwargs["QASTATUSKEY"]=status
  
          if "ReferenceMetrics" in kwargs:
              r=kwargs["ReferenceMetrics"]
              if key in r:
                  kwargs["REFERENCE"]=r[key]
  
          if "CHECKFLAT_WARN_RANGE" in parms and "CHECKFLAT_NORMAL_RANGE" in parms:
              kwargs["RANGES"]=[(np.asarray(parms["CHECKFLAT_WARN_RANGE"]),QASeverity.WARNING),
                                (np.asarray(parms["CHECKFLAT_NORMAL_RANGE"]),QASeverity.NORMAL)]
  
          MonitoringAlg.__init__(self,name,fr,config,logger)
      def run(self,*args,**kwargs):
          if len(args) == 0 :
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
              
          if not self.is_compatible(type(args[0])):
              #raise qlexceptions.ParameterException("Incompatible input. Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
              log.critical("Incompatible input!")
              sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))
  
          if kwargs["singleqa"] == 'Check_FiberFlat':
              night = kwargs['night']
              expid = '{:08d}'.format(kwargs['expid'])
              camera = kwargs['camera']
          else: fibflat=args[0]
          inputs=get_inputs(*args,**kwargs)
  
          return self.run_qa(fibflat,inputs)
  
      def run_qa(self,fibflat,inputs):
          camera=inputs["camera"]
          paname=inputs["paname"]
          fibermap=inputs["fibermap"]
          amps=inputs["amps"]
          qafile=inputs["qafile"]
          qafig=inputs["qafig"]
          param=inputs["param"]
          refmetrics=inputs["refmetrics"]
          
          kwargs=self.config['kwargs']
          retval={}
          retval["PANAME"] = paname
          retval["QATIME"] = datetime.datetime.now().isoformat()
          retval["PROGRAM"] = 'FLAT'
          retval["FLAVOR"] = 'flat'
          retval["NIGHT"] = kwargs['night']
          retval["CAMERA"] = fibflat.header['CAMERA']
          retval["EXPID"] = '{:08d}'.format(kwargs['expid'])
  
          if param is None:
              log.critical("No parameter is given for this QA! ")
              sys.exit("Check the configuration file")
  
          retval["PARAMS"] = param
  
          #- Calculate mean and rms fiberflat value for each fiber
          fiberflat = fibflat.fiberflat
          avg_fiberflat=[]
          rms=[]
          for fib in range(len(fiberflat)):
              avg_fiberflat.append(np.mean(fiberflat[fib]))
              rms.append(np.std(fiberflat[fib]))
  
          #- Calculate mean of the fiber means for scalar metric
          avg_all = np.mean(avg_fiberflat)
  
          retval['METRICS'] = {"FLATMEAN":avg_fiberflat, "FLATRMS":rms, "CHECKFLAT":avg_all}
  
          ###############################################################
          # This section is for adding QA metrics for plotting purposes #
          ###############################################################
  
          ###############################################################
  
          return retval
  
      def get_default_config(self):
          return {}
  
apis: numpy.sum numpy.isnan numpy.array numpy.where numpy.unique
apis: numpy.allclose numpy.abs numpy.mean numpy.isnan
apis: numpy.ones numpy.logical_or numpy.asarray numpy.nan_to_num numpy.float64 numpy.isfinite numpy.abs numpy.genfromtxt numpy.log10 numpy.delete numpy.where numpy.mean numpy.sqrt numpy.zeros numpy.median numpy.arange numpy.all numpy.std numpy.array numpy.percentile numpy.zeros_like numpy.gradient numpy.concatenate