text: """This module provides functions that interface with the JWST calibration code to determine
  things like "reference types used by a pipeline."
  
  >>> header_to_reftypes(test_header("0.7.0", "FGS_DARK"))
  ['ipc', 'linearity', 'mask', 'refpix', 'rscd', 'saturation', 'superbias']
  
  >>> header_to_reftypes(test_header("0.7.0", "NRS_BRIGHTOBJ"))
  ['area', 'camera', 'collimator', 'dark', 'disperser', 'distortion', 'drizpars', 'extract1d', 'filteroffset', 'fore', 'fpa', 'fringe', 'gain', 'ifufore', 'ifupost', 'ifuslicer', 'ipc', 'linearity', 'mask', 'msa', 'ote', 'pathloss', 'photom', 'readnoise', 'refpix', 'regions', 'rscd', 'saturation', 'specwcs', 'straymask', 'superbias', 'v2v3', 'wavelengthrange']
  
  >>> header_to_reftypes(test_header("0.13.0", "MIR_IMAGE"))
  ['area', 'camera', 'collimator', 'dark', 'dflat', 'disperser', 'distortion', 'drizpars', 'fflat', 'filteroffset', 'flat', 'fore', 'fpa', 'gain', 'ifufore', 'ifupost', 'ifuslicer', 'linearity', 'mask', 'msa', 'ote', 'persat', 'photom', 'readnoise', 'refpix', 'regions', 'rscd', 'saturation', 'sflat', 'specwcs', 'superbias', 'trapdensity', 'trappars', 'wavelengthrange', 'wfssbkg']
  
  
  >>> header = test_header("0.13.0", "MIR_LRS-FIXEDSLIT")
  >>> header_to_pipelines(header)
  ['calwebb_detector1.cfg', 'calwebb_nrslamp-spec2.cfg']
  >>> header_to_reftypes(header)
  ['barshadow', 'camera', 'collimator', 'cubepar', 'dark', 'disperser', 'distortion', 'drizpars', 'extract1d', 'filteroffset', 'fore', 'fpa', 'gain', 'ifufore', 'ifupost', 'ifuslicer', 'linearity', 'mask', 'msa', 'msaoper', 'ote', 'persat', 'readnoise', 'refpix', 'regions', 'resol', 'rscd', 'saturation', 'specwcs', 'superbias', 'trapdensity', 'trappars', 'wavecorr', 'wavelengthrange', 'wfssbkg']
  
  >>> header_to_pipelines(test_header("0.7.0", "FGS_DARK"))
  ['calwebb_dark.cfg', 'skip_2b.cfg']
  
  >>> header_to_pipelines(test_header("0.7.0", "NRS_BRIGHTOBJ"))
  ['calwebb_sloper.cfg', 'calwebb_spec2.cfg']
  
  >>> header = test_header("0.13.0", "MIR_IMAGE", tsovisit="F")
  >>> header_to_pipelines(header)
  ['calwebb_detector1.cfg', 'calwebb_image2.cfg']
  
  >>> header = test_header("0.13.0", "MIR_IMAGE", tsovisit="T")
  >>> header_to_pipelines(header)
  ['calwebb_tso1.cfg', 'calwebb_tso-image2.cfg']
  
  >>> header_to_pipelines(test_header("0.7.0", "MIR_IMAGE"))
  ['calwebb_sloper.cfg', 'calwebb_image2.cfg']
  
  >>> header_to_pipelines(test_header("0.7.0", "MIR_LRS-FIXEDSLIT"))
  ['calwebb_sloper.cfg', 'calwebb_spec2.cfg']
  
  >>> header = test_header("0.13.8", "MIR_IMAGE", visitype="GENERIC")
  >>> _get_pipelines("MIR_IMAGE", "0.13.8", "jwst_0552.pmap")
  ['calwebb_detector1.cfg', 'calwebb_image2.cfg']
  >>> header_to_pipelines(header)
  ['calwebb_detector1.cfg', 'calwebb_image2.cfg']
  
  >>> header = test_header("0.13.8", "MIR_IMAGE", visitype="PRIME_WFSC_ROUTINE")
  >>> header_to_pipelines(header)
  ['calwebb_detector1.cfg', 'calwebb_wfs-image2.cfg']
  
  >>> _get_missing_calver("0.7.0")
  '0.7.0'
  
  >>> s = _get_missing_calver()
  >>> isinstance(s, str)
  True
  
  >>> _get_missing_context()
  'jwst-operational'
  
  >>> _get_missing_context('jwst_0341.pmap')
  'jwst_0341.pmap'
  
  >>> os.path.basename(_get_config_refpath("jwst_0552.pmap", "0.7.7"))
  'jwst_system_crdscfg_b7.yaml'
  
  >>> os.path.basename(_get_config_refpath("jwst_0552.pmap", "0.9.3"))
  'jwst_system_crdscfg_b7.1.3.yaml'
  
  >>> os.path.basename(_get_config_refpath("jwst_0552.pmap", "0.9.7"))
  'jwst_system_crdscfg_b7.1.3.yaml'
  
  >>> os.path.basename(_get_config_refpath("jwst_0477.pmap", "0.10.1"))
  'jwst_system_crdscfg_b7.2.yaml'
  
  >>> os.path.basename(_get_config_refpath("jwst_0552.pmap", "0.13.8"))
  'jwst_system_crdscfg_b7.4.yaml'
  """
  
  import os.path
  from collections import defaultdict
  import re
  
  # import yaml     DEFERRED
  
  from pkg_resources import parse_version
  
  # --------------------------------------------------------------------------------------
  
  # from jwst import __version__     DEFERRED
  
  # --------------------------------------------------------------------------------------
  import crds
  from crds.core import log, utils, config
  from crds.core.log import srepr
  from crds.client import api
  
  # --------------------------------------------------------------------------------------
  
  def test_header(calver, exp_type, tsovisit="F", lamp_state="OFF", visitype="GENERIC"):
      """Create a header-like dictionary from `calver` and `exp_type` to
      support testing header-based functions.
      """
      header = {
          "META.INSTRUMENT.NAME" : "SYSTEM",
          "REFTYPE" : "CRDSCFG",
          "META.CALIBRATION_SOFTWARE_VERSION" : calver,
          "META.EXPOSURE.TYPE" : exp_type,
          "META.VISIT.TSOVISIT" : tsovisit,
          "META.INSTRUMENT.LAMP_STATE" : lamp_state,
          "META.VISIT.TITLE" : visitype,
          }
      return header
  
  # --------------------------------------------------------------------------------------
  
  def _get_missing_calver(cal_ver=None):
      """If `cal_ver` is None, return the calibration software version for
      the installed version of calibration code.  Otherwise return `cal_ver`
      unchanged.
      """
      if cal_ver is None:
          from jwst import __version__ as calver2
          return calver2
      return cal_ver
  
  def _get_missing_context(context=None):
      """Default the context to `jwst-operational` if `context` is None, otherwise
      return context unchanged.
      """
      return "jwst-operational" if context is None else context
  
  # --------------------------------------------------------------------------------------
  
  def header_to_reftypes(header, context=None):
      """Given a dataset `header`,  extract the EXP_TYPE or META.EXPOSURE.TYPE keyword
      from and use it to look up the reftypes required to process it.
  
      Return a list of reftype names.
      """
      with log.warn_on_exception("Failed determining reftypes for", log.PP(header)):
          exp_type, cal_ver = _header_to_exptype_calver(header)
          config_manager = _get_config_manager(context, cal_ver)
          pipelines = header_to_pipelines(header, context)
          reftypes = set()
          for cfg in pipelines:
              steps = config_manager.pipeline_cfgs_to_steps[cfg]
              for step in steps:
                  reftypes |= set(config_manager.steps_to_reftypes[step])
          return sorted(list(reftypes))
      return []
  
  # This is potentially an external interface to system data processing (SDP) / the archive pipeline.
  def header_to_pipelines(header, context=None):
      """Given a dataset `header`,  extract the EXP_TYPE or META.EXPOSURE.TYPE keyword
      from and use it to look up the pipelines required to process it.
  
      Return a list of pipeline .cfg names.
      """
      with log.augment_exception("Failed determining exp_type, cal_ver from header", log.PP(header)):
          exp_type, cal_ver = _header_to_exptype_calver(header)
      config_manager = _get_config_manager(context, cal_ver)
      pipelines = _get_pipelines(exp_type, cal_ver, context)   # uncorrected
      if config_manager.pipeline_exceptions:  # correction based on extra non-EXP_TYPE params
          pipelines2 = []
          for cfg in pipelines:
              for param, exceptions in config_manager.pipeline_exceptions.items():
                  exceptions = dict(exceptions)
                  dont_replace = exceptions.pop("dont_replace", [])
                  do_replace = exceptions.pop("do_replace", [])
                  default_missing  = exceptions.pop("default_missing", "UNDEFINED")
                  paramval = header.get(param.upper(), default_missing)
                  for dont in dont_replace:
                      if re.match(config.complete_re(dont), paramval):
                          break
                  else:
                      if dont_replace:
                          cfg = exceptions.get(cfg, cfg)
                  for do in do_replace:
                      if re.match(config.complete_re(do), paramval):
                          cfg = exceptions.get(cfg, cfg)
              pipelines2.append(cfg)
          pipelines = pipelines2
      log.verbose("Applicable pipelines for", srepr(exp_type), "are", srepr(pipelines))
      return pipelines
  
  def _get_pipelines(exp_type, cal_ver=None, context=None):
      """Given `exp_type` and `cal_ver` and `context`,  locate the appropriate SYSTEM CRDSCFG
      reference file and determine the sequence of pipeline .cfgs required to process that
      exp_type.
  
      NOTE:  This is an uncorrected result,  config_manager.pipeline_exceptions is used to
      alter this based on other header parameters.
      """
      with log.augment_exception("Failed determining required pipeline .cfgs for",
                                 "EXP_TYPE", srepr(exp_type)):
          config_manager = _get_config_manager(context, cal_ver)
          return config_manager.exptype_to_pipelines(exp_type)
  
  def reftype_to_pipelines(reftype, cal_ver=None, context=None):
      """Given `exp_type` and `cal_ver` and `context`,  locate the appropriate SYSTEM CRDSCFG
      reference file and determine the sequence of pipeline .cfgs required to process that
      exp_type.
      """
      with log.augment_exception("Failed determining required pipeline .cfgs for",
                                 "REFTYPE", srepr(reftype)):
          config_manager = _get_config_manager(context, cal_ver)
          return config_manager.reftype_to_pipelines(reftype)
  
  def _header_to_exptype_calver(header):
      """Given dataset `header`,  return the EXP_TYPE and CAL_VER values."""
      cal_ver = header.get("META.CALIBRATION_SOFTWARE_VERSION", header.get("CAL_VER"))
      cal_ver = _get_missing_calver(cal_ver)
      exp_type = header.get("META.EXPOSURE.TYPE",  header.get("EXP_TYPE", "UNDEFINED"))
      return exp_type, cal_ver
  
  @utils.cached  # for caching,  pars must be immutable, ideally simple
  def _get_config_manager(context, cal_ver):
      """Given `context` and calibration s/w version `cal_ver`,  identify the appropriate
      SYSTEM CRDSCFG reference file and create a CrdsCfgManager from it.
      """
      refpath = _get_config_refpath(context, cal_ver)
      return _load_refpath(context, refpath)
  
  def _load_refpath(context, refpath):
      """Given `context` and SYSTEM CRDSCFG reference at `refpath`,  construct a CrdsCfgManager."""
      import yaml
      with open(refpath) as opened:
          crdscfg =  yaml.safe_load(opened)
      return CrdsCfgManager(context, refpath, crdscfg)
  
  # --------------------------------------------------------------------------------------
  
  HERE = os.path.dirname(__file__) or "."
  
  REFPATHS = [
      ('0.7.7', "jwst_system_crdscfg_b7.yaml"),
      ('0.9.0', "jwst_system_crdscfg_b7.1.yaml"),
      ('0.9.1', "jwst_system_crdscfg_b7.1.1.yaml"),
      ('0.9.3', "jwst_system_crdscfg_b7.1.3.yaml"),
      ('0.10.0', "jwst_system_crdscfg_b7.2.yaml"),
      ('0.13.0', "jwst_system_crdscfg_b7.3.yaml"),
      ('0.13.8', "jwst_system_crdscfg_b7.4.yaml"),
      ('0.16.0', "jwst_system_crdscfg_b7.5.yaml"),
      ('0.17.0', "jwst_system_crdscfg_b7.6.yaml"),
      ('999.0.0', "jwst_system_crdscfg_b7.6.yaml"),   # latest backstop
  ]
  
  def _get_config_refpath(context, cal_ver):
      """Given CRDS `context` and calibration s/w version `cal_ver`,  identify the applicable
      SYSTEM CRDSCFG reference file, cache it, and return the file path.
      """
      context = _get_missing_context(context)
      cal_ver = _get_missing_calver(cal_ver)
      i = 0
      while (i < len(REFPATHS)-1 and not _versions_lt(cal_ver, REFPATHS[i+1][0])):
          i += 1
      refpath = os.path.join(HERE, REFPATHS[i][1])
      try:  # Use a normal try/except because exceptions are expected.
          header = {
              "META.INSTRUMENT.NAME" : "SYSTEM",
              "META.CALIBRATION_SOFTWARE_VERSION": cal_ver
          }
          pmap = crds.get_symbolic_mapping(context)
          imap = pmap.get_imap("system")
          rmapping = imap.get_rmap("crdscfg")
          ref = rmapping.get_best_ref(header)
          refpath = rmapping.locate_file(ref)
          api.dump_references(context, [ref])
      except Exception:
          log.verbose_warning(
              "Failed locating SYSTEM CRDSCFG reference",
              "under context", repr(context),
              "and cal_ver", repr(cal_ver) + ".   Using built-in references.")
      log.verbose("Using", srepr(os.path.basename(refpath)),
                  "to determine applicable default reftypes for", srepr(cal_ver))
      return refpath
  
  def _versions_lt(v1, v2):
      """Compare two semantic version numbers and account for issues like '10' < '9'.
      Return True IFF `v1` < `v2`.
  
      >>> _versions_gte("0.7.7", "0.7.7")
      True
      >>> _versions_gte("0.7.8", "0.7.7")
      True
      >>> _versions_gte("0.7.6", "0.7.7")
      False
      >>> _versions_gte("0.7.7", "0.7.8")
      False
      >>> _versions_gte("0.7.7", "0.7.6")
      True
      >>> _versions_gte("0.10.1", "0.7.7")
      True
      >>> _versions_gte("0.10.1", "0.10.0")
      True
      >>> _versions_gte("0.10.0", "0.10.1")
      False
      >>> _versions_gte("0.10.1", "0.10.1")
      True
      >>> _versions_gte("0.10.1", "0.10.1a1")
      True
      >>> _versions_gte("0.10.2", "0.10.1a1")
      True
      >>> _versions_gte("0.10.1", "0.11.1a1")
      False
      >>> _versions_gte("0.9.7", "0.10.0")
      False
      >>> _versions_gte("0.9.2", "0.10.1dev20000")
      False
      >>> _versions_gte("0.99.7", "0.10.1dev20000")
      True
      >>> _versions_gte("0.10.1", "0.10.1dev20000")
      True
      """
      return _reduce_ver(v1) < _reduce_ver(v2)
  
  def _versions_gte(v1, v2):
      """"""
      return _reduce_ver(v1) >= _reduce_ver(v2)
  
  def _reduce_ver(ver):
      """
      >>> _reduce_ver("1")
      (1, 0, 0)
      >>> _reduce_ver("1.2")
      (1, 2, 0)
      >>> _reduce_ver("1.2.3")
      (1, 2, 3)
      >>> _reduce_ver("1.2.3.4")
      (1, 2, 3)
      """
      ver = parse_version(ver).base_version
      parts = ver.split(".")
      while len(parts) < 3:
          parts.append("0")
      return tuple(map(int, parts))[:3]
  
  class CrdsCfgManager:
      """The CrdsCfgManager handles using SYSTEM CRDSCFG information to compute things."""
      def __init__(self, context, refpath, crdscfg):
          self._context = context
          self._refpath = refpath
          self._crdscfg = utils.Struct(crdscfg)
          self.pipeline_exceptions = self._crdscfg.get("pipeline_exceptions", {})
          self.pipeline_cfgs_to_steps = self._crdscfg.get("pipeline_cfgs_to_steps", {})
          self.steps_to_reftypes = self._crdscfg.get("steps_to_reftypes", {})
  
      def exptype_to_reftypes(self, exp_type):
          """For a given EXP_TYPE string, return a list of reftypes needed to process that
          EXP_TYPE through the data levels appropriate for that EXP_TYPE.
  
          Return [reftypes, ... ]
          """
          reftypes = self._crdscfg.exptypes_to_reftypes[exp_type]
          log.verbose("Applicable reftypes for", srepr(exp_type),
                      "determined by", srepr(os.path.basename(self._refpath)),
                      "are", srepr(reftypes))
          return reftypes
  
      def exptype_to_pipelines(self, exp_type):
          """For a given EXP_TYPE string, return a list of pipeline .cfg's needed to
          process that EXP_TYPE through the appropriate data levels.
  
          Return [.cfg's, ... ]
          """
          pipelines = self._crdscfg.exptypes_to_pipelines[exp_type]
          return pipelines
  
      def reftype_to_pipelines(self, reftype):
          pipelines = []
          reftypes_to_steps = invert_list_mapping(self._crdscfg.steps_to_reftypes)
          steps_to_pipelines = invert_list_mapping(self._crdscfg.pipeline_cfgs_to_steps)
          for step in reftypes_to_steps[reftype]:
              pipelines += steps_to_pipelines[step]
          return list(sorted(set(pipelines)))
  
  def invert_list_mapping(mapping):
      """Invert a dictionary of lists into another dictionary of lists such that each
      element of each original list is a key somewhere in the inversion, and each key is
      an element of at least one list in the inversion.
      """
      inverted = defaultdict(set)
      for key, values in mapping.items():
          for value in values:
              inverted[value].add(key)
      return { key:list(sorted(values))
               for (key,values) in inverted.items() }
  
  def scan_exp_type_coverage():
      """Verify that there is some get_reftypes() response for all available exp_types."""
      from . import schema as crds_schema
      exp_types = crds_schema.get_exptypes()
      for exp_type in exp_types:
          if exp_type in ["ANY","N/A"]:
              continue
          with log.warn_on_exception("failed determining reftypes for", repr(exp_type)):
              reftypes = _get_reftypes(exp_type)
              log.verbose("Reftypes for", repr(exp_type), "=", repr(reftypes))
  
  def _get_reftypes(exp_type, cal_ver=None, context=None):
      """Given `exp_type` and `cal_ver` and `context`,  locate the appropriate SYSTEM CRDSCFG
      reference file and determine the reference types required to process every pipeline Step
      nominally associated with that exp_type.
      """
      with log.warn_on_exception("Failed determining required reftypes from",
                                 "EXP_TYPE", srepr(exp_type)):
          config_manager = _get_config_manager(context, cal_ver)
          return config_manager.exptype_to_reftypes(exp_type)
      return []
  
  def test():
      """Run the module doctests."""
      import doctest
      from crds.jwst import pipeline
      return doctest.testmod(pipeline)
  
  # --------------------------------------------------------------------------------------
  
  if __name__ == "__main__":
      print(test())
  