text: #!/usr/bin/env python
  from __future__ import print_function
  import sys
  import re
  if sys.hexversion < 0x03000000:
      from future_builtins import zip
  ################################################################################
  #
  #   ruffus_utility.py
  #
  #
  #   Copyright (c) 10/9/2009 Leo Goodstadt
  #
  #   Permission is hereby granted, free of charge, to any person obtaining a copy
  #   of this software and associated documentation files (the "Software"), to deal
  #   in the Software without restriction, including without limitation the rights
  #   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  #   copies of the Software, and to permit persons to whom the Software is
  #   furnished to do so, subject to the following conditions:
  #
  #   The above copyright notice and this permission notice shall be included in
  #   all copies or substantial portions of the Software.
  #
  #   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  #   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  #   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  #   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  #   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  #   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  #   THE SOFTWARE.
  #################################################################################
  
  
  """
  
  ********************************************
  :mod:`ruffus_utility` -- Overview
  ********************************************
  
  
  .. moduleauthor:: Leo Goodstadt <ruffus@llew.org.uk>
  
      Common utility functions
  
  
  """
  
  
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   imports
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  import os,copy
  import re
  import types
  import glob
  from functools import reduce
  if __name__ == '__main__':
      import sys
      sys.path.insert(0,".")
  from .ruffus_exceptions import *
  #import task
  import collections
  from collections import defaultdict
  import multiprocessing.managers
  import hashlib
  import marshal
  try:
      import cPickle as pickle
  except:
      import pickle as pickle
  import operator
  from . import dbdict
  from itertools import chain
  if sys.hexversion >= 0x03000000:
      # everything is unicode in python3
      path_str_type = str
  else:
      path_str_type = basestring
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   Constants
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #
  # file to store history out to
  #
  RUFFUS_HISTORY_FILE = '.ruffus_history.sqlite'
  # If DEFAULT_RUFFUS_HISTORY_FILE is specified in the environment variables, use that instead
  if "DEFAULT_RUFFUS_HISTORY_FILE" in os.environ:
      RUFFUS_HISTORY_FILE = os.environ["DEFAULT_RUFFUS_HISTORY_FILE"]
  
  
  CHECKSUM_FILE_TIMESTAMPS      = 0     # only rerun when the file timestamps are out of date (classic mode)
  CHECKSUM_HISTORY_TIMESTAMPS   = 1     # also rerun when the history shows a job as being out of date
  CHECKSUM_FUNCTIONS            = 2     # also rerun when function body has changed
  CHECKSUM_FUNCTIONS_AND_PARAMS = 3     # also rerun when function parameters or function body change
  
  CHECKSUM_REGENERATE           = 2     # regenerate checksums
  
  #_________________________________________________________________________________________
  
  #   t_extra_inputs
  #       namespaced enum
  
  #_________________________________________________________________________________________
  class t_extra_inputs:
      (ADD_TO_INPUTS, REPLACE_INPUTS, KEEP_INPUTS, KEEP_OUTPUTS) = list(range(4))
  
  #_________________________________________________________________________________________
  
  #   inputs
  
  #_________________________________________________________________________________________
  class inputs(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'inputs%r' % (self.args,)
  
  #_________________________________________________________________________________________
  
  #   add_inputs
  
  #_________________________________________________________________________________________
  class add_inputs(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'add_inputs%r' % (self.args,)
  
  #_________________________________________________________________________________________
  #
  #   get_default_checksum_level
  #
  #_________________________________________________________________________________________
  def get_default_checksum_level ():
      """
      Use the checksum level from the environmental variable DEFAULT_RUFFUS_CHECKSUM_LEVEL
      Otherwise default to CHECKSUM_HISTORY_TIMESTAMPS
      """
  
      #
      #   environmental variable not set
      #
      if "DEFAULT_RUFFUS_CHECKSUM_LEVEL" not in os.environ:
          return CHECKSUM_HISTORY_TIMESTAMPS
  
  
  
      #
      # lookup value from list of CHECKSUM_XXX constants
      #
      checksum_level = None
      env_checksum_level = os.environ["DEFAULT_RUFFUS_CHECKSUM_LEVEL"]
      if len(env_checksum_level) == 1 and env_checksum_level in "0123":
          checksum_level = int(env_checksum_level)
      else:
          global_var = globals()
          for key in global_var:
              if key.startswith('CHECKSUM') and global_var[key] == env_checksum_level:
                  checksum_level = value
  
      #
      #   check environmental variable is valid string
      #
      if checksum_level is None:
          raise error_checksum_level(("The environmental value "
                                     "DEFAULT_RUFFUS_CHECKSUM_LEVEL should be: [0-3 | "
                                     "CHECKSUM_FILE_TIMESTAMPS | "
                                     "CHECKSUM_HISTORY_TIMESTAMPS | "
                                     "CHECKSUM_FUNCTIONS | "
                                     "CHECKSUM_FUNCTIONS_AND_PARAMS] (rather than '%s') ")
                                     % (env_checksum_level,))
  
      return checksum_level
  
  
  #_________________________________________________________________________________________
  
  #   open_job_history
  
  #_________________________________________________________________________________________
  def get_default_history_file_name ():
      history_file = RUFFUS_HISTORY_FILE
      #
      #   try path expansion using the main script name
      #
      try:
          import __main__ as main
          path_parts = path_decomposition (os.path.abspath(main.__file__))
          history_file = history_file.format(**path_parts)
      except Exception:
          pass
      return history_file
  
  def open_job_history (history_file):
      """
      Given a history file name, opens the correspond sqllite db file and returns the handle
      """
      if not history_file:
          history_file = get_default_history_file_name ()
  
      return dbdict.open(history_file, picklevalues=True)
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   Functions
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  class JobHistoryChecksum:
      """Class to remember exactly how an output file was created and when."""
  
      def __str__(self):
          from time import strftime, gmtime
          if hasattr(self, "params"):
              return   str([self.outfile,
                        strftime("%d %b %Y %H:%M:%S", gmtime(self.mtime)),
                        self.params,
                        self.task_name
                       ])
          else:
              return strftime("%d %b %Y %H:%M:%S", gmtime(self.mtime))
  
      def __init__(self, outfile, mtime, params, task):
          # filename and modification time
          self.outfile = outfile
          self.mtime = mtime
  
          # Uncomment next two lines to debug:
          #self.params = params
          #self.task_name = task._name
  
          # checksum exact params used to generate this output file
          self.chksum_params = hashlib.md5(pickle.dumps(params)).hexdigest()
          # checksum the function bytecode as well as the function context
          # Don't use func_code alone-- changing the line number of the function,
          # what global variables are available, etc would all change the checksum
          if sys.hexversion >= 0x03000000:
              code = task.user_defined_work_func.__code__
              func_defaults = task.user_defined_work_func.__defaults__
          else:
              code = task.user_defined_work_func.func_code
              func_defaults = task.user_defined_work_func.func_defaults
          func_code = marshal.dumps(code.co_code)
  
  
          #
          #   pickle code very defensively, but hopefully without breaking Jake Biesinger's pipelines!
          #
          attributes_to_pickle = [func_defaults,
                                  code.co_argcount,
                                  code.co_consts,
                                  code.co_names,
                                  code.co_nlocals,
                                  code.co_varnames]
  
          pickle_results = []
          for aa in attributes_to_pickle:
              # Can't cpickle nested functions: typically blows up with func_code.co_consts
              try:
                  pickle_results.append(pickle.dumps(aa))
                  continue
              except:
                  pass
              # Marshal seems to be less sensitive: try that
              try:
                  pickle_results.append(marshal.dumps(aa))
                  continue
              except:
                  pass
              # Just make a string out of the attribute
              try:
                  pickle_results.append(str(aa))
                  continue
              except:
                  pass
              # OK give up, do nothing: On your head it is
  
  
          func_extras = reduce(operator.add, pickle_results)
          self.chksum_func = hashlib.md5(func_code + func_extras).hexdigest()
  
  
  
  #_________________________________________________________________________________________
  #
  #   parameter_list_as_string
  #
  #_________________________________________________________________________________________
  def parameter_list_as_string (parameters):
      """
      Input list of parameters
         Turn this into a string for display
  
          E.g.
  
      """
      if parameters is None:
          return ""
      elif not isinstance(parameters, list):
          raise Exception("Unexpected parameter list %s" % (parameters,))
      else:
          return str(parameters)[1:-1]
  
  #_________________________________________________________________________________________
  #
  #   path_decomposition
  #
  #_________________________________________________________________________________________
  def path_decomposition (orig_path):
      """
      returns a dictionary identifying the components of a file path:
          This has the following keys
              basename: (any) base (file) name of the path not including the extension. No slash included
              ext:      (any) extension of the path including the "."
              path:     a list of subpaths created by removing subdirectory names
              subdir:   a list of subdirectory names from the most nested to the root
          For example
              apath = "/a/b/c/d/filename.txt"
              {   'basename': 'filename',
                  'ext':      '.txt'
                  'path':     ['/a/b/c/d', '/a/b/c', '/a/b', '/a', '/'], ,
                  'subdir': ['d', 'c', 'b', 'a', '/']
              }
              "{path[2]}/changed/{subdir[0]}".format(**res) = '/a/b/changed/d'
              "{path[3]}/changed/{subdir[1]}".format(**res) = '/a/changed/c'
      """
      def recursive_split (a_path):
          """
          split the path into its subdirectories recursively
          """
          if not len(a_path):
              return [[],[]]
          if a_path == "/" or a_path == "//":
              return [ [a_path] , [a_path]]
          sub_path_part, sub_dir_part = os.path.split(a_path)
          if sub_dir_part:
              sub_path_parts, sub_dir_parts = recursive_split (sub_path_part)
              return [ [a_path] + sub_path_parts,
                        [sub_dir_part] + sub_dir_parts]
          else:
              return [ [] , ["/"]]
      #
      if not len(orig_path):
          return {'path': [], 'basename': '', 'ext': '', 'subdir': []}
  
      # stop normpath from being too clever and removing initial ./ and terminal slash, turning paths into filenames
      if orig_path in [ "./", "/."]:
          a_path = orig_path
      else:
          a_path = os.path.normpath(orig_path)
          if orig_path[0:2] == "./" and a_path[0:2] != "./":
              a_path = "./" + a_path
  
          if orig_path[-1] == "/" and a_path[-1:] != "/":
              a_path += "/"
  
      path_part, file_part = os.path.split(a_path)
      file_part, ext_part = os.path.splitext(file_part)
      subpaths, subdirs = recursive_split (path_part)
      return {'basename': file_part,
              'ext':      ext_part,
              'subpath':  subpaths,
              'subdir':   subdirs,
              'path':     path_part}
  
  
  #_________________________________________________________________________________________
  #
  #   get_nth_nested_level_of_path
  #
  #_________________________________________________________________________________________
  def get_nth_nested_level_of_path (orig_path, n_levels):
      """
      Return path with up to N levels of subdirectories
      0 = full path
      N = 1 : basename
      N = 2 : basename + one subdirectory
  
      For example
          0   /test/this/now/or/not.txt
          1   not.txt
          2   or/not.txt
          3   now/or/not.txt
          4   this/now/or/not.txt
          5   test/this/now/or/not.txt
          6   /test/this/now/or/not.txt
          7   /test/this/now/or/not.txt
      """
      if not n_levels or n_levels < 0:
          return orig_path
      res = path_decomposition(orig_path)
      basename = res["basename"] + res["ext"]
      shortened_path = os.path.join(*(list(reversed(res["subdir"][0:(n_levels - 1)]))+[basename]))
      if len(shortened_path) < len(orig_path):
          return ".../" + shortened_path
  
  
  
  #_________________________________________________________________________________________
  #
  #   swap_nesting_order
  #
  #_________________________________________________________________________________________
  def swap_nesting_order (orig_coll):
      """
      Reverse nested order so that coll[3]['a'] becomes coll['a'][3]
      """
      new_dict = defaultdict(dict)
      new_list = []
      for ii, ii_item in enumerate(orig_coll):
          for jj, value in ii_item.items():
              if isinstance(jj, int):
                  # resize
                  new_list += [{}]*(jj + 1 - len(new_list))
                  new_list[jj][ii] = value
              else:
                  new_dict[jj][ii] = value
      return new_list, new_dict
  
  #_________________________________________________________________________________________
  #
  #   swap_doubly_nested_order
  #
  #_________________________________________________________________________________________
  def swap_doubly_nested_order (orig_coll):
      """
      Reverse nested order so that coll[3]['a'] becomes coll['a'][3]
      """
      new_dict = dict()
      new_list = []
      for ii, ii_item in enumerate(orig_coll):
          for jj, jj_item in enumerate(ii_item):
              for kk, value in jj_item.items():
                  if isinstance(kk, int):
                      # resize
                      new_list += [{}]*(kk + 1 - len(new_list))
                      if ii not in new_list[kk]:
                          new_list[kk][ii] = dict()
                      new_list[kk][ii][jj] = value
                  else:
                      if kk not in new_dict:
                          new_dict[kk] = dict()
                      if ii not in new_dict[kk]:
                          new_dict[kk][ii] = dict()
                      new_dict[kk][ii][jj] = value
  
      return new_list, new_dict
  
  
  #_________________________________________________________________________________________
  #
  #   regex_match_str
  #
  #_________________________________________________________________________________________
  def regex_match_str(test_str, compiled_regex):
      """
      Returns result of regular expression match in a dictionary
          combining both named and unnamed captures
      """
      if compiled_regex:
          if isinstance(compiled_regex, path_str_type):
              compiled_regex = re.compile(compiled_regex)
          mm = compiled_regex.search(test_str)
          # Match failed
          if mm is None:
              return False
          else:
              # No capture
              if mm.lastindex is None:
                  return {0: mm.group(0)}
              # Combined named and unnamed captures
              else:
                  # no dictionary comprehensions in python 2.6 :-(
                  #matchdicts.append({i : mm.group(i) for i in (range(mm.lastindex) + mm.groupdict().keys())})
                  return dict((i, mm.group(i)) for i in (chain(   iter(range(mm.lastindex + 1)),
                                                                  iter(mm.groupdict().keys()))))
  
      else:
          return None
  
  
  #_________________________________________________________________________________________
  #
  #   path_decomposition_regex_match
  #
  #_________________________________________________________________________________________
  def path_decomposition_regex_match (test_str, compiled_regex):
      """
      Returns a dictionary identifying the components of a file path.
  
      This includes both the components of a path:
          basename: (any) base (file) name of the path not including the extension. No slash included
          ext:      (any) extension of the path including the "."
          path:     a list of subpaths created by removing subdirectory names
          subdir:   a list of subdirectory names from the most nested to the root
      and regular expression matches
          The keys are the index or name of the capturing group.
  
  
      If compiled_regexes is not specified, return path decomposition only
  
      If compiled_regexes is specified, and the corresponding regular expression does not match,
          the entire match fails
  
      For example
  
          path_decomposition_regex_match("/a/b/c/sample1.bam", r"(.*)(?P<id>\d+)\..+")
  
              {
                  0:          '/a/b/c/sample1.bam',           // captured by index
                  1:          '/a/b/c/sample',                // captured by index
                  'id':       '1'                             // captured by name
                  'ext':      '.bam',
                  'subdir':   ['c', 'b', 'a', '/'],
                  'subpath':  ['/a/b/c', '/a/b', '/a', '/'],
                  'path':     '/a/b/c',
                  'basename': 'sample1',
              },
  
          path_decomposition_regex_match("dbsnp15.vcf", r"(.*)(?P<id>\d+)\..+")
              {
                  0: 'dbsnp15.vcf',                           // captured by index
                  1: 'dbsnp1',                                // captured by index
                  'id': '5'                                   // captured by name
                  'ext': '.vcf',
                  'subdir': [],
                  'path': [],
                  'basename': 'dbsnp15',
              },
  
  
          // fail
          path_decomposition_regex_match("/test.txt", r"(.*)(?P<id>\d+)\..+")
              {}
  
          // path components only
          path_decomposition_regex_match("/test.txt", None)
              {
                  'ext': '.txt',
                  'subdir': ['/']
                  'subpath': ['/'],
                  'path': '/',
                  'basename': 'test',
              }
  
      """
      pp = path_decomposition(test_str)
  
      # regular expression not specified
      # just path
      if compiled_regex is None:
          return pp
  
      rr = regex_match_str(test_str, compiled_regex)
  
      # regular expression match failed
      # nothing
      if rr == False:
          return {}
  
      #
      #   regular expression matches override file decomposition values in
      #       case of clashes between predefined keys such as "basename" and
      #       regular expression named capture groups
      #
      pp.update(rr)
      return pp
  
  
  #_________________________________________________________________________________________
  #
  #   check_compiled_regexes
  #
  #_________________________________________________________________________________________
  def check_compiled_regexes (compiled_regexes, expected_num):
      """
      check compiled_regexes are of the right type and number
      """
      if compiled_regexes is None:
          return [None] * expected_num
  
      if not isinstance(compiled_regexes, list):
          raise Exception("Expecting list of None and strings")
  
      #   pad compiled_regexes with None
      if len(compiled_regexes) < expected_num:
          compiled_regexes.extend([None] * (expected_num - len(compiled_regexes)))
  
      #   Turn strings to regular expression just in case
      #   We don't want to do this here because the error messages are not very nice:
      #   There is not much context left
      compiled_regexes = [re.compile(rr) if isinstance(rr, path_str_type) else rr for rr in compiled_regexes]
  
      #   check types
      regex_types = type(re.compile("")), type(None)
      for rr in compiled_regexes:
          if not isinstance(rr, regex_types):
              raise Exception("Unexpected type %s ('%s') specified in regular expression list. Expecting string or compiled regular expression" % (type(rr), rr))
  
      return compiled_regexes
  
  
  #_________________________________________________________________________________________
  #
  #   get_all_paths_components
  #
  #_________________________________________________________________________________________
  def get_all_paths_components(paths, compiled_regexes):
      """
          For each path in a list,
      """
      #
      #   merge regular expression matches and path decomposition
      #
      compiled_regexes = check_compiled_regexes (compiled_regexes, len(paths))
      return [path_decomposition_regex_match (pp, rr) for (pp, rr) in zip(paths, compiled_regexes)]
  
  
  
  #_________________________________________________________________________________________
  #
  #   apply_func_to_sequence
  #
  #_________________________________________________________________________________________
  def apply_func_to_sequence(seq, func, tuple_of_conforming_types = (path_str_type,), tuple_of_sequences_types = (list, tuple,set)):
      """
      Recurses into list/tuple/set sequences to apply func to conforming types
      Non-conforming types are left alone
      """
      if isinstance(seq, tuple_of_conforming_types):
          return func(seq)
      elif isinstance(seq, tuple_of_sequences_types):
          return type(seq)(apply_func_to_sequence(pp, func, tuple_of_conforming_types, tuple_of_sequences_types) for pp in seq)
      else:
          return seq
  
  
  #_________________________________________________________________________________________
  #
  #   t_regex_replace
  #
  #_________________________________________________________________________________________
  class t_regex_replace(object):
      def __init__ (self, filename, regex_str, compiled_regex, regex_or_suffix):
          self.regex_or_suffix = regex_or_suffix
          self.compiled_regex = compiled_regex
          self.regex_str = regex_str
          self.filename = filename
      def __call__(self, p):
          #
          #   check if substitution pattern is mis-specified
          #
          if "\1"in p or "\2" in p :
              raise error_unescaped_regular_expression_forms("['%s'] "  % (p.replace("\1", r"\1").replace("\2", r"\2")) +
                                                             "The special regular expression characters "
                                                             r"\1 and \2 need to be 'escaped' in python. "
                                                             r"The easiest option is to use python 'raw' strings "
                                                             r"e.g. r'\1_in_a string\2'. See http://docs.python.org/library/re.html.")
          #
          #   For suffix(), replaces the suffix part by adding leading r"\1" to the substitution pattern
          #
          #   If r"\1" is specified, then we presume you know what you are doing...
          #
          if self.regex_or_suffix == SUFFIX_SUBSTITUTE:
              if r"\1" not in p and r"\g<1>" not in p:
                  match_p = r"\g<1>" + p
              else:
                  match_p = p
  
              # throw exception if doesn't match regular expression at all
              (res_str, cnt_replacements) = self.compiled_regex.subn(match_p, self.filename)
              if cnt_replacements == 0:
                  raise error_input_file_does_not_match("File '%s' does not match suffix('%s') and pattern '%s'" % (self.filename, self.regex_str, p))
              return res_str
  
          #
          #   Normal substitution
          #
          #   For suffix(), complete replacement by the specified pattern text
          #           only substitute if r"\1" or r"\g<1>" is specified
          #
          #
          err_str = ""
          try:
              (res_str, cnt_replacements) = self.compiled_regex.subn(p, self.filename)
              if cnt_replacements > 0:
                  return res_str
          except re.error:
              exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
              err_str = str(exceptionValue)
              raise fatal_error_input_file_does_not_match("File '%s' does not match regex('%s') and pattern '%s':\n\t%s\n" % (self.filename, self.regex_str, p, err_str))
          except IndexError:
              exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
              err_str = str(exceptionValue)
              raise fatal_error_input_file_does_not_match("File '%s' does not match regex('%s') and pattern '%s':\n\t%s\n" % (self.filename, self.regex_str, p, err_str))
  
          #except (re.error, IndexError):
              #err_str = str(sys.exc_info()[1]),
  
          raise error_input_file_does_not_match("File '%s' does not match regex('%s') and pattern '%s'\n%s\n" % (self.filename, self.regex_str, p, err_str))
  
  #_________________________________________________________________________________________
  #
  #   t_formatter_replace
  #
  #_________________________________________________________________________________________
  class t_formatter_replace(object):
      def __init__ (self, filenames, regex_strings, compiled_regexes = None):
          self.filenames = filenames
          # get the full absolute, normalised paths
          filenames = [os.path.abspath(f) for f in filenames]
          self.path_regex_components = get_all_paths_components(filenames, compiled_regexes)
          self.display_regex_strings = parameter_list_as_string(regex_strings)
  
  
  
      def __call__(self, p):
          # swapped nesting order makes the syntax easier to explain:
          #   The first level of indirection is always the path component
          #   So basename[0] is the file name for the first file
          #   This looks better than the normal 0[basename]
  
          # some contortions because format decodes {0} as an offset into a list and not not a lookup into a dict...
          dl, dd = swap_nesting_order(self.path_regex_components)
          try:
              return p.format(*dl, **dd)
          except (KeyError, IndexError):
              raise error_input_file_does_not_match("Field '%s' in ('%s') using formatter(%s) fails to match Files '%s'."
                                                    "."
                                                    % (   str(sys.exc_info()[1]),
                                                          p,
                                                          self.display_regex_strings,
                                                          self.filenames))
  
  
  #_________________________________________________________________________________________
  #
  #   t_nested_formatter_replace
  #
  #_________________________________________________________________________________________
  class t_nested_formatter_replace(object):
      """
      Like  t_formatter_replace but with one additional level of nesting
      I.e. everything is a list comprehension!
      For combinatorics @decorators
      """
      def __init__ (self, filenames, regex_strings, compiled_regexes):
          # make sure that we have the same level of nestedness for regular expressions and file names etc.
          if len(filenames) != len(regex_strings) or len(filenames) != len(compiled_regexes):
              raise Exception("Logic Error.")
          self.filenames = filenames
          # get the full absolute, normalised paths
          filenames = [[os.path.abspath(f) for f in filegroups] for filegroups in filenames]
          self.path_regex_components = [get_all_paths_components(f, r) for (f,r) in zip(filenames, compiled_regexes)]
          self.display_regex_strs = [parameter_list_as_string(ss) for ss in regex_strings]
  
  
      def __call__(self, p):
          # swapped nesting order makes the syntax easier to explain:
          #   The first level of indirection is always the path component
          #   So basename[0] is the file name for the first file
          #   This looks better than the normal 0[basename]
  
          # some contortions because format decodes {0} as an offset into a list and not not a lookup into a dict...
          dl, dd = swap_doubly_nested_order(self.path_regex_components)
          try:
              return p.format(*dl, **dd)
          except (KeyError, IndexError):
              formatter_str = ", ".join("formatter(%s)" % ss for ss in self.display_regex_strs)
              raise error_input_file_does_not_match("Unmatched field %s in ('%s') using %s fails to match Files '%s'"
                                                    "."
                                                    % (   str(sys.exc_info()[1]),
                                                          p,
                                                          formatter_str,
                                                          self.filenames))
  
  
  #_________________________________________________________________________________________
  #
  #   regex_replace
  #
  #_________________________________________________________________________________________
  
  #
  #   Perform normal regular expression substitution
  #
  REGEX_SUBSTITUTE               = 0
  #
  #   An extra peculiar mode to help suffix along:
  #   Suffix regular expression have an implicit capture for everything up to the specified
  #       suffix text
  
  #
  #   By default, replaces the suffix part by adding leading r"\1" to the substitution pattern
  #       If r"\1" is already specified in the pattern, then we presume you know what
  #       you are doing, and will let you get along with it
  #
  SUFFIX_SUBSTITUTE              = 1
  
  #
  #   REGEX_SUBSTITUTE is used for suffix() matches in 'extra' arguments (additional to output)
  #   which are strings
  #
  #   Complete replacement happens. If you wish to retain the prefix text
  #       before the suffix, you can do so by adding r"\1"
  #
  
  def regex_replace(filename, regex_str, compiled_regex, substitution_patterns, regex_or_suffix = REGEX_SUBSTITUTE):
      return apply_func_to_sequence(substitution_patterns, t_regex_replace(filename, regex_str, compiled_regex, regex_or_suffix))
  
  def formatter_replace (filenames, regex_str, compiled_regex, substitution_patterns):
      return apply_func_to_sequence(substitution_patterns, t_formatter_replace(filenames, regex_str, compiled_regex))
  
  def nested_formatter_replace (filenames, regex_strings, compiled_regexes, substitution_patterns):
      return apply_func_to_sequence(substitution_patterns, t_nested_formatter_replace(filenames, regex_strings, compiled_regexes))
  
  
  #_________________________________________________________________________________________
  
  #   non_str_sequence
  
  #_________________________________________________________________________________________
  def non_str_sequence (arg):
      """
      Whether arg is a sequence.
      We treat strings / dicts however as a singleton not as a sequence
  
      """
      #will only dive into list and set, everything else is not regarded as a sequence
      #loss of flexibility but more conservative
      #if (isinstance(arg, (basestring, dict, multiprocessing.managers.DictProxy))):
      if (not isinstance(arg, (list, tuple, set))):
          return False
      try:
          test = iter(arg)
          return True
      except TypeError:
          return False
  
  #_________________________________________________________________________________________
  
  #   get_strings_in_flattened_sequence_aux
  
  #       helper function for next function
  
  #_________________________________________________________________________________________
  def get_strings_in_flattened_sequence_aux(p, l = None):
      """
      Unravels arbitrarily nested sequence and returns lists of strings
      """
      if l is None:
          l = []
      if isinstance(p, path_str_type):
          l.append(p)
      elif non_str_sequence (p):
          for pp in p:
              get_strings_in_flattened_sequence_aux(pp, l)
      return l
  
  
  #_________________________________________________________________________________________
  
  #   non_str_sequence
  
  #_________________________________________________________________________________________
  def get_strings_in_flattened_sequence (p):
      """
      Traverses nested sequence and for each element, returns first string encountered
      """
      if p is None:
          return []
  
      #
      #  string is returned as list of single string
      #
      if isinstance(p, path_str_type):
          return [p]
  
      #
      #  Get all strings flattened into list
      #
      return get_strings_in_flattened_sequence_aux(p)
  
  
  #_________________________________________________________________________________________
  
  #   get_first_string_in_nested_sequence
  
  #_________________________________________________________________________________________
  def get_first_string_in_nested_sequence (p):
      strings = get_strings_in_flattened_sequence (p)
      if len(strings):
          return strings[0]
      return None
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   Encoders: turn objects and filenames into a more presentable format
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  def ignore_unknown_encoder(obj):
      if non_str_sequence (obj):
          return "[%s]" % ", ".join(map(ignore_unknown_encoder, obj))
      try:
          s= str(obj)
          if " object" in s and s[0] == '<' and s[-1] == '>':
              pos = s.find(" object")
              s = "<" + s[1:pos].replace("__main__.", "") + ">"
          return s.replace('"', "'")
      except:
          return "<%s>" % str(obj.__class__).replace('"', "'")
  
  #_________________________________________________________________________________________
  #
  #   shorten_filenames_encoder
  #________________________________________________________________________________________
  def shorten_filenames_encoder (obj, n_levels = 2):
      """
      Convert a set of parameters into a string
          Paths with > N levels of nested-ness are truncated
      """
  
      #
      #   if < 0, nest by 2
      #
      if n_levels < 0:
          desired_len = - n_levels
          prev_encoded_len = 0
          #
          #   try more and more nestedness up to 9 if that fits inside desired length
          #       stop when increasing nestedness makes no difference
          #
          for nestedness in range(1, 20):
              res = shorten_filenames_encoder (obj, nestedness)
              if len(res) > desired_len or "..." not in res:
                  break
              prev_encoded_len = len(res)
          desired_len = max(4, desired_len - 5)
          offset = len(res) - desired_len
          if offset < 0:
              return res
          return "<???> " + res[offset:]
  
  
  
      #
      #   Recurse into lists and tuples
      #
      if non_str_sequence (obj):
          return "[%s]" % ", ".join(map(shorten_filenames_encoder, obj, [n_levels] * len(obj)))
  
      #
      #   Only shorten strings
      #
      if not isinstance(obj, path_str_type):
          return ignore_unknown_encoder(obj)
  
  
      #
      #   level = 0 means return full absolute path
      #
      if not n_levels:
          return ignore_unknown_encoder(os.path.abspath(obj))
  
      #
      # Shorten both relative and absolute (full) paths
      #
  
      # if within bounds, return that
      if obj[1:].count('/') < n_levels:
          return ignore_unknown_encoder(obj)
  
      # use relative path if that has <= nested level
      rel_path = os.path.relpath(obj)
      if rel_path.count('/') <= n_levels:
          #print >>sys.stderr, "relative path only one nested level"
          return ignore_unknown_encoder(rel_path)
  
      # get last N nested levels
      #print >>sys.stderr, "full path last N nested level"
      return ignore_unknown_encoder(get_nth_nested_level_of_path (obj, n_levels))
  
  
  
  
  
  #_________________________________________________________________________________________
  #
  #   get_tasks_filename_globs_in_nested_sequence
  #
  #________________________________________________________________________________________
  glob_letters = set('*[]?')
  def is_glob(s):
      """Check whether 's' contains ANY of glob chars"""
      return len(glob_letters.intersection(s)) > 0
  
  
  
  #_________________________________________________________________________________________
  #
  #   get_nested_tasks_or_globs
  #
  #________________________________________________________________________________________
  def get_nested_tasks_or_globs(p, treat_strings_as_tasks = False, runtime_data_names=None, tasks=None, globs = None):
      """
      Get any tasks or globs which are within parameter
          tasks are returned as functions or function names
      """
      #
      # create storage if this is not a recursive call
      #
      if globs is None:
          runtime_data_names, tasks, globs = set(), list(), set()
  
      #
      #   task function
      #
      if (isinstance(p, collections.Callable)):
          tasks.append(p)
      elif p.__class__.__name__ == 'Task' or p.__class__.__name__ == 'Pipeline':
          tasks.append(p)
      elif isinstance(p, runtime_parameter):
          runtime_data_names.add(p)
  
      #
      #   output_from treats all arguments as tasks or task names
      #
      elif isinstance(p, output_from):
          for pp in p.args:
              get_nested_tasks_or_globs(pp, True, runtime_data_names, tasks, globs)
  
      elif isinstance(p, path_str_type):
          if treat_strings_as_tasks:
              tasks.append(p)
          elif is_glob(p):
              globs.add(p)
  
      elif non_str_sequence (p):
          for pp in p:
              get_nested_tasks_or_globs(pp, treat_strings_as_tasks, runtime_data_names, tasks, globs)
      return tasks, globs, runtime_data_names
  
  #_________________________________________________________________________________________
  #
  #   replace_placeholders_with_tasks_in_input_params
  #
  #________________________________________________________________________________________
  def replace_placeholders_with_tasks_in_input_params(p, func_or_name_to_task, treat_strings_as_tasks = False):
      """
      Replaces task functions or task name (strings) with the tasks they represent
      Also replaces Tasks and Pipelines with the correct Tasks
      func_or_name_to_task are a dictionary of function and task names to tasks
  
      """
      if p.__class__.__name__ == 'Pipeline':
          return func_or_name_to_task["PIPELINE=%s=PIPELINE" % p.name]
  
      if p.__class__.__name__ == 'Task' and p in func_or_name_to_task:
          return func_or_name_to_task[p]
  
      #
      # Expand globs or tasks as a list only if they are top level
      #
      if isinstance(p, collections.Callable):
      #if type(p) == types.FunctionType:
          return func_or_name_to_task[p]
  
      #
      #   output_from treats all arguments as tasks or task names
      #
      if isinstance(p, output_from):
          if len(p.args) == 1:
              return replace_placeholders_with_tasks_in_input_params(p.args[0], func_or_name_to_task, True)
          else:
              return [replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, True) for pp in p.args]
  
      #
      # strings become tasks if treat_strings_as_tasks
      #
      if isinstance(p, path_str_type):
          if treat_strings_as_tasks:
              return func_or_name_to_task[p]
          return p
  
      #
      # No conversions within dictionaries
      #
      if isinstance(p, dict):
          return p
  
  
      #
      # Other sequences are recursed down
      #
      elif non_str_sequence(p):
          l = list()
          for pp in p:
  
              #
              #   To be intuitive:
              #   arguments wrapped by output_from are always treated "in-line"
              #           e.g. 1, output_from("a") => 1, task_a
              #           e.g. 1, output_from("a", 2) => 1, task_a, 2
              #
              if isinstance(pp, output_from):
                  if len(pp.args) > 1:
                      l.extend(tuple(replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, True)))
                  elif len(pp.args) == 1:
                      l.append(replace_placeholders_with_tasks_in_input_params(pp.args[0], func_or_name_to_task, True))
                  # else len(pp.args) == 0 !! do nothing
  
              else:
                  l.append(replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, treat_strings_as_tasks))
          return type(p)(l)
  
      #
      # No expansions of non-string/non-sequences
      #
      else:
          return p
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   compiling regular expressions
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  #_________________________________________________________________________________________
  
  #   suffix
  
  #_________________________________________________________________________________________
  class suffix(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'suffix%r' % (self.args,)
  
  #_________________________________________________________________________________________
  
  #   regex
  
  #_________________________________________________________________________________________
  class regex(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'regex%r' % (self.args,)
  
  #_________________________________________________________________________________________
  
  #   regex
  
  #_________________________________________________________________________________________
  class formatter(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'formatter%r' % (self.args,)
  
  #_________________________________________________________________________________________
  
  #   wrap_exception_as_string
  
  #_________________________________________________________________________________________
  def wrap_exception_as_string ():
      """
      return exception as string to be rethrown
      """
      exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
      msg = "%s.%s" % (exceptionType.__module__, exceptionType.__name__)
      exception_value  = str(exceptionValue)
      if len(exception_value):
          return msg + ": (%s)" % exception_value
      return msg
  
  
  #_________________________________________________________________________________________
  
  #   compile_formatter
  
  #_________________________________________________________________________________________
  def compile_formatter(enclosing_task, formatter_obj, error_object, descriptor_string):
      """
      Given list of [string|None]
      Return compiled regular expressions.
      """
  
      compiled_regexes = []
      for ss in formatter_obj.args:
          # ignore None
          if ss is None:
              compiled_regexes.append(None)
              continue
  
          formatter_args = str(formatter_obj.args)[1:-1]
          # regular expression should be strings
          if not isinstance(ss, path_str_type):
              raise error_object(enclosing_task, ("{descriptor_string}: "
                                         "formatter({formatter_args}) is malformed\n"
                                          "formatter(...) should only be used to wrap "
                                          'regular expression strings or None (not "{ss}")')
                                          .format(descriptor_string = descriptor_string,
                                                  formatter_args = formatter_args,
                                                  ss = ss)
                                 )
  
          try:
              compiled_regexes.append(re.compile(ss))
          except:
              raise error_object(enclosing_task, ("{descriptor_string}: "
                                          "in formatter({formatter_args}) \n"
                                         'regular expression "{ss}" is malformed\n'
                                          "[{except_str}]")
                                          .format(descriptor_string = descriptor_string,
                                                  formatter_args = formatter_args,
                                                  ss = ss,
                                                  except_str = wrap_exception_as_string())
                             )
      return compiled_regexes
  
  
  
  #_________________________________________________________________________________________
  
  #   compile_regex
  
  #_________________________________________________________________________________________
  def compile_regex(enclosing_task, regex, error_object, descriptor_string, regex_object_name = "regex"):
      """
      throw error unless regular expression compiles
      """
      if not len(regex.args) or len(regex.args) > 1 or not isinstance(regex.args[0], path_str_type):
          regex_str = str(regex.args)
          if len(regex.args) > 1:
              regex_str = regex_str[1:-1]
          elif len(regex.args) == 0:
              regex_str = ''
          else:
              regex_str = regex_str
          raise error_object(enclosing_task, ("{descriptor_string}: "
                                     "{regex_object_name}({regex_str}) is malformed\n"
                                      "{regex_object_name}(...) should only be used to wrap a single regular expression string")
                                      .format(descriptor_string = descriptor_string,
                                              regex_str = regex_str,
                                              regex_object_name = regex_object_name)
                             )
      try:
          matching_regex = re.compile(regex.args[0])
          return matching_regex
      except:
          raise error_object(enclosing_task, ("{descriptor_string}: "
                                     "regular expression {regex_object_name}('{regex_str}') is malformed\n"
                                      "[{except_str}]")
                                      .format(descriptor_string = descriptor_string,
                                              regex_object_name = regex_object_name,
                                              regex_str = regex.args[0],
                                              except_str = wrap_exception_as_string())
                             )
  
  #_________________________________________________________________________________________
  
  #   compile_suffix
  
  #_________________________________________________________________________________________
  def compile_suffix(enclosing_task, regex, error_object, descriptor_string):
      """
      throw error unless regular expression compiles
      """
      if not len(regex.args):
          raise error_object(enclosing_task, "%s: " % descriptor_string +
                                      "suffix() is malformed.\n" +
                                       "suffix(...) should be used to wrap a string matching the suffices of file names")
      if len(regex.args) > 1 or not isinstance(regex.args[0], path_str_type):
          raise error_object(enclosing_task, "%s: " % descriptor_string +
                                     "suffix('%s') is malformed.\n" % (regex.args,) +
                                      "suffix(...) should only be used to wrap a single string matching the suffices of file names")
      try:
          matching_regex = re.compile("(.*)" + re.escape(regex.args[0]) + "$")
          return matching_regex
      except:
          raise error_object(enclosing_task, "%s: " % descriptor_string +
                                     "suffix('%s') is somehow malformed\n" % regex.args[0] +
                                      "[%s]" %  wrap_exception_as_string())
  
  #_________________________________________________________________________________________
  
  #   check_parallel_parameters
  
  #_________________________________________________________________________________________
  def check_parallel_parameters (enclosing_task, params, error_object):
      """
      Helper function for @files
      Checks format of parameters and
      whether there are input and output files specified for each job
      """
      if not len(params):
          raise Exception("@parallel parameters is empty.")
  
      for job_param in params:
          if isinstance(job_param, path_str_type):
              message = ("Wrong syntax for @parallel.\n"
                          "@parallel(%s)\n" % ignore_unknown_encoder(params) +
                          "If you are supplying parameters for a task "
                          "running as a single job, "
                          "either don't put enclosing brackets at all (with each parameter "
                          "separated by commas) or enclose all parameters as a nested list of "
                          "lists, e.g. [['input', 'output' ...]]. "
                          )
              raise error_object(enclosing_task, message)
  
  
  
  #_________________________________________________________________________________________
  
  #   check_files_io_parameters
  
  #_________________________________________________________________________________________
  def check_files_io_parameters (enclosing_task, params, error_object):
      """
      Helper function for @files
      Checks format of parameters and
      whether there are input and output files specified for each job
      """
      #if not len(params):
      #    raise Exception("@files I/O parameters is empty.")
  
      try:
          for job_param in params:
              if isinstance(job_param, path_str_type):
                  raise TypeError
  
              if len(job_param) < 1:
                  raise error_object(enclosing_task, "Missing input files for job " +
                                                      ignore_unknown_encoder(job_param))
              if len(job_param) < 2:
                  raise error_object(enclosing_task, "Missing output files for job " +
                                                      ignore_unknown_encoder(job_param))
              #if len(get_strings_in_flattened_sequence(job_param[0:2])) == 0:
              #    raise error_object(enclosing_task, "Input or output file parameters should "
              #                                        "contain at least one or more file names strings. "
              #                                        "Consider using @parallel if you are not using files. " +
              #                                        ignore_unknown_encoder(job_param))
      except TypeError:
          #
          # job_param was not a list
          #
          message = ("Wrong syntax for @files.\n@files(%s)\n" % ignore_unknown_encoder(params) +
                      "If you are supplying parameters for a task "
                      "running as a single job, "
                      "either don't put enclosing brackets at all (with each parameter "
                      "separated by commas) or enclose all parameters as a nested list of "
                      "lists, e.g. [['input', 'output' ...]]. "
                      )
          raise error_object(enclosing_task, message)
  
  #_________________________________________________________________________________________
  #
  #   expand_nested_tasks_or_globs
  #
  #________________________________________________________________________________________
  def expand_nested_tasks_or_globs(p, tasksglobs_to_filenames):
      """
      Expand globs and tasks "in-line", unless they are the top level, in which case turn
      it into a list
  
      N.B. Globs are only expanded if they are in tasksglobs_to_filenames
           This function is called for @split descriptors which leave output globs untouched
           for clarity. Thanks to Noah Spies for spotting this!
      """
  
      #
      # Expand globs or tasks as a list only if they are top level
      #
      if (    (isinstance(p, path_str_type) and is_glob(p) and p in tasksglobs_to_filenames) or
              p.__class__.__name__ == 'Task'     or
              isinstance(p, runtime_parameter)    ):
          return tasksglobs_to_filenames[p]
  
      #
      # No expansions of strings and dictionaries
      #
      if isinstance(p, (path_str_type, dict)):
          return p
  
      #
      # Other sequences are recursed down
      #
      elif non_str_sequence(p):
          l = list()
          for pp in p:
              if (isinstance(pp, path_str_type) and pp in tasksglobs_to_filenames):
                  l.extend(tasksglobs_to_filenames[pp])
              elif pp.__class__.__name__ == 'Task' or isinstance(pp, runtime_parameter):
                  files = tasksglobs_to_filenames[pp]
                  # task may have produced a single output: in which case append
                  if non_str_sequence(files):
                      l.extend(files)
                  else:
                      l.append(files)
              else:
                  l.append(expand_nested_tasks_or_globs(pp, tasksglobs_to_filenames))
          return type(p)(l)
  
      #
      # No expansions of non-string/non-sequences
      #
      else:
          return p
  
  
  
  #_________________________________________________________________________________________
  
  #   get_parsed_arguments_str_for_errors
  
  #       helper funciton for parse_task_arguments()
  
  #_________________________________________________________________________________________
  def get_parsed_arguments_str_for_errors (task_description, bad_arg_str, unnamed_result_strs, named_result_strs):
      """
      Helper function for parse_task_arguments
          Prints out offending argument (bad_arg_str) in the context of already parsed
          arguments so that we can quickly figure out where the error is coming from
      """
      indent = task_description.find("(") + 1
      parsed_arg_str = ", ".join(unnamed_result_strs + named_result_strs)
      # make function names clearer in arg list
      parsed_arg_str = re.sub(r"<function (\w+) at 0x[0-9a-f]+>", r"\1", parsed_arg_str)
      return task_description %  (parsed_arg_str + ", ...\n" +
                                  # mark out problem
                                  (" " * (indent-5 if indent - 5 > 0 else 0)) + "===> " +
                                  bad_arg_str)
  
  
  
  #_________________________________________________________________________________________
  
  #   parse_task_arguments
  
  #_________________________________________________________________________________________
  def parse_task_arguments ( orig_unnamed_arguments, orig_named_arguments, expected_arguments, task_description):
      """
      Parse arguments parsed into decorators or Pipeline.transform etc.
          Special handling for optional arguments in the middle of argument list
              1) @product
                  can have (input, filter, input1, filter1, input2, filter2....)
              2) @transform, @subdivide, @collate, @product, @combinatorics which have
                      (..., [add_inputs(...)|inputs(...)],...)
                      or ([add_inputs=...|replace_inputs=...])
                      or ([add_inputs=add_inputs(...)|replace_inputs=inputs(...)])
          Special handling for variable number of arguments at the end of the argument list
              which all become "extras"
  
      #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
      #
      #   N.B. Missing non-mandatory arguments are returned as an empty list
      #
      #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  
                  """
      results = {}
      unnamed_arguments   = list(orig_unnamed_arguments)
      named_arguments     = dict(orig_named_arguments)
      # parsed results in string form for error messages
      unnamed_result_strs = []
      named_result_strs = []
  
  
      #________________________________________________________________________________________
      #
      #   parse_add_inputs()
      #
      #________________________________________________________________________________________
      def parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, result_strs):
          """
          Parse arguments for add_inputs and replace_inputs, i.e. 'inputs()' and 'add_inputs()'
              input_type =inputs|add_inputs
              arg_name = replace_inputs | add_inputs
              modify_inputs_mode = t_extra_inputs.REPLACE_INPUTS| t_extra_inputs.ADD_TO_INPUTS
          """
          results["modify_inputs_mode"] = modify_inputs_mode
          if input_type == inputs:
              # inputs() only takes a single argument. Throw error otherwise
              if len(parsed_arg.args) != 1:
                  err_msg = "inputs() expects a single argument:\n%s" % (
                              get_parsed_arguments_str_for_errors(task_description,   # bad arg in context of parsed
                                                                  "%s%r" % (input_type.__name__, tuple(parsed_arg.args)),
                                                                  unnamed_result_strs,
                                                                  named_result_strs))
                  #print (err_msg, file=sys.stderr)
                  raise error_inputs_multiple_args(err_msg)
  
          # unpack add_inputs / inputs and save results
              results["modify_inputs"] = parsed_arg.args[0]
          else:
              results["modify_inputs"] = parsed_arg.args
          result_strs.append("%s=%r" % (arg_name, parsed_arg.args))
  
  
  
      #________________________________________________________________________________________
      #
      #   check_argument_type
      #
      #________________________________________________________________________________________
      def check_argument_type (arg_name, parsed_arg, argument_types):
          """
          check if parsed_arg is right type
          """
          if argument_types and not isinstance(parsed_arg, argument_types):
              err_msg = ("The '%s' argument should be %s:\n%s" %
                          (arg_name,                                                  # argument name
                          " or ".join("%s" % tt.__name__ for tt in argument_types),   # type names
                          get_parsed_arguments_str_for_errors(task_description,       # bad arg in context of parsed
                                                              "%s = %r" % (arg_name, parsed_arg),
                                                              unnamed_result_strs, named_result_strs)))
              #print (err_msg, file=sys.stderr)
              raise TypeError(err_msg)
  
          return parsed_arg
      #________________________________________________________________________________________
      #
      #   parse_argument
      #       helper function for parsing a single arguement
      #________________________________________________________________________________________
      def parse_argument (arg_name, expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory, argument_types = None):
          """
          All missing, non-mandatory are empty list
          """
  
          # ignore if not on list
          if not arg_name in expected_arguments:
              return
  
          #
          # look among unnamed arguments first
          #
          if len(unnamed_arguments):
              #
              # check correct type
              #
              parsed_arg = check_argument_type (arg_name, unnamed_arguments[0], argument_types)
              #
              #   Save parsed results
              #
              results[arg_name] = parsed_arg
              unnamed_result_strs.append("%s=%r" % (arg_name, parsed_arg))
              del unnamed_arguments[0]
  
          #
          # then named
          #
          elif arg_name in named_arguments:
              #
              # check correct type
              #
              parsed_arg = check_argument_type (arg_name, named_arguments[arg_name], argument_types)
              #
              #   Save parsed results
              #
              results[arg_name] = parsed_arg
              named_result_strs.append("%s=%r" % (arg_name, parsed_arg))
              del named_arguments[arg_name]
  
          #
          # complain or ignore missing?
          #
          else:
              if mandatory:
                  err_msg = "Missing '%s' argument:\n%s" % (
                              arg_name,                                               # argument name
                              get_parsed_arguments_str_for_errors(task_description,   # bad arg in
                                                                  arg_name + " = ???",#   context of parsed
                                                                  unnamed_result_strs,
                                                                  named_result_strs))
                  #print (err_msg, file=sys.stderr)
                  raise error_missing_args(err_msg)
  
              #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
              #
              #   N.B. Missing non-mandatory arguments are returned as an empty list
              #
              #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
              else:
                  results[arg_name] = []
  
      #
      #   Missing input is empty list
      #
      parse_argument ('input', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = False)
  
      #
      #   filter is mandatory if expected
      #
      parse_argument ('filter', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True, argument_types = (formatter, regex, suffix))
  
      if "filter" in results:
          if isinstance(results["filter"], suffix):
              parse_argument ("output_dir", expected_arguments, [], named_arguments, results, task_description, mandatory = False)
  
  
  
      #
      #   inputN
      #
      if 'inputN' in expected_arguments:
          #
          # put already parsed input and filter into the list
          #
          results["input"] = [results["input"]]
          results["filter"] = [results["filter"]]
          cnt_inputN = 1
          #
          #   parse argument pairs at a time, so long as the second argument is a formatter
          #
          while len(unnamed_arguments) >= 2 and isinstance(unnamed_arguments[1], formatter):
              filter_name = "filter%d"    % (cnt_inputN + 1)
              input_name  = "input%d"     % (cnt_inputN + 1)
              unnamed_result_strs.append("%s=%r" % (input_name, unnamed_arguments[0]))
              unnamed_result_strs.append("%s=%r" % (filter_name, unnamed_arguments[1]))
              results["input"].append(unnamed_arguments[0])
              results["filter"].append(unnamed_arguments[1])
              cnt_inputN += 1
              del unnamed_arguments[0:2]
  
          #
          #   parse named arguments while there is a filter2, filter3 etc.
          #
          filter_name = "filter%d"    % (cnt_inputN + 1)
          input_name  = "input%d"     % (cnt_inputN + 1)
          while filter_name in named_arguments:
              results["filter"].append(named_arguments[filter_name])
              named_result_strs.append("%s=%r" % (filter_name, named_arguments[filter_name]))
              del named_arguments[filter_name]
              #   parse input2, input3 or leave blank as empty list
              if input_name in named_arguments:
                  results["input"].append(named_arguments[input_name])
                  named_result_strs.append("%s=%r" % (input_name, named_arguments[input_name]))
                  del named_arguments[input_name]
              else:
                  results["input"].append([])
              cnt_inputN += 1
              filter_name = "filter%d"    % (cnt_inputN + 1)
              input_name  = "input%d"     % (cnt_inputN + 1)
  
  
      #
      #   tuple size is int and mandatory if exists
      #
      parse_argument ('tuple_size', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True, argument_types = (int,))
  
      #
      #   add_inputs / inputs are optional
      #
      if 'modify_inputs' in expected_arguments:
          results["modify_inputs_mode"] = t_extra_inputs.KEEP_INPUTS
          results["modify_inputs"] = None
          parse_add_inputs = ((inputs, "inputs", "replace_inputs", t_extra_inputs.REPLACE_INPUTS), (add_inputs, "add_inputs", "add_inputs", t_extra_inputs.ADD_TO_INPUTS))
          if len(unnamed_arguments):
              #
              #   Is add_inputs or inputs in unnamed arguments?
              #       Parse out contents and save in results["replace_inputs"] or results["add_inputs"]
              #
              for input_type, input_type_name, arg_name, modify_inputs_mode in parse_add_inputs:
                  parsed_arg = unnamed_arguments[0]
                  if isinstance(parsed_arg, input_type):
                      parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, unnamed_result_strs)
                      del unnamed_arguments[0]
                      break
          #
          #   Otherwise is add_inputs or inputs in named arguments?
          #       Parse out contents only if necessary and save in results["replace_inputs"] or results["add_inputs"]
          #
          if results["modify_inputs_mode"] == t_extra_inputs.KEEP_INPUTS:
              for input_type, input_type_name, arg_name, modify_inputs_mode in parse_add_inputs:
                  if arg_name in named_arguments:
                      parsed_arg = named_arguments[arg_name]
                      if isinstance(parsed_arg, input_type):
                          parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, named_result_strs)
                      else:
                          results["modify_inputs"] = parsed_arg
                      results["modify_inputs_mode"] = modify_inputs_mode
                      del named_arguments[arg_name]
                      break
  
  
      #
      #   output is mandatory if exists
      #
      parse_argument ('output', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True)
  
      #
      #   extras is mandatory if exists
      #
      if 'extras' in expected_arguments:
          if len(unnamed_arguments):
              # move list to results: remember python does shallow copies of lists
              results['extras'] = unnamed_arguments
              unnamed_result_strs.append("%s=%r" % ("extras", unnamed_arguments))
              unnamed_arguments = []
              #del unnamed_arguments[:]
          elif 'extras'  in named_arguments:
              results['extras' ] = named_arguments['extras' ]
              named_result_strs.append("%s=%r" % ("extras", named_arguments['extras' ]))
              del named_arguments['extras' ]
          else:
              results['extras' ] = []
  
  
      if len(unnamed_arguments):
          err_msg = ("Too many unnamed arguments leftover:\n%s" %
                      get_parsed_arguments_str_for_errors(task_description,       # bad arg in context of parsed
                                                          (", ".join(("%r" % a) for a in unnamed_arguments)),
                                                          unnamed_result_strs, named_result_strs))
          #print (err_msg, file=sys.stderr)
          raise error_too_many_args(err_msg)
      if len(named_arguments):
          err_msg = ("Duplicate, conflicting or unrecognised arguments:\n%s" %
                      get_parsed_arguments_str_for_errors(task_description,       # bad arg in context of parsed
                                                          ", ".join("%s=%r" %(k, v) for k,v in named_arguments.items()),
                                                          unnamed_result_strs, named_result_strs))
          #print (err_msg, file=sys.stderr)
          raise error_too_many_args(err_msg)
  
  
      return results
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   special markers used by @files_re
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  class combine(object):
      def __init__ (self, *args):
          self.args = args
  
  class output_from(object):
      def __init__ (self, *args):
          self.args = args
      def __repr__ (self, *args):
          return 'output_from%r' % (self.args,)
  
  class runtime_parameter(object):
      def __init__ (self, *args):
          if len(args) != 1 or not isinstance(args[0], path_str_type):
              raise Exception("runtime_parameter takes the name of the run time parameter as a single string")
          self.args = args
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  #   Unit Testing code in test/test_ruffus_utility.py
  
  
  #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  
  