text: # -*- coding: utf-8 -*-
  """ This is the runner module that collects config information and runs the
  initial code
  """
  from __future__ import print_function
  import click
  import logging
  import sys
  import os
  import os.path
  import sys
  import getopt
  import hashlib
  import io
  import re
  
  # Setup Logging
  
  try:  # Python 2.7+
      from logging import NullHandler
  except ImportError:
      class NullHandler(logging.Handler):
          """ This is a dummy NullHandler implementation in-case NullHandler class not found"""
          def emit(self, record):
              pass
  
  logging.getLogger(__name__).addHandler(NullHandler())
  
  LOGGER = logging.getLogger(__name__)
  
  def eprint(*args, **kwargs):
      """ prints to stderr, using the 'from __future__ import print_function' """
      print(*args, file=sys.stderr, **kwargs)
  
  __version__ = '1.7.2'
  __author__ = 'Steve Morin'
  __script_name__ = 'filegardener'
  
  # Done setting up logging
  
  def print_version(ctx, param, value):
      """ Prints the version of this software"""
      # pylint: disable=unused-argument
      if not value:
          return
      click.echo('Version '+__version__)
      ctx.exit()
  
  CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help', '-?'])
  
  # This makes it so when there is no subcommand --help isn't automatically called
  @click.group(invoke_without_command=False, context_settings=CONTEXT_SETTINGS)
  @click.option('--debug/--no-debug', '-d', default=False, envvar='BS_DEBUG',
                help='turn on/off debug mode')
  @click.option('--version', is_flag=True, callback=print_version, expose_value=False, is_eager=True,
                help='print programs version')
  @click.pass_context
  def cli(ctx, debug):
      """ For help on individual commands type:
  
  \b
          filegardener <command> --help
      """
      ctx.obj = dict(debug=debug)
      verbosity = None # currently not set or used
      configure_logger(debug,verbosity)
  
      if ctx.invoked_subcommand is None:
          raise Exception("Runtime Error")
          # This should never happen because invoke_without_command=False, to allow this to be called set invoke_without_command=True
      else:
          pass
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('checkdir', nargs=-1, required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.pass_obj
  def countfiles(ctx, checkdir):
      """
      countfiles command counts the number of files in the directories you give it
      """
  
      return_value = count_files(ctx, checkdir)
  
      click.echo(return_value)
  
  def count_files(ctx, checkdir):
      innercount = 0
      LOGGER.debug("checkdir: %s" % checkdir)
      if type(checkdir) == str:
          raise Exception("Wrong input type should be a list")
      for mydir in checkdir:
          for dirpath, dirnames, files in os.walk(mydir):
              for filename in files:
                  LOGGER.debug("filename: %s" % filename)
                  innercount = innercount + 1
      return innercount
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('checkdir', nargs=-1, required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.pass_obj
  def countdirs(ctx, checkdir):
      """
      countdirs command counts the number of directories in the directories you give it (excludes dirs you give it)
      """
      return_value = count_dirs(ctx, checkdir)
  
      click.echo(return_value)
      
  def count_dirs(ctx, checkdir):
      """ count_dirs does the actual counting of the directories """
      innercount = 0
      if type(checkdir) == str:
          raise Exception("Wrong input type should be a list")
      for mydir in checkdir:
          once = False
          for dirpath, dirnames, files in os.walk(mydir):
              LOGGER.debug(dirpath)
              once = True
              innercount = innercount + 1
          if once:
              innercount = innercount - 1 # decrement by one so it excludes counting itself
      return innercount
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('file', nargs=-1, required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.option('--basedir', '-b', default=False, help='base directory to join each file path to', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--exitonfail/--no-exitonfail', '-e', default=False, help='turn on/off exit on first failure')
  @click.pass_obj
  def rmfiles(ctx, file, basedir, exitonfail):
      """
      rmfiles will delete a set of files listed in the input file(s)
      """    
      if not (type(file) == list or type(file) == tuple):
          raise Exception("Wrong input type should be a list: %s" % type(file))
      
      result = True
      reason = None
      failed_once = False
      
      for file_name in file:
          with io.open(file_name, "r", encoding="utf8") as f:
              for line in f:
                  file_path = line.rstrip()
                  if basedir:
                      file_path = os.path.abspath(os.path.join(basedir, file_path))
                  try:
                      result, reason = rmfile(file_path)
                  except OSError as excep:
                      result = False
                      reason = str(excep)
                  if not result:
                      click.echo("%s\t%s" % (file_path, reason))
                      failed_once = True
                      if exitonfail:
                          sys.exit(1)
  
      if failed_once:
          sys.exit(1)
  
  def rmfile(file_path):
      result = False
      reason = None
      if validate_test_file(file_path):
          os.remove(file_path) # tmp don't remove files till tested
          result = True
      else:
          result = False
          reason = "wasn't a file"
      return result, reason
      
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('file', nargs=-1, required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.option('--basedir', '-b', default=False, help='base directory to join each file path to', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--exitonfail/--no-exitonfail', '-e', default=False, help='turn on/off exit on first failure')
  @click.pass_obj
  def rmdirs(ctx, file, basedir, exitonfail):
      """
      rmdirs will delete a set of dirs listed in the input file(s)
      """
      if not (type(file) == list or type(file) == tuple):
          raise Exception("Wrong input type should be a list: %s" % type(file))
      
      result = True
      reason = None
      failed_once = False
      
      
      for file_name in file:
          with io.open(file_name, "r", encoding="utf8") as f:
              all_dirs = f.readlines()
              all_dirs = sorted(all_dirs, key=len, reverse=True) # so long child dirs get removed first
              for line in all_dirs:
                  file_path = line.rstrip()
                  if basedir:
                      file_path = os.path.abspath(os.path.join(basedir, file_path))
                  try:
                      result, reason = rmdir(file_path)
                  except OSError as excep:
                      result = False
                      reason = str(excep)
                  if not result:
                      click.echo("%s\t%s" % (file_path, reason))
                      failed_once = True
                      if exitonfail:
                          sys.exit(1)
      if failed_once:
          sys.exit(1)
  
  def rmdir(dir_path):
      result = False
      reason = None
      if validate_test_dir(dir_path):
          os.rmdir(dir_path) # tmp don't remove files till tested
          result = True
      else:
          reason = "wasn't a directory"
      return result, reason
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('destdir', nargs=1, required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--basedir', '-b', default=False, help='base directory to join each file path to', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--targetdir', '-b', default=False, help='location to move all files from', required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--file', '-f', default=False, help='file for input files', required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.pass_obj
  def mvbase(ctx, destdir, basedir, file):
      """
      mvbase will move a set of files from their locations, at target directory to destdir
      """
      click.echo("TODO: not implemented yet")
      sys.exit(1)
      failed = False
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('file', nargs=-1, required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.option('--basedir', '-b', default=False, help='base directory to join each file path to', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--exitonfail/--no-exitonfail', '-e', default=False, help='turn on/off exit on first failure')
  @click.pass_obj
  def validatefiles(ctx, file, basedir, exitonfail):
      """
      validatefiles reads in a file of file paths and checks that it exists
      """
      Result, Count = validate_files(file, basedir, exitonfail)
      if not Result:
          sys.exit(1)
      else:
          sys.exit(0)
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('file', nargs=-1, required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.option('--basedir', '-b', default=False, help='base directory to join each file path to', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--exitonfail/--no-exitonfail', '-e', default=False, help='turn on/off exit on first failure')
  @click.pass_obj
  def validatedirs(ctx, file, basedir, exitonfail):
      """
      validatedirs reads in a file of dir paths and checks that it exists and passes test
      """
      Result, Count = validate_dirs(file, basedir, exitonfail)
      if not Result:
          sys.exit(1)
      else:
          sys.exit(0)
  
  def validate_files(file, basedir, exitonfail):
      return validate_paths(file, basedir, exitonfail, validate_test_file)
  
  def validate_dirs(file, basedir, exitonfail):
      return validate_paths(file, basedir, exitonfail, validate_test_dir)
  
  def validate_paths(file_paths, basedir, exitonfail, path_tester):
      failed = False
      count = 0
      if not type(file_paths) == list:
          raise Exception("Wrong input type should be a list")
  
      for failed_path in validatepath_yield(file_paths, basedir, path_tester):
          failed = True
          count = count + 1
          click.echo(failed_path)
          if failed and exitonfail:
              return False, count
      if failed:
          return False, count
      else:
          return True, count
  
  def validatepath_yield(files, basedir, path_tester):
      for file_name in files:
          with io.open(file_name, "r", encoding="utf8") as f:
              for line in f:
                  file_path = line.rstrip()
                  if basedir:
                      file_path = os.path.abspath(os.path.join(basedir, file_path))
                  if not path_tester(file_path):
                      yield file_path
  
  def validate_test_file(file_path):
      if os.path.isfile(file_path):
          return True
      else:
          return False
  
  def validate_test_dir(dir_path):
      if os.path.isdir(dir_path):
          return True
      else:
          return False
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.argument('checkdir', nargs=-1, required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--relpath/--no-relpath', '-r', default=False, help='turn on/off relative path - default off')
  @click.pass_obj
  def emptydirs(ctx, checkdir, relpath):
      """
      emptydir command lists all the directories that no file in it or it's sub directories
      """
      if relpath:
          basepath = os.getcwd()
          for i in emptydirs_yield(checkdir):
              click.echo(os.path.relpath(i, basepath))
      else:
          for i in emptydirs_yield(checkdir):
              click.echo(i)
  
  def get_parent_dir(mydir):
  	return os.path.abspath(os.path.join(mydir,os.path.pardir))
  
  def emptydirs_yield(checkdir):
      """
      emptydirs command prints list of directories in one or more checkdirs
      """
      # http://stackoverflow.com/questions/19699127/efficient-array-concatenation
      for mydir in checkdir:
          is_leaf = False
          is_empty = False
          parent_dict = {}
          my_parent = None
          abs_dirpath = None
          for dirpath, dirnames, files in os.walk(mydir, topdown=False):
              # if ctx['debug']: # debug needs to be defined
              #     LOGGER.debug("%s %s %s" % (dirpath, dirnames, files))
  
              if len(dirnames) == 0:
                  is_leaf = True
              else:
                  is_leaf = False
  
              if len(files) == 0:
                  if is_leaf:
                      is_empty = True
                  else:
                      is_empty = parent_dict[os.path.abspath(dirpath)][0]
              else:
                  is_empty = False
  
              my_parent = get_parent_dir(dirpath)
  
              if my_parent in parent_dict:
                  entry = parent_dict[my_parent]
                  parent_dict[my_parent] = [is_empty and entry[0], entry[1] + 1]
              else:
                  parent_dict[my_parent] = [is_empty,  1]
              
              if is_leaf:
                  if is_empty:
                      yield dirpath
              else:
                  abs_dirpath = os.path.abspath(dirpath)
                  if abs_dirpath in parent_dict:
                      entry = parent_dict[abs_dirpath]
                      del parent_dict[abs_dirpath] # because your traversing the dirs with topdown=False it should have already seen all children
                      if len(dirnames) != entry[1]:
                          raise Exception("Entries don't match the number of dirs, so a subdir wasn't checked.")
                      else:
                          if entry[0] and is_empty:
                              yield abs_dirpath
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.option('--srcdir', '-s', multiple=True, required=True, help='directories to check',  type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.argument('checkdir', nargs=-1, required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--relpath/--no-relpath', '-r', default=False, help='turn on/off relative path - default off')
  @click.pass_obj
  def dedup(ctx, srcdir, checkdir, relpath):
      """
      Dedup command prints list of duplicate files in one or more checkdirs
      """
      if relpath:
          basepath = os.getcwd()
          for i in dedup_yield(srcdir, checkdir):
              click.echo(os.path.relpath(i, basepath))
      else:
          for i in dedup_yield(srcdir, checkdir):
              click.echo(i)
      
  def dedup_yield(srcdir, checkdir):
      """
      Dedup command prints list of duplicate files in one or more checkdirs
      """
      check_dir_no_overlap(srcdir, checkdir)
  
      # http://stackoverflow.com/questions/19699127/efficient-array-concatenation
  
      basefiles = []
      for mydir in srcdir:
          innerdir = get_files_and_size_from_dir(mydir)
          basefiles.extend(innerdir)
  
      size_dict = create_size_dict(basefiles)
  
      for comparedir in checkdir:
          dup_files = get_duplicate_files(size_dict, comparedir)
  
          LOGGER.debug("Length %s" % len(dup_files))
          for thefile in dup_files:
              yield thefile
  
  # TODO: add file size distribution function
  
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.pass_obj
  def hello(ctx):
      print('world')
  
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.pass_obj
  def test(ctx):
      innerdir = get_files_and_size_from_dir('/Users/user', regex=None)
  
      with open('/Users/user/test_fg', 'w') as f:
          for tup in innerdir:
              f.write("{}:{}\n".format(tup[1], tup[0]))
  
      second_innerdir = create_tuple_list('/Users/user/test_fg')
  
  
  def create_tuple_list(file_name):
      tup_list = []
      with open('/Users/user/test_fg', 'r') as f:
          for line in f:
              line = line.strip()
              list_result = line.split(":", 2)
              list_result.reverse()
              tup_list.append(tuple(list_result))
      return tup_list
  
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.option('--srcdir', '-s', multiple=True, required=True, help='directories to check',
                type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--relpath/--no-relpath', '-r', default=False, help='turn on/off relative path - default off')
  @click.option('--failonerror/--no-failonerror', '-f', default=True, help='turn on/off failing on error - default on')
  @click.option('--include-regex', default=None, type=click.STRING,
                help='use this regex on src and dest files to test if they should be included if they match') # add callback for validation
  @click.option('--include-src-regex', default=None, type=click.STRING,
                help='use this regex on src files to test if they should be included if they match') # add callback for validation
  @click.option('--include-dst-regex', default=None, type=click.STRING,
                help='use this regex on dst files to test if they should be included if they match') # add callback for validation
  @click.pass_obj
  def srcfile(ctx, srcdir, relpath, failonerror, include_regex, include_src_regex, include_dst_regex):
      """
      srcfile prints list of all the file size and paths.  You can use this to speed up processing, if your
      using the same src file sets multiple times.  Instead feed the file you can create from this, but sometimes
      100 times faster.
      
      format:
          size1:file_path1
          size2:file_path2
      """
      # TODO: Test and write unit tests
  
      # validate the regex options
      check_regex(include_regex)
      if include_regex is not None:
          include_src_regex = include_regex
          include_dst_regex = include_regex
      check_regex(include_src_regex)
      check_regex(include_dst_regex)
  
      if relpath:
          basepath = os.getcwd()
          for i in srcfile_yield(srcdir, failonerror, src_regex=include_src_regex, dst_regex=include_dst_regex):
              click.echo("{}:{}".format(i[1], os.path.relpath(i[0], basepath)))
      else:
          for i in srcfile_yield(srcdir, failonerror, src_regex=include_src_regex, dst_regex=include_dst_regex):
              click.echo("{}:{}".format(i[1], i[0]))
  
  
  def srcfile_yield(srcdir, failonerror=True, src_regex=None, dst_regex=None):
      """
      sizepath will return a tuple at a time (path, size)
      """
      # TODO: Test and write unit tests
  
      # http://stackoverflow.com/questions/19699127/efficient-array-concatenation
  
      basefiles = []
      for mydir in srcdir:
          innerdir = get_files_and_size_from_dir(mydir, regex=src_regex)
          basefiles.extend(innerdir)
  
      for tup in basefiles:
          yield tup
  
  @cli.command(context_settings=CONTEXT_SETTINGS)
  @click.option('--srcdir', '-s', multiple=True, required=False, help='directories to check',
                type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--srcfile', '-f', multiple=True, required=False, help='files of sizes and paths',
                type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
  @click.argument('checkdir', nargs=-1, required=True,
                  type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True))
  @click.option('--relpath/--no-relpath', '-r', default=False, help='turn on/off relative path - default off')
  @click.option('--failonerror/--no-failonerror', '-f', default=True, help='turn on/off failing on error - default on')
  @click.option('--include-regex', default=None, type=click.STRING,
                help='use this regex on src and dest files to test if they should be included if they match') # add callback for validation
  @click.option('--include-src-regex', default=None, type=click.STRING,
                help='use this regex on src files to test if they should be included if they match') # add callback for validation
  @click.option('--include-dst-regex', default=None, type=click.STRING,
                help='use this regex on dst files to test if they should be included if they match') # add callback for validation
  @click.pass_obj
  def onlycopy(ctx, srcdir, srcfile, checkdir, relpath, failonerror, include_regex, include_src_regex, include_dst_regex):
      """
      onlycopy command prints list of all the files that aren't in the srcdir
      """
  
      if not (srcdir or srcfile):
          raise click.BadOptionUsage("The regex didn't compile. For correct usage"
                                     " see:  https://docs.python.org/2/library/re.html")
      # validate the regex options
      check_regex(include_regex)
      if include_regex is not None:
          include_src_regex = include_regex
          include_dst_regex = include_regex
      check_regex(include_src_regex)
      check_regex(include_dst_regex)
  
      if relpath:
          basepath = os.getcwd()
          for i in onlycopy_yield(srcdir, checkdir, failonerror, src_regex=include_src_regex, dst_regex=include_dst_regex,
                                  srcfile=srcfile):
              click.echo(os.path.relpath(i, basepath))
      else:
          for i in onlycopy_yield(srcdir, checkdir, failonerror, src_regex=include_src_regex, dst_regex=include_dst_regex,
                                  srcfile=srcfile):
              click.echo(i)
  
  
  def check_regex(regexstring):
      """ method to check that the regex works"""
      if regexstring is not None:
          try:
              compiledregex = re.compile(regexstring, flags=re.IGNORECASE)
              # result = compiledregex.match(string)
          except:
              raise click.BadOptionUsage("The regex didn't compile. For correct usage"
                                         " see:  https://docs.python.org/2/library/re.html")
  
  
  def onlycopy_yield(srcdir, checkdir, failonerror=True, src_regex=None, dst_regex=None, srcfile=None):
      """
      onlycopy command prints list of all the files that aren't in the srcdir
      """
  
      if srcdir is None and srcfile is None:
          raise ValueError("srcdir or srcfile must be supplied to onlycopy_yield")
  
      if srcdir is not None:
          check_dir_no_overlap(srcdir, checkdir)
      # Skip check on srcfile, too many entries to check, so assume it's correct
      # http://stackoverflow.com/questions/19699127/efficient-array-concatenation
  
      basefiles = []
      if srcdir is not None:
          for mydir in srcdir:
              # return a list of tuples with (path, size)
              innerdir = get_files_and_size_from_dir(mydir, regex=src_regex)
              basefiles.extend(innerdir)
  
      if srcfile is not None:
          for myfile in srcfile:
              # return a list of tuples with (path, size)
              innerdir = get_files_and_size_from_file(myfile, regex=src_regex)
              basefiles.extend(innerdir)
  
      size_dict = create_size_dict(basefiles)
  
      for comparedir in checkdir:
          only_files = get_only_copy(size_dict, comparedir, failonerror, regex=dst_regex)
  
          LOGGER.debug("Length %s" % len(only_files))
          for thefile in only_files:
              yield thefile
  
  
  def check_dir_no_overlap(srcdir, checkdir):
      for x in checkdir:
          for y in srcdir:
              if x == y:
                  click.echo("source and checkdir are the same, they must be different")
                  sys.exit(1)
              elif x.startswith(y) or y.startswith(x):
                  click.echo("source and checkdir are a subdirectory of one or the other source(%s), checkdir(%s)" % (x, y) )
                  sys.exit(1)    
  
  
  def get_files_and_size_from_dir(topdir, regex=None):
      """ Finds all the files under a specific directory. Returns a list of the absolute path for
      all of these files."""
      if regex is not None:
          compiled_regex = re.compile(regex, flags=re.IGNORECASE)
  
          def is_re_match(file_path):
              if compiled_regex.match(file_path):
                  return True
              else:
                  return False
      else:
  
          def is_re_match(file_path):
              return True
  
      if not os.path.isdir(topdir):
          raise Exception("You submitted a director that isn't one'")
      files = [(os.path.abspath(os.path.join(dirpath, filename)), os.path.getsize(os.path.join(dirpath, filename)))
               for dirpath, dirnames, files in os.walk(topdir)
               for filename in files if not os.path.islink(os.path.join(dirpath, filename))
               and is_re_match(os.path.abspath(os.path.join(dirpath, filename)))
               ]
      return files
  
  
  def create_is_string_fn():
      is_unicode_class = True
      try:
          unicode
      except:
          is_unicode_class = False
  
      is_string = None
  
      if is_unicode_class:
          def is_string_fn(my_string):
              return isinstance(my_string, str) # or isinstance(my_string, unicode)
  
          is_string = is_string_fn
      else:
          def is_string_fn(my_string):
              return isinstance(my_string, str)
  
          is_string = is_string_fn
  
      return is_string
  
  is_string = create_is_string_fn()
  
  
  def create_convert_to_unicode_fn():
      is_unicode_class = True
      try:
          unicode
      except:
          is_unicode_class = False
      convert_string = None
  
      if is_unicode_class:
          def convert_string_fn(my_string):
              return my_string.decode()
  
          convert_string = convert_string_fn
      else:
          def convert_string_fn(my_string):
              return my_string
  
          convert_string = convert_string_fn
      return convert_string
  
  convert_to_unicode = create_convert_to_unicode_fn()
  
  
  def get_files_and_size_from_file(sizepathfile, regex=None):
      """ Finds all the files listed in a srcfile. Returns a list of the absolute path for
      all of these files.  This should be a list of tuples [(path1, size1), (path2, size2)]"""
      if regex is not None:
          compiled_regex = re.compile(regex, flags=re.IGNORECASE)
  
          def is_re_match(file_path):
              if compiled_regex.match(file_path):
                  return True
              else:
                  return False
      else:
  
          def is_re_match(file_path):
              return True
  
      if not os.path.isfile(sizepathfile):
          raise Exception("You submitted a file that isn't one'")
  
      tuple_list = []
      i = 0
      import codecs
      with codecs.open(sizepathfile, encoding='utf-8', errors='strict') as f:
          for line in f:
              i = i + 1
              line_string = line.rstrip()  # remove new lines from each line
              size, path = line_string.split(':', 1)
              path = os.path.abspath(os.path.join(os.getcwd(), path))
              # needs to be put here because abs_path_fn functions convert back to str
              if is_string(path):
                  try:
                      path = convert_to_unicode(path)
                  except UnicodeDecodeError as e:
                      eprint("UnicodeDecodeError Line:{}, File:{}".format(i, sizepathfile))
                      raise e
                  except UnicodeEncodeError as e:
                      eprint("UnicodeEncodeError Line:{}, File:{}".format(i, sizepathfile))
                      raise e
              # make sure it passes regex if any
              if is_re_match(path):
                  tuple_list.append((path, int(size)))
      return tuple_list
  
  
  def create_size_dict(files_size):
      size_dict = {}
      for file, size in files_size:
          if size in size_dict:
              size_dict[size].append(file)
          else:
              size_dict[size] = [file]
      return size_dict
  
  # Example usage of os.walk
  # topdir = '.'
  # all_paths = [ (dirpath, dirnames, files) for dirpath, dirnames, files in os.walk(topdir) ]
  #
  # for item in all_paths:
  #     print(item)
  # ('./test_dedup/1dup/seconddir/tests', ['testserver'], ['__init__.py', 'compat.py', 'conftest.py', 'test_hooks.py', 'test_lowlevel.py', 'test_requests.py', 'test_structures.py', 'test_testserver.py', 'test_utils.py', 'utils.py'])
  # ('./test_dedup/1dup/seconddir/tests/testserver', [], ['__init__.py', 'server.py'])
  # ('./test_dedup/identialdirs', ['firstdir', 'seconddir'], [])
  
  
  def get_duplicate_files(size_dict, topdir):
      """
      if_match_return_true=True means find duplicates
      if_match_return_true=False means find if there is only one copy of it.
      """
          
      files = []
      
      if_match_return_true = True
  
      files = [os.path.abspath(os.path.join(dirpath, filename))  # return absolute path of file that matches criteria
                  for dirpath, dirnames, files in os.walk(topdir) 
                      for filename in files 
                          if not os.path.islink(os.path.join(dirpath, filename)) and
                          os.path.getsize(os.path.join(dirpath, filename)) in size_dict
                          and is_match(os.path.getsize(os.path.join(dirpath, filename)),  # size
                          os.path.abspath(os.path.join(dirpath, filename)),  # file
                          size_dict[os.path.getsize(os.path.join(dirpath, filename))],  # file_list
                          if_match_return_true)]  # if_match_return_value
      return files  
  
  
  def get_only_copy(size_dict, topdir, failonerror=True, regex=None):
      """
      if_match_return_true=True means find duplicates
      if_match_return_true=False means find if there is only one copy of it.
      """
      
      files = []
  
      if regex is not None:
          compiled_regex = re.compile(regex, flags=re.IGNORECASE)
  
          def is_re_match(file_path):
              if compiled_regex.match(file_path):
                  return True
              else:
                  return False
      else:
  
          def is_re_match(file_path):
              return True
      
      if_match_return_true = False
      
      # The logic is for all the directories and for each file in each directory
      # test the following if is_not_symlink and (not in size or is_not_match )
      files = [os.path.abspath(os.path.join(dirpath, filename))  # return absolute path of file that matches criteria
               for dirpath, dirnames, files in os.walk(topdir)
               for filename in files
               # TODO: possibly pull os.path functions into a delegate to add try except block to be fault tolerant
               if not os.path.islink(os.path.join(dirpath, filename))
               and is_re_match(os.path.abspath(os.path.join(dirpath, filename)))
               and (os.path.getsize(os.path.join(dirpath, filename))
                    not in size_dict
                    or is_match(os.path.getsize(os.path.join(dirpath, filename)),  # size
                                os.path.abspath(os.path.join(dirpath, filename)),  # file
                                size_dict[os.path.getsize(os.path.join(dirpath, filename))],  # file_list
                                if_match_return_true,  # if_match_return_value
                                failonerror)
                    )]  # failonerror
      return files 
    
  
  def is_match(size, file, file_list, if_match_return_value, is_error_fatal=True):
      """
      if_match_return_value=True means find duplicates
      if_match_return_value=False means find not duplicates
      is_error_fatal=True
      
      Algorithm:
      files being compared are the same size
      
      if file is less than 4096 bytes
          just compare if bytes of source file and file_list are the same
      if file is 4096 bytes or greater
          for the source file and each file in the checklist
              compare first 4k of source file and check file
              if match check md5 of each file
      
      """
      if size < 4096:
          try:
              return compare_whole_file(size, file, file_list, if_match_return_value, is_error_fatal)
          except IOError as myioerror:
              if is_error_fatal:
                  raise myioerror
              else:
                  eprint()
              
      else:
          for file_to_compare in file_list:
              try:
                  if compare_first_4k(file, file_to_compare):
                      # TODO: Optimize by saving the md5 for each check if a list
                      if compare_md5(file, file_to_compare):
                          return if_match_return_value
              except IOError as myioerror:
                  if is_error_fatal:
                      raise myioerror
                  else:
                      eprint("{}:{}".format(file, file_to_compare))
          return not if_match_return_value
  
  def compare_whole_file(size, file, file_list, if_match_return_value, is_error_fatal):
      file_bytes = b''
      file_to_compare_bytes = b''
      try:
          with open(file,"rb") as f:
              file_bytes = f.read(size)
              if len(file_bytes) != size:
                  raise Exception("Did not read the expected file size")
      except IOError as myioerror:
          if is_error_fatal:
              raise myioerror
          else:
              eprint("{}".format(file))
      
      for file_to_compare in file_list:
          try:
              with open(file_to_compare,"rb") as f:
                  file_to_compare_bytes = f.read(size)
                  if len(file_to_compare_bytes) != size:
                      raise IOError("Filegardener - compare_whole_file: Did not read the expected file size:{}".format(size))
          except IOError as myioerror:
              if is_error_fatal:
                  raise myioerror
              else:
                  eprint("{}:{}".format(file, file_to_compare))
          if file_bytes == file_to_compare_bytes:
              return if_match_return_value
      return not if_match_return_value
  
  def compare_first_4k(file, file_to_compare):
      '''
          Should compare the first 4k of each file to see if they match
      '''
      file_bytes = b''
      file_to_compare_bytes = b''
      with open(file,"rb") as f:
          file_bytes = f.read(4096)
          if len(file_bytes) != 4096:
              raise IOError("Filegardener - compare_first_4k: Did not read the expected file size:{}".format(4096))
      with open(file_to_compare,"rb") as f:
          file_to_compare_bytes = f.read(4096)
          if len(file_bytes) != 4096:
              raise IOError("Filegardener - compare_first_4k: Did not read the expected file size:{}".format(4096))
      if file_bytes == file_to_compare_bytes:
          return True
      else:
          return False
  
  def compare_md5(file, file_to_compare):
      '''
          compares the md5 to see if there is a match
  
          Block size directly depends on the block size of your filesystem
          to avoid performances issues
          Here I have blocks of 4096 octets (Default NTFS)
      '''
      # TODO I have to check that block size thing
      md5_base = None
      md5_to_compare = None
  
      # find out blocksize
      # echo “hello” > blocksize.txt
      # du -h blocksize.txt
  
      block_size=256*128*2 # 256*128*2=65536
      md5 = hashlib.md5()
      with open(file,'rb') as f:
          for chunk in iter(lambda: f.read(block_size), b''):
              md5.update(chunk)
          md5_base = md5.hexdigest()
  
      block_size=256*128*2 # 256*128*2=32768
      md5 = hashlib.md5()
      with open(file_to_compare,'rb') as f:
          for chunk in iter(lambda: f.read(block_size), b''):
              md5.update(chunk)
          md5_to_compare = md5.hexdigest()
      
      if md5_base == md5_to_compare:
          return True
      else:
          return False
          
          
  def configure_logger(debug_on, verbosity):
      """ Configure the top level logger with sensible defaults that can be configured
      with a configuration file
      """
      # TODO: make logger accept configuration for config location and log destination
      if debug_on or verbosity:
          log_level = logging.WARN
  
          if verbosity == 1:
              log_level = logging.INFO
          elif verbosity == 2:
              log_level = logging.DEBUG
  
          if debug_on:
              log_level = logging.DEBUG
  
          toplevel_logger = logging.getLogger(__script_name__)
          toplevel_logger.setLevel(log_level)
  
          # create console handler and set level to debug
          strhdlr = logging.FileHandler('%s.log' % __script_name__)
          strhdlr.setLevel(log_level)
  
          # create formatter
          formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  
          # add formatter to strhdlr
          strhdlr.setFormatter(formatter)
  
          # add strhdlr to logger
          toplevel_logger.addHandler(strhdlr)
  
  # This allows the function "def cli" above to be easily called because of click library
  if __name__ == '__main__':
      # pylint: disable=no-value-for-parameter
      cli()
  
  