"""simple docstring"""


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

UpperCAmelCase__  		=						logging.get_logger(__name__)


UpperCAmelCase__  		=						{name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}






def     __UpperCAmelCase					(			lowercase   ,lowercase   ,lowercase   ,lowercase		):







  """simple docstring"""
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(f'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()		)}.'''		)

  if tokenizer_name is None:
    _UpperCAmelCase						 =      TOKENIZER_CLASSES
  else:
    _UpperCAmelCase						 =      {tokenizer_name: getattr(lowercase   ,tokenizer_name + """Fast"""		)}

  logger.info(f'''Loading tokenizer classes: {tokenizer_names}'''		)

  for tokenizer_name in tokenizer_names:
    _UpperCAmelCase						 =      TOKENIZER_CLASSES[tokenizer_name]

    _UpperCAmelCase						 =      True
    if checkpoint_name is None:
      _UpperCAmelCase						 =      list(tokenizer_class.max_model_input_sizes.keys()		)
    else:
      _UpperCAmelCase						 =      [checkpoint_name]

    logger.info(f'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''		)

    for checkpoint in checkpoint_names:
      logger.info(f'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''		)

      # Load tokenizer
      _UpperCAmelCase						 =      tokenizer_class.from_pretrained(lowercase   ,force_download=lowercase		)

      # Save fast tokenizer
      logger.info(f'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''		)

      # For organization names we create sub-directories
      if "/" in checkpoint:
        _UpperCAmelCase ,				_UpperCAmelCase						 =      checkpoint.split("""/"""		)
        _UpperCAmelCase						 =      os.path.join(lowercase   ,lowercase		)
      elif add_prefix:
        _UpperCAmelCase						 =      checkpoint
        _UpperCAmelCase						 =      dump_path
      else:
        _UpperCAmelCase						 =      None
        _UpperCAmelCase						 =      dump_path

      logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''		)

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()		)[0]:
        _UpperCAmelCase						 =      list(tokenizer.pretrained_vocab_files_map.values()		)[0][checkpoint]
        _UpperCAmelCase						 =      file_path.split(lowercase		)[-1][0]
        if next_char == "/":
          _UpperCAmelCase						 =      os.path.join(lowercase   ,lowercase		)
          _UpperCAmelCase						 =      None

        logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''		)

      _UpperCAmelCase						 =      tokenizer.save_pretrained(
          lowercase   ,legacy_format=lowercase   ,filename_prefix=lowercase		)
      logger.info(f'''=> File names {file_names}'''		)

      for file_name in file_names:
        if not file_name.endswith("""tokenizer.json"""		):
          os.remove(lowercase		)
          logger.info(f'''=> removing {file_name}'''		)


if __name__ == "__main__":
    UpperCAmelCase__  		=						argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
    )
    parser.add_argument(
        """--tokenizer_name""",
        default=None,
        type=str,
        help=(
            F'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
            """download and convert all the checkpoints from AWS."""
        ),
    )
    parser.add_argument(
        """--checkpoint_name""",
        default=None,
        type=str,
        help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
    )
    parser.add_argument(
        """--force_download""",
        action="""store_true""",
        help="""Re-download checkpoints.""",
    )
    UpperCAmelCase__  		=						parser.parse_args()

    convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

"""simple docstring"""


def     __UpperCAmelCase					(			lowercase		):







  """simple docstring"""
  _UpperCAmelCase						 =      abs(lowercase		)
  _UpperCAmelCase						 =      0
  while n > 0:
    res += n % 10
    n //= 10
  return res






def     __UpperCAmelCase					(			lowercase		):







  """simple docstring"""
  _UpperCAmelCase						 =      abs(lowercase		)
  return n if n < 10 else n % 10 + sum_of_digits(n // 10		)






def     __UpperCAmelCase					(			lowercase		):







  """simple docstring"""
  return sum(int(lowercase		) for c in str(abs(lowercase		)		)		)






def     __UpperCAmelCase					(			):







  """simple docstring"""
  from collections.abc import Callable
  from timeit import timeit

  def benchmark_a_function(lowercase   ,lowercase		) -> None:
    _UpperCAmelCase						 =      f'''{func.__name__}({value})'''
    _UpperCAmelCase						 =      timeit(f'''__main__.{call}'''   ,setup="""import __main__"""		)
    print(f'''{call:56} = {func(lowercase		)} -- {timing:.4f} seconds'''		)

  for value in (26_21_44, 11_25_89_99_06_84_26_24, 1_26_76_50_60_02_28_22_94_01_49_67_03_20_53_76):
    for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
      benchmark_a_function(lowercase   ,lowercase		)
    print()


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()
