

'''simple docstring'''






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__SCREAMING_SNAKE_CASE     =						logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE     =						{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}



def 			__a		(       lowerCAmelCase__ :    str     ,  lowerCAmelCase__ :    List[str]     ,  lowerCAmelCase__ :    Dict     ,  lowerCAmelCase__ :    str      ):



     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()      )}.'      )

     if tokenizer_name is None:
          a__   :      List[str]     =						TOKENIZER_CLASSES
     else:
          a__   :      Dict     =						{tokenizer_name: getattr(lowerCAmelCase__     ,  tokenizer_name + '''Fast'''      )}

     logger.info(F'Loading tokenizer classes: {tokenizer_names}'      )

     for tokenizer_name in tokenizer_names:
          a__   :      Tuple     =						TOKENIZER_CLASSES[tokenizer_name]

          a__   :      Optional[int]     =						True
          if checkpoint_name is None:
               a__   :      str     =						list(tokenizer_class.max_model_input_sizes.keys()      )
          else:
               a__   :      Union[str, Any]     =						[checkpoint_name]

          logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'      )

          for checkpoint in checkpoint_names:
               logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'      )

               # Load tokenizer
               a__   :      List[str]     =						tokenizer_class.from_pretrained(lowerCAmelCase__     ,  force_download=lowerCAmelCase__      )

               # Save fast tokenizer
               logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'      )

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    a__     ,    a__   :      List[Any]     =						checkpoint.split('''/'''      )
                    a__   :      int     =						os.path.join(lowerCAmelCase__     ,  lowerCAmelCase__      )
               elif add_prefix:
                    a__   :      Any     =						checkpoint
                    a__   :      Optional[int]     =						dump_path
               else:
                    a__   :      Optional[Any]     =						None
                    a__   :      List[str]     =						dump_path

               logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'      )

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()      )[0]:
                    a__   :      Optional[Any]     =						list(tokenizer.pretrained_vocab_files_map.values()      )[0][checkpoint]
                    a__   :      List[str]     =						file_path.split(lowerCAmelCase__      )[-1][0]
                    if next_char == "/":
                         a__   :      Any     =						os.path.join(lowerCAmelCase__     ,  lowerCAmelCase__      )
                         a__   :      List[str]     =						None

                    logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'      )

               a__   :      Union[str, Any]     =						tokenizer.save_pretrained(
                   lowerCAmelCase__     ,  legacy_format=lowerCAmelCase__     ,  filename_prefix=lowerCAmelCase__      )
               logger.info(F'=> File names {file_names}'      )

               for file_name in file_names:
                    if not file_name.endswith('''tokenizer.json'''      ):
                         os.remove(lowerCAmelCase__      )
                         logger.info(F'=> removing {file_name}'      )


if __name__ == "__main__":
 __SCREAMING_SNAKE_CASE     =						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
 )
 parser.add_argument(
     '--tokenizer_name',
     default=None,
     type=str,
     help=(
         f'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
         'download and convert all the checkpoints from AWS.'
     ),
 )
 parser.add_argument(
     '--checkpoint_name',
     default=None,
     type=str,
     help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
 )
 parser.add_argument(
     '--force_download',
     action='store_true',
     help='Re-download checkpoints.',
 )
 __SCREAMING_SNAKE_CASE     =						parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

def 							UpperCAmelCase		(       lowercase				):







				"""simple docstring"""



				if not isinstance(lowercase				,     lowercase				):
								raise ValueError('''Input must be an integer'''				)
				if input_num <= 0:
								raise ValueError('''Input must be positive'''				)
				return sum(
				    divisor for divisor in range(1				,     input_num // 2 + 1				) if input_num % divisor == 0				)


if __name__ == "__main__":
		import doctest

		doctest.testmod()