



"""simple docstring"""


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__A   :							Dict										=   logging.get_logger(__name__)


__A   :							List[str]										=   {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						Optional[Any]					,     _SCREAMING_SNAKE_CASE   :						int					,     _SCREAMING_SNAKE_CASE   :						Dict					,     _SCREAMING_SNAKE_CASE   :						List[str]				):






				'''simple docstring'''



				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(f'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}.'				)

				if tokenizer_name is None:
								_UpperCAmelCase					=    TOKENIZER_CLASSES
				else:
								_UpperCAmelCase					=    {tokenizer_name: getattr(_SCREAMING_SNAKE_CASE					,     tokenizer_name + '''Fast'''				)}

				logger.info(f'Loading tokenizer classes: {tokenizer_names}'				)

				for tokenizer_name in tokenizer_names:
								_UpperCAmelCase					=    TOKENIZER_CLASSES[tokenizer_name]

								_UpperCAmelCase					=    True
								if checkpoint_name is None:
												_UpperCAmelCase					=    list(tokenizer_class.max_model_input_sizes.keys()				)
								else:
												_UpperCAmelCase					=    [checkpoint_name]

								logger.info(f'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'				)

								for checkpoint in checkpoint_names:
												logger.info(f'Loading {tokenizer_class.__class__.__name__} {checkpoint}'				)

												# Load tokenizer
												_UpperCAmelCase					=    tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE					,     force_download=_SCREAMING_SNAKE_CASE				)

												# Save fast tokenizer
												logger.info(f'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'				)

												# For organization names we create sub-directories
												if "/" in checkpoint:
																_UpperCAmelCase		,						_UpperCAmelCase					=    checkpoint.split('''/'''				)
																_UpperCAmelCase					=    os.path.join(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)
												elif add_prefix:
																_UpperCAmelCase					=    checkpoint
																_UpperCAmelCase					=    dump_path
												else:
																_UpperCAmelCase					=    None
																_UpperCAmelCase					=    dump_path

												logger.info(f'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
																_UpperCAmelCase					=    list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
																_UpperCAmelCase					=    file_path.split(_SCREAMING_SNAKE_CASE				)[-1][0]
																if next_char == "/":
																				_UpperCAmelCase					=    os.path.join(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)
																				_UpperCAmelCase					=    None

																logger.info(f'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

												_UpperCAmelCase					=    tokenizer.save_pretrained(
												    _SCREAMING_SNAKE_CASE					,     legacy_format=_SCREAMING_SNAKE_CASE					,     filename_prefix=_SCREAMING_SNAKE_CASE				)
												logger.info(f'=> File names {file_names}'				)

												for file_name in file_names:
																if not file_name.endswith('''tokenizer.json'''				):
																				os.remove(_SCREAMING_SNAKE_CASE				)
																				logger.info(f'=> removing {file_name}'				)


if __name__ == "__main__":
					__A   :							List[Any]										=   argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
					)
					parser.add_argument(
					    "--tokenizer_name",
					    default=None,
					    type=str,
					    help=(
					        f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
					        "download and convert all the checkpoints from AWS."
					    ),
					)
					parser.add_argument(
					    "--checkpoint_name",
					    default=None,
					    type=str,
					    help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
					)
					parser.add_argument(
					    "--force_download",
					    action="store_true",
					    help="Re-download checkpoints.",
					)
					__A   :							Any										=   parser.parse_args()

					convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)








"""simple docstring"""


def  lowercase	(      _SCREAMING_SNAKE_CASE   :						int					,     _SCREAMING_SNAKE_CASE   :						int				):






				'''simple docstring'''



				if a < 0 or b < 0:
								raise ValueError('''the value of both inputs must be positive'''				)

				_UpperCAmelCase					=    str(bin(_SCREAMING_SNAKE_CASE				)				)[2:]  # remove the leading "0b"
				_UpperCAmelCase					=    str(bin(_SCREAMING_SNAKE_CASE				)				)[2:]  # remove the leading "0b"

				_UpperCAmelCase					=    max(len(_SCREAMING_SNAKE_CASE				)					,     len(_SCREAMING_SNAKE_CASE				)				)

				return "0b" + "".join(
				    str(int(char_a == '''1''' and char_b == '''1'''				)				)
				    for char_a, char_b in zip(a_binary.zfill(_SCREAMING_SNAKE_CASE				)					,     b_binary.zfill(_SCREAMING_SNAKE_CASE				)				)				)


if __name__ == "__main__":
					import doctest

					doctest.testmod()



