


'''simple docstring'''






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__UpperCAmelCase					       =							logging.get_logger(__name__)


__UpperCAmelCase					       =							{name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}


def       _snake_case     (  A				,					A				,					A				,					A )		->					List[str]:
							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys() )}.""" )

							if tokenizer_name is None:
														lowerCAmelCase__      =			TOKENIZER_CLASSES
							else:
														lowerCAmelCase__      =			{tokenizer_name: getattr(A				,					tokenizer_name + '''Fast''' )}

							logger.info(F"""Loading tokenizer classes: {tokenizer_names}""" )

							for tokenizer_name in tokenizer_names:
														lowerCAmelCase__      =			TOKENIZER_CLASSES[tokenizer_name]

														lowerCAmelCase__      =			True
														if checkpoint_name is None:
																					lowerCAmelCase__      =			list(tokenizer_class.max_model_input_sizes.keys() )
														else:
																					lowerCAmelCase__      =			[checkpoint_name]

														logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}""" )

														for checkpoint in checkpoint_names:
																					logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}""" )

																					# Load tokenizer
																					lowerCAmelCase__      =			tokenizer_class.from_pretrained(A				,					force_download=A )

																					# Save fast tokenizer
																					logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}""" )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												lowerCAmelCase__     ,      lowerCAmelCase__      =			checkpoint.split('''/''' )
																												lowerCAmelCase__      =			os.path.join(A				,					A )
																					elif add_prefix:
																												lowerCAmelCase__      =			checkpoint
																												lowerCAmelCase__      =			dump_path
																					else:
																												lowerCAmelCase__      =			None
																												lowerCAmelCase__      =			dump_path

																					logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}""" )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values() )[0]:
																												lowerCAmelCase__      =			list(tokenizer.pretrained_vocab_files_map.values() )[0][checkpoint]
																												lowerCAmelCase__      =			file_path.split(A )[-1][0]
																												if next_char == "/":
																																			lowerCAmelCase__      =			os.path.join(A				,					A )
																																			lowerCAmelCase__      =			None

																												logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}""" )

																					lowerCAmelCase__      =			tokenizer.save_pretrained(
																					    A				,					legacy_format=A				,					filename_prefix=A )
																					logger.info(F"""=> File names {file_names}""" )

																					for file_name in file_names:
																												if not file_name.endswith('''tokenizer.json''' ):
																																			os.remove(A )
																																			logger.info(F"""=> removing {file_name}""" )


if __name__ == "__main__":
		__UpperCAmelCase					       =							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
		)
		parser.add_argument(
		    '''--tokenizer_name''',
		    default=None,
		    type=str,
		    help=(
		        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
		        '''download and convert all the checkpoints from AWS.'''
		    ),
		)
		parser.add_argument(
		    '''--checkpoint_name''',
		    default=None,
		    type=str,
		    help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
		)
		parser.add_argument(
		    '''--force_download''',
		    action='''store_true''',
		    help='''Re-download checkpoints.''',
		)
		__UpperCAmelCase					       =							parser.parse_args()

		convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)



'''simple docstring'''






from manim import *




class 	a__       (							a__						):





							'''simple docstring'''






							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[str]:
														lowerCAmelCase__      =			Rectangle(height=0.5      ,					width=0.5     )
														lowerCAmelCase__      =			Rectangle(height=0.46      ,					width=0.46     ).set_stroke(width=0     )

														lowerCAmelCase__      =			[mem.copy() for i in range(6     )]
														lowerCAmelCase__      =			[mem.copy() for i in range(6     )]
														lowerCAmelCase__      =			VGroup(*lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0     )
														lowerCAmelCase__      =			VGroup(*lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0     )
														lowerCAmelCase__      =			VGroup(lowerCamelCase_      ,					lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0     )
														lowerCAmelCase__      =			Text('''CPU'''      ,					font_size=24     )
														lowerCAmelCase__      =			Group(lowerCamelCase_      ,					lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0.5      ,					aligned_edge=lowerCamelCase_     )
														cpu.move_to([-2.5, -0.5, 0]     )
														self.add(lowerCamelCase_     )

														lowerCAmelCase__      =			[mem.copy() for i in range(1     )]
														lowerCAmelCase__      =			VGroup(*lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0     )
														lowerCAmelCase__      =			Text('''GPU'''      ,					font_size=24     )
														lowerCAmelCase__      =			Group(lowerCamelCase_      ,					lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0.5      ,					aligned_edge=lowerCamelCase_     )
														gpu.align_to(lowerCamelCase_      ,					lowerCamelCase_     )
														gpu.set_x(gpu.get_x() - 1     )

														self.add(lowerCamelCase_     )

														lowerCAmelCase__      =			[mem.copy() for i in range(6     )]
														lowerCAmelCase__      =			VGroup(*lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0     )

														lowerCAmelCase__      =			Text('''Model'''      ,					font_size=24     )
														lowerCAmelCase__      =			Group(lowerCamelCase_      ,					lowerCamelCase_     ).arrange(lowerCamelCase_      ,					buff=0.5      ,					aligned_edge=lowerCamelCase_     )
														model.move_to([3, -1.0, 0]     )

														self.play(
														    Create(lowerCamelCase_      ,					run_time=1     )      ,					Create(lowerCamelCase_      ,					run_time=1     )      ,					Create(lowerCamelCase_      ,					run_time=1     )      ,					)

														lowerCAmelCase__      =			MarkupText(
														    F"""First, an empty model skeleton is loaded\ninto <span fgcolor='{YELLOW}'>memory</span> without using much RAM."""      ,					font_size=24      ,					)

														lowerCAmelCase__      =			Square(side_length=2.2     )
														key.move_to([-5, 2, 0]     )

														lowerCAmelCase__      =			MarkupText(
														    F"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""      ,					font_size=18      ,					)

														key_text.move_to([-5, 2.4, 0]     )

														step_a.move_to([2, 2, 0]     )
														self.play(Write(lowerCamelCase_      ,					run_time=2.5     )      ,					Write(lowerCamelCase_     )      ,					Write(lowerCamelCase_     )     )

														self.add(lowerCamelCase_     )

														lowerCAmelCase__      =			[]
														lowerCAmelCase__      =			[]
														lowerCAmelCase__      =			[]
														for i, rect in enumerate(lowerCamelCase_     ):
																					lowerCAmelCase__      =			Rectangle(height=0.46      ,					width=0.46     ).set_stroke(width=0.0     ).set_fill(lowerCamelCase_      ,					opacity=0.7     )
																					cpu_target.move_to(lowerCamelCase_     )
																					cpu_target.generate_target()
																					lowerCAmelCase__      =			0.46 / 4
																					lowerCAmelCase__      =			0.46 / 3

																					if i == 0:
																												cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT     )      ,					buff=0.02      ,					direction=lowerCamelCase_     )
																												cpu_target.target.set_x(cpu_target.target.get_x() + 0.1     )
																					elif i == 3:
																												cpu_target.target.next_to(cpu_targs[0].target      ,					direction=lowerCamelCase_      ,					buff=0.0     )
																					else:
																												cpu_target.target.next_to(cpu_targs[i - 1].target      ,					direction=lowerCamelCase_      ,					buff=0.0     )
																					cpu_targs.append(lowerCamelCase_     )

																					first_animations.append(rect.animate(run_time=0.5     ).set_stroke(lowerCamelCase_     )     )
																					second_animations.append(MoveToTarget(lowerCamelCase_      ,					run_time=1.5     )     )

														self.play(*lowerCamelCase_     )
														self.play(*lowerCamelCase_     )

														self.wait()