



import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command


class 				_a						(       unittest.TestCase			):





							'''simple docstring'''

							A				:Dict =       inspect.getfile(accelerate.test_utils			)
							A				:Dict =       os.path.sep.join(mod_file.split(os.path.sep			)[:-1] + ["scripts", "test_cli.py"]			)

							A				:Tuple =       ["accelerate", "launch"]
							A				:Union[str, Any] =       Path.home() / ".cache/huggingface/accelerate"
							A				:Union[str, Any] =       "default_config.yaml"
							A				:List[Any] =       config_folder / config_file
							A				:Any =       config_folder / "_default_config.yaml"

							A				:int =       Path("tests/test_configs"			)



							@classmethod
							def 		_A	(							cls				):







											"""simple docstring"""



											if cls.config_path.is_file():
															cls.config_path.rename(cls.changed_path				)



							@classmethod
							def 		_A	(							cls				):







											"""simple docstring"""



											if cls.changed_path.is_file():
															cls.changed_path.rename(cls.config_path				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Union[str, Any]      						= self.base_cmd
											if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
															cmd += ["--multi_gpu"]
											execute_subprocess_async(cmd + [self.test_file_path] ,					env=os.environ.copy()				)



							def 		_A	(							self				):







											"""simple docstring"""



											for config in sorted(self.test_config_path.glob("**/*.yaml"				)				):
															with self.subTest(config_file=__UpperCAmelCase				):
																			execute_subprocess_async(
																			    self.base_cmd + ["--config_file", str(__UpperCAmelCase				), self.test_file_path] ,					env=os.environ.copy()				)




							def 		_A	(							self				):







											"""simple docstring"""



											execute_subprocess_async(["accelerate", "test"] ,					env=os.environ.copy()				)





class 				_a						(       unittest.TestCase			):





							'''simple docstring'''

							A				:List[str] =       "test-tpu"
							A				:Optional[Any] =       "us-central1-a"
							A				:Optional[Any] =       "ls"
							A				:str =       ["accelerate", "tpu-config"]
							A				:str =       "cd /usr/share"
							A				:Dict =       "tests/test_samples/test_command_file.sh"
							A				:int =       "Running gcloud compute tpus tpu-vm ssh"



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							List[str]      						= run_command(
											    self.cmd
											    + ["--command", self.command, "--tpu_zone", self.tpu_zone, "--tpu_name", self.tpu_name, "--debug"] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Dict      						= run_command(
											    self.cmd
											    + [
											        "--config_file",
											        "tests/test_configs/0_12_0.yaml",
											        "--command",
											        self.command,
											        "--tpu_zone",
											        self.tpu_zone,
											        "--tpu_name",
											        self.tpu_name,
											        "--debug",
											    ] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							List[str]      						= run_command(
											    self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--debug"] ,					return_stdout=__UpperCAmelCase				)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							str      						= run_command(
											    self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--command", self.command, "--debug"] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							List[str]      						= run_command(
											    self.cmd
											    + [
											        "--config_file",
											        "tests/test_configs/latest.yaml",
											        "--command",
											        self.command,
											        "--command",
											        "echo \"Hello World\"",
											        "--debug",
											    ] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Dict      						= run_command(
											    self.cmd
											    + ["--config_file", "tests/test_configs/latest.yaml", "--command_file", self.command_file, "--debug"] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							List[Any]      						= run_command(
											    self.cmd
											    + [
											        "--config_file",
											        "tests/test_configs/0_12_0.yaml",
											        "--command_file",
											        self.command_file,
											        "--tpu_zone",
											        self.tpu_zone,
											        "--tpu_name",
											        self.tpu_name,
											        "--debug",
											    ] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all' ,					__UpperCAmelCase ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Any      						= run_command(
											    self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--install_accelerate", "--debug"] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all' ,					__UpperCAmelCase ,					)




							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Union[str, Any]      						= run_command(
											    self.cmd
											    + [
											        "--config_file",
											        "tests/test_configs/latest.yaml",
											        "--install_accelerate",
											        "--accelerate_version",
											        "12.0.0",
											        "--debug",
											    ] ,					return_stdout=__UpperCAmelCase ,					)
											self.assertIn(
											    f'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all' ,					__UpperCAmelCase ,					)







import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}






def     SCREAMING_SNAKE_CASE(      __UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase )	->						Dict:
				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys() )}.' )

				if tokenizer_name is None:
								a__	:							str      						= TOKENIZER_CLASSES
				else:
								a__	:							int      						= {tokenizer_name: getattr(__UpperCamelCase	,				tokenizer_name + "Fast" )}

				logger.info(F'Loading tokenizer classes: {tokenizer_names}' )

				for tokenizer_name in tokenizer_names:
								a__	:							Any      						= TOKENIZER_CLASSES[tokenizer_name]

								a__	:							Dict      						= True
								if checkpoint_name is None:
												a__	:							Tuple      						= list(tokenizer_class.max_model_input_sizes.keys() )
								else:
												a__	:							List[str]      						= [checkpoint_name]

								logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}' )

								for checkpoint in checkpoint_names:
												logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}' )

												# Load tokenizer
												a__	:							List[str]      						= tokenizer_class.from_pretrained(__UpperCamelCase	,				force_download=__UpperCamelCase )

												# Save fast tokenizer
												logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}' )

												# For organization names we create sub-directories
												if "/" in checkpoint:
																a__						,   a__	:							Dict      						= checkpoint.split("/" )
																a__	:							Any      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
												elif add_prefix:
																a__	:							int      						= checkpoint
																a__	:							Optional[Any]      						= dump_path
												else:
																a__	:							Dict      						= None
																a__	:							Optional[Any]      						= dump_path

												logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values() )[0]:
																a__	:							Dict      						= list(tokenizer.pretrained_vocab_files_map.values() )[0][checkpoint]
																a__	:							Optional[Any]      						= file_path.split(__UpperCamelCase )[-1][0]
																if next_char == "/":
																				a__	:							Union[str, Any]      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
																				a__	:							Any      						= None

																logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												a__	:							Any      						= tokenizer.save_pretrained(
												    __UpperCamelCase	,				legacy_format=__UpperCamelCase	,				filename_prefix=__UpperCamelCase )
												logger.info(F'=> File names {file_names}' )

												for file_name in file_names:
																if not file_name.endswith("tokenizer.json" ):
																				os.remove(__UpperCamelCase )
																				logger.info(F'=> removing {file_name}' )


if __name__ == "__main__":
						lowerCamelCase		       =      argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
						)
						parser.add_argument(
						    """--tokenizer_name""",
						    default=None,
						    type=str,
						    help=(
						        F'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
						        """download and convert all the checkpoints from AWS."""
						    ),
						)
						parser.add_argument(
						    """--checkpoint_name""",
						    default=None,
						    type=str,
						    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
						)
						parser.add_argument(
						    """--force_download""",
						    action="""store_true""",
						    help="""Re-download checkpoints.""",
						)
						lowerCamelCase		       =      parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)


