




"""simple docstring"""

import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

a     =      logging.get_logger(__name__)


a     =      {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}






def 						_snake_case					( _snake_case							: List[str]    ,  _snake_case							: Dict    ,  _snake_case							: Any    ,  _snake_case							: int							)   -> Dict:






      '''simple docstring'''


      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(F'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()							)}.'''							)

      if tokenizer_name is None:
            _A        =       TOKENIZER_CLASSES
      else:
            _A        =       {tokenizer_name: getattr(lowerCAmelCase__    ,  tokenizer_name + 'Fast'							)}

      logger.info(F'''Loading tokenizer classes: {tokenizer_names}'''							)

      for tokenizer_name in tokenizer_names:
            _A        =       TOKENIZER_CLASSES[tokenizer_name]

            _A        =       True
            if checkpoint_name is None:
                  _A        =       list(tokenizer_class.max_model_input_sizes.keys()							)
            else:
                  _A        =       [checkpoint_name]

            logger.info(F'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''							)

            for checkpoint in checkpoint_names:
                  logger.info(F'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''							)

                  # Load tokenizer
                  _A        =       tokenizer_class.from_pretrained(lowerCAmelCase__    ,  force_download=lowerCAmelCase__							)

                  # Save fast tokenizer
                  logger.info(F'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''							)

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        _A        =       checkpoint.split('/'							)
                        _A        =       os.path.join(lowerCAmelCase__    ,  lowerCAmelCase__							)
                  elif add_prefix:
                        _A        =       checkpoint
                        _A        =       dump_path
                  else:
                        _A        =       None
                        _A        =       dump_path

                  logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''							)

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()							)[0]:
                        _A        =       list(tokenizer.pretrained_vocab_files_map.values()							)[0][checkpoint]
                        _A        =       file_path.split(lowerCAmelCase__							)[-1][0]
                        if next_char == "/":
                              _A        =       os.path.join(lowerCAmelCase__    ,  lowerCAmelCase__							)
                              _A        =       None

                        logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''							)

                  _A        =       tokenizer.save_pretrained(
                      lowerCAmelCase__    ,  legacy_format=lowerCAmelCase__    ,  filename_prefix=lowerCAmelCase__							)
                  logger.info(F'''=> File names {file_names}'''							)

                  for file_name in file_names:
                        if not file_name.endswith('tokenizer.json'							):
                              os.remove(lowerCAmelCase__							)
                              logger.info(F'''=> removing {file_name}'''							)


if __name__ == "__main__":
       a     =      argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
       )
       parser.add_argument(
           '''--tokenizer_name''',
           default=None,
           type=str,
           help=(
               F'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
               '''download and convert all the checkpoints from AWS.'''
           ),
       )
       parser.add_argument(
           '''--checkpoint_name''',
           default=None,
           type=str,
           help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
       )
       parser.add_argument(
           '''--force_download''',
           action='''store_true''',
           help='''Re-download checkpoints.''',
       )
       a     =      parser.parse_args()

       convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)







"""simple docstring"""

import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command





class     lowercase_	(					unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase   :					str				  =     inspect.getfile(accelerate.test_utils							)
 UpperCAmelCase   :					Optional[Any]				  =     os.path.sep.join(mod_file.split(os.path.sep							)[:-1] + ['''scripts''', '''test_cli.py''']							)

 UpperCAmelCase   :					List[Any]				  =     ['''accelerate''', '''launch''']
 UpperCAmelCase   :					Dict				  =     Path.home() / '''.cache/huggingface/accelerate'''
 UpperCAmelCase   :					Union[str, Any]				  =     '''default_config.yaml'''
 UpperCAmelCase   :					Union[str, Any]				  =     config_folder / config_file
 UpperCAmelCase   :					Union[str, Any]				  =     config_folder / '''_default_config.yaml'''

 UpperCAmelCase   :					List[Any]				  =     Path('''tests/test_configs'''							)

 @classmethod
 def       lowerCAmelCase_							(			cls	:	List[Any]		):
       if cls.config_path.is_file():
             cls.config_path.rename(cls.changed_path		)

 @classmethod
 def       lowerCAmelCase_							(			cls	:	Tuple		):
       if cls.changed_path.is_file():
             cls.changed_path.rename(cls.config_path		)

 def       lowerCAmelCase_							(			self	:	List[Any]		):
       _A        =       self.base_cmd
       if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
             cmd += ["--multi_gpu"]
       execute_subprocess_async(cmd + [self.test_file_path]   ,							env=os.environ.copy()		)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       for config in sorted(self.test_config_path.glob('**/*.yaml'		)		):
             with self.subTest(config_file=_UpperCAmelCase		):
                   execute_subprocess_async(
                       self.base_cmd + ['--config_file', str(_UpperCAmelCase		), self.test_file_path]   ,							env=os.environ.copy()		)



 def       lowerCAmelCase_							(			self	:	Any		):
       execute_subprocess_async(['accelerate', 'test']   ,							env=os.environ.copy()		)





class     lowercase_	(					unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase   :					Dict				  =     '''test-tpu'''
 UpperCAmelCase   :					Optional[int]				  =     '''us-central1-a'''
 UpperCAmelCase   :					List[str]				  =     '''ls'''
 UpperCAmelCase   :					str				  =     ['''accelerate''', '''tpu-config''']
 UpperCAmelCase   :					Optional[Any]				  =     '''cd /usr/share'''
 UpperCAmelCase   :					Optional[Any]				  =     '''tests/test_samples/test_command_file.sh'''
 UpperCAmelCase   :					str				  =     '''Running gcloud compute tpus tpu-vm ssh'''

 def       lowerCAmelCase_							(			self	:	Any		):
       _A        =       run_command(
           self.cmd
           + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	Dict		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/0_12_0.yaml',
               '--command',
               self.command,
               '--tpu_zone',
               self.tpu_zone,
               '--tpu_name',
               self.tpu_name,
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug']   ,							return_stdout=_UpperCAmelCase		)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	List[str]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/latest.yaml',
               '--command',
               self.command,
               '--command',
               'echo "Hello World"',
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       run_command(
           self.cmd
           + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	List[Any]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/0_12_0.yaml',
               '--command_file',
               self.command_file,
               '--tpu_zone',
               self.tpu_zone,
               '--tpu_name',
               self.tpu_name,
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	int		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)



 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/latest.yaml',
               '--install_accelerate',
               '--accelerate_version',
               '12.0.0',
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

