



'''simple docstring'''

import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

snake_case_			=     logging.get_logger(__name__)


snake_case_			=     {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}






def       __lowercase  (_SCREAMING_SNAKE_CASE      :Optional[Any]    ,	_SCREAMING_SNAKE_CASE      :int    ,	_SCREAMING_SNAKE_CASE      :Dict    ,	_SCREAMING_SNAKE_CASE      :List[str]       ):



    if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
        raise ValueError(F'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()       )}.'''       )

    if tokenizer_name is None:
        SCREAMING_SNAKE_CASE  :						Any      					= TOKENIZER_CLASSES
    else:
        SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= {tokenizer_name: getattr(_SCREAMING_SNAKE_CASE    ,	tokenizer_name + '''Fast'''       )}

    logger.info(F'''Loading tokenizer classes: {tokenizer_names}'''       )

    for tokenizer_name in tokenizer_names:
        SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= TOKENIZER_CLASSES[tokenizer_name]

        SCREAMING_SNAKE_CASE  :						str      					= True
        if checkpoint_name is None:
            SCREAMING_SNAKE_CASE  :						Any      					= list(tokenizer_class.max_model_input_sizes.keys()       )
        else:
            SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= [checkpoint_name]

        logger.info(F'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''       )

        for checkpoint in checkpoint_names:
            logger.info(F'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''       )

            # Load tokenizer
            SCREAMING_SNAKE_CASE  :						Dict      					= tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE    ,	force_download=_SCREAMING_SNAKE_CASE       )

            # Save fast tokenizer
            logger.info(F'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''       )

            # For organization names we create sub-directories
            if "/" in checkpoint:
                SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE  :						Any      					= checkpoint.split('''/'''       )
                SCREAMING_SNAKE_CASE  :						Optional[int]      					= os.path.join(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )
            elif add_prefix:
                SCREAMING_SNAKE_CASE  :						Any      					= checkpoint
                SCREAMING_SNAKE_CASE  :						int      					= dump_path
            else:
                SCREAMING_SNAKE_CASE  :						Dict      					= None
                SCREAMING_SNAKE_CASE  :						int      					= dump_path

            logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''       )

            if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()       )[0]:
                SCREAMING_SNAKE_CASE  :						Dict      					= list(tokenizer.pretrained_vocab_files_map.values()       )[0][checkpoint]
                SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= file_path.split(_SCREAMING_SNAKE_CASE       )[-1][0]
                if next_char == "/":
                    SCREAMING_SNAKE_CASE  :						Optional[int]      					= os.path.join(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )
                    SCREAMING_SNAKE_CASE  :						Optional[Any]      					= None

                logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''       )

            SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= tokenizer.save_pretrained(
                _SCREAMING_SNAKE_CASE    ,	legacy_format=_SCREAMING_SNAKE_CASE    ,	filename_prefix=_SCREAMING_SNAKE_CASE       )
            logger.info(F'''=> File names {file_names}'''       )

            for file_name in file_names:
                if not file_name.endswith('''tokenizer.json'''       ):
                    os.remove(_SCREAMING_SNAKE_CASE       )
                    logger.info(F'''=> removing {file_name}'''       )


if __name__ == "__main__":
 snake_case_			=     argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
 )
 parser.add_argument(
     """--tokenizer_name""",
     default=None,
     type=str,
     help=(
         f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
         """download and convert all the checkpoints from AWS."""
     ),
 )
 parser.add_argument(
     """--checkpoint_name""",
     default=None,
     type=str,
     help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
 )
 parser.add_argument(
     """--force_download""",
     action="""store_true""",
     help="""Re-download checkpoints.""",
 )
 snake_case_			=     parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)









'''simple docstring'''

import inspect
import os
import unittest
from dataclasses import dataclass

import torch

from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
from accelerate.state import AcceleratorState
from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
from accelerate.utils import KwargsHandler


@dataclass
class 	a__     (				_lowercase	):
	__magic_name__		:    int	    =     0
	__magic_name__		:    bool	    =     False
	__magic_name__		:    float	    =     3.0



class 	a__     (				unittest.TestCase	):


	def 				lowercase__  (self : str      )     ->			Union[str, Any]:

					"""simple docstring"""

					self.assertDictEqual(MockClass().to_kwargs(),					{}      )
					self.assertDictEqual(MockClass(a=2      ).to_kwargs(),					{'''a''': 2}      )
					self.assertDictEqual(MockClass(a=2,					b=__UpperCAmelCase      ).to_kwargs(),					{'''a''': 2, '''b''': True}      )
					self.assertDictEqual(MockClass(a=2,					c=2.25      ).to_kwargs(),					{'''a''': 2, '''c''': 2.25}      )


	@require_cuda
	def 				lowercase__  (self : Dict      )     ->			str:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[str]      					= GradScalerKwargs(init_scale=1024,					growth_factor=2      )
					AcceleratorState._reset_state()
					SCREAMING_SNAKE_CASE  :						Tuple      					= Accelerator(mixed_precision='''fp16''',					kwargs_handlers=[scaler_handler]      )
					print(accelerator.use_fpaa      )
					SCREAMING_SNAKE_CASE  :						str      					= accelerator.scaler

					# Check the kwargs have been applied
					self.assertEqual(scaler._init_scale,					1024.0      )
					self.assertEqual(scaler._growth_factor,					2.0      )

					# Check the other values are at the default
					self.assertEqual(scaler._backoff_factor,					0.5      )
					self.assertEqual(scaler._growth_interval,					2000      )
					self.assertEqual(scaler._enabled,					__UpperCAmelCase      )






	@require_multi_gpu
	def 				lowercase__  (self : Optional[Any]      )     ->			Dict:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						str      					= ['''torchrun''', F'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__      )]
					execute_subprocess_async(__UpperCAmelCase,					env=os.environ.copy()      )


if __name__ == "__main__":
	snake_case_			=     DistributedDataParallelKwargs(bucket_cap_mb=15, find_unused_parameters=True)
	snake_case_			=     Accelerator(kwargs_handlers=[ddp_scaler])
	snake_case_			=     torch.nn.Linear(100, 200)
	snake_case_			=     accelerator.prepare(model)

	# Check the values changed in kwargs
	snake_case_			=     """"""
	snake_case_			=     model.bucket_bytes_cap // (1024 * 1024)
	if observed_bucket_cap_map != 15:
		error_msg += f"Kwargs badly passed, should have `15` but found {observed_bucket_cap_map}.\n"
	if model.find_unused_parameters is not True:
		error_msg += f"Kwargs badly passed, should have `True` but found {model.find_unused_parameters}.\n"

	# Check the values of the defaults
	if model.dim != 0:
		error_msg += f"Default value not respected, should have `0` but found {model.dim}.\n"
	if model.broadcast_buffers is not True:
		error_msg += f"Default value not respected, should have `True` but found {model.broadcast_buffers}.\n"
	if model.gradient_as_bucket_view is not False:
		error_msg += f"Default value not respected, should have `False` but found {model.gradient_as_bucket_view}.\n"

	# Raise error at the end to make sure we don't stop at the first failure.
	if len(error_msg) > 0:
		raise ValueError(error_msg)




