




import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE 						={name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     ):
							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(F"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()     )}."     )

							if tokenizer_name is None:
														SCREAMING_SNAKE_CASE_   	=    TOKENIZER_CLASSES
							else:
														SCREAMING_SNAKE_CASE_   	=    {tokenizer_name: getattr(_lowerCAmelCase  ,    tokenizer_name + '''Fast'''     )}

							logger.info(F"Loading tokenizer classes: {tokenizer_names}"     )

							for tokenizer_name in tokenizer_names:
														SCREAMING_SNAKE_CASE_   	=    TOKENIZER_CLASSES[tokenizer_name]

														SCREAMING_SNAKE_CASE_   	=    True
														if checkpoint_name is None:
																					SCREAMING_SNAKE_CASE_   	=    list(tokenizer_class.max_model_input_sizes.keys()     )
														else:
																					SCREAMING_SNAKE_CASE_   	=    [checkpoint_name]

														logger.info(F"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"     )

														for checkpoint in checkpoint_names:
																					logger.info(F"Loading {tokenizer_class.__class__.__name__} {checkpoint}"     )

																					# Load tokenizer
																					SCREAMING_SNAKE_CASE_   	=    tokenizer_class.from_pretrained(_lowerCAmelCase  ,    force_download=_lowerCAmelCase     )

																					# Save fast tokenizer
																					logger.info(F"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"     )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    checkpoint.split('''/'''     )
																												SCREAMING_SNAKE_CASE_   	=    os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase     )
																					elif add_prefix:
																												SCREAMING_SNAKE_CASE_   	=    checkpoint
																												SCREAMING_SNAKE_CASE_   	=    dump_path
																					else:
																												SCREAMING_SNAKE_CASE_   	=    None
																												SCREAMING_SNAKE_CASE_   	=    dump_path

																					logger.info(F"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"     )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()     )[0]:
																												SCREAMING_SNAKE_CASE_   	=    list(tokenizer.pretrained_vocab_files_map.values()     )[0][checkpoint]
																												SCREAMING_SNAKE_CASE_   	=    file_path.split(_lowerCAmelCase     )[-1][0]
																												if next_char == "/":
																																			SCREAMING_SNAKE_CASE_   	=    os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase     )
																																			SCREAMING_SNAKE_CASE_   	=    None

																												logger.info(F"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"     )

																					SCREAMING_SNAKE_CASE_   	=    tokenizer.save_pretrained(
																					    _lowerCAmelCase  ,    legacy_format=_lowerCAmelCase  ,    filename_prefix=_lowerCAmelCase     )
																					logger.info(F"=> File names {file_names}"     )

																					for file_name in file_names:
																												if not file_name.endswith('''tokenizer.json'''     ):
																																			os.remove(_lowerCAmelCase     )
																																			logger.info(F"=> removing {file_name}"     )


if __name__ == "__main__":
			__SCREAMING_SNAKE_CASE 						=argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
			)
			parser.add_argument(
			    """--tokenizer_name""",
			    default=None,
			    type=str,
			    help=(
			        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
			        """download and convert all the checkpoints from AWS."""
			    ),
			)
			parser.add_argument(
			    """--checkpoint_name""",
			    default=None,
			    type=str,
			    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
			)
			parser.add_argument(
			    """--force_download""",
			    action="""store_true""",
			    help="""Re-download checkpoints.""",
			)
			__SCREAMING_SNAKE_CASE 						=parser.parse_args()

			convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)








# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
			import torch






class  __magic_name__						(  TensorFormatter[Mapping, "torch.Tensor", Mapping]):
				'''simple docstring'''







				def __init__(							self:   int		,					_lowerCamelCase:   int=None		,					**_lowerCamelCase:   List[Any]		):
											super().__init__(features=_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    torch_tensor_kwargs
											import torch  # noqa import torch at initialization


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   Optional[Any]		):
											import torch

											if isinstance(_lowerCamelCase		,					_lowerCamelCase		) and column:
																		if all(
																		    isinstance(_lowerCamelCase		,					torch.Tensor		) and x.shape == column[0].shape and x.dtype == column[0].dtype
																		    for x in column		):
																									return torch.stack(_lowerCamelCase		)
											return column


				def     _A							(							self:   Tuple		,					_lowerCamelCase:   List[str]		):
											import torch

											if isinstance(_lowerCamelCase		,					(str, bytes, type(_lowerCamelCase		))		):
																		return value
											elif isinstance(_lowerCamelCase		,					(np.character, np.ndarray)		) and np.issubdtype(value.dtype		,					np.character		):
																		return value.tolist()

											SCREAMING_SNAKE_CASE_   	=    {}

											if isinstance(_lowerCamelCase		,					(np.number, np.ndarray)		) and np.issubdtype(value.dtype		,					np.integer		):
																		SCREAMING_SNAKE_CASE_   	=    {'''dtype''': torch.intaa}
											elif isinstance(_lowerCamelCase		,					(np.number, np.ndarray)		) and np.issubdtype(value.dtype		,					np.floating		):
																		SCREAMING_SNAKE_CASE_   	=    {'''dtype''': torch.floataa}
											elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																		import PIL.Image

																		if isinstance(_lowerCamelCase		,					PIL.Image.Image		):
																									SCREAMING_SNAKE_CASE_   	=    np.asarray(_lowerCamelCase		)
											return torch.tensor(_lowerCamelCase		,					**{**default_dtype, **self.torch_tensor_kwargs}		)


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   Dict		):
											import torch

											# support for torch, tf, jax etc.
											if hasattr(_lowerCamelCase		,					'''__array__'''		) and not isinstance(_lowerCamelCase		,					torch.Tensor		):
																		SCREAMING_SNAKE_CASE_   	=    data_struct.__array__()
											# support for nested types like struct of list of struct
											if isinstance(_lowerCamelCase		,					np.ndarray		):
																		if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
																									return self._consolidate([self.recursive_tensorize(_lowerCamelCase		) for substruct in data_struct]		)
											elif isinstance(_lowerCamelCase		,					(list, tuple)		):
																		return self._consolidate([self.recursive_tensorize(_lowerCamelCase		) for substruct in data_struct]		)
											return self._tensorize(_lowerCamelCase		)


				def     _A							(							self:   str		,					_lowerCamelCase:   dict		):
											return map_nested(self._recursive_tensorize		,					_lowerCamelCase		,					map_list=_lowerCamelCase		)


				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   pa.Table		):
											SCREAMING_SNAKE_CASE_   	=    self.numpy_arrow_extractor().extract_row(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.python_features_decoder.decode_row(_lowerCamelCase		)
											return self.recursive_tensorize(_lowerCamelCase		)


				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   pa.Table		):
											SCREAMING_SNAKE_CASE_   	=    self.numpy_arrow_extractor().extract_column(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.python_features_decoder.decode_column(_lowerCamelCase		,					pa_table.column_names[0]		)
											SCREAMING_SNAKE_CASE_   	=    self.recursive_tensorize(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self._consolidate(_lowerCamelCase		)
											return column





				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   pa.Table		):
											SCREAMING_SNAKE_CASE_   	=    self.numpy_arrow_extractor().extract_batch(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.python_features_decoder.decode_batch(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.recursive_tensorize(_lowerCamelCase		)
											for column_name in batch:
																		SCREAMING_SNAKE_CASE_   	=    self._consolidate(batch[column_name]		)
											return batch


