



'''simple docstring'''



import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase									=							logging.get_logger(__name__)


lowerCamelCase									=							{name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()       )}."""       )

							if tokenizer_name is None:
														__lowercase													=TOKENIZER_CLASSES
							else:
														__lowercase													={tokenizer_name: getattr(__a	, tokenizer_name + 'Fast'       )}

							logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""       )

							for tokenizer_name in tokenizer_names:
														__lowercase													=TOKENIZER_CLASSES[tokenizer_name]

														__lowercase													=True
														if checkpoint_name is None:
																					__lowercase													=list(tokenizer_class.max_model_input_sizes.keys()       )
														else:
																					__lowercase													=[checkpoint_name]

														logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""       )

														for checkpoint in checkpoint_names:
																					logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""       )

																					# Load tokenizer
																					__lowercase													=tokenizer_class.from_pretrained(__a	, force_download=__a       )

																					# Save fast tokenizer
																					logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""       )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												__lowercase													=checkpoint.split('/'       )
																												__lowercase													=os.path.join(__a	, __a       )
																					elif add_prefix:
																												__lowercase													=checkpoint
																												__lowercase													=dump_path
																					else:
																												__lowercase													=None
																												__lowercase													=dump_path

																					logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""       )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()       )[0]:
																												__lowercase													=list(tokenizer.pretrained_vocab_files_map.values()       )[0][checkpoint]
																												__lowercase													=file_path.split(__a       )[-1][0]
																												if next_char == "/":
																																			__lowercase													=os.path.join(__a	, __a       )
																																			__lowercase													=None

																												logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""       )

																					__lowercase													=tokenizer.save_pretrained(
																					    __a	, legacy_format=__a	, filename_prefix=__a       )
																					logger.info(f"""=> File names {file_names}"""       )

																					for file_name in file_names:
																												if not file_name.endswith('tokenizer.json'       ):
																																			os.remove(__a       )
																																			logger.info(f"""=> removing {file_name}"""       )


if __name__ == "__main__":
		lowerCamelCase									=							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
		)
		parser.add_argument(
		    """--tokenizer_name""",
		    default=None,
		    type=str,
		    help=(
		        f"Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will "
		        """download and convert all the checkpoints from AWS."""
		    ),
		)
		parser.add_argument(
		    """--checkpoint_name""",
		    default=None,
		    type=str,
		    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
		)
		parser.add_argument(
		    """--force_download""",
		    action="""store_true""",
		    help="""Re-download checkpoints.""",
		)
		lowerCamelCase									=							parser.parse_args()

		convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)








"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


a_							=    logging.get_logger(__name__)

a_							=    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a_							=    {
    """vocab_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"""
        ),
        """squeezebert/squeezebert-mnli""": """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt""",
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"""
        ),
    },
}

a_							=    {
    """squeezebert/squeezebert-uncased""": 5_12,
    """squeezebert/squeezebert-mnli""": 5_12,
    """squeezebert/squeezebert-mnli-headless""": 5_12,
}


a_							=    {
    """squeezebert/squeezebert-uncased""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli-headless""": {"""do_lower_case""": True},
}







class 					A_(SCREAMING_SNAKE_CASE_      ):







			"""simple docstring"""



			a_       :			Union[str, Any]												=			VOCAB_FILES_NAMES
			a_       :			List[str]												=			PRETRAINED_VOCAB_FILES_MAP
			a_       :			Optional[Any]												=			PRETRAINED_INIT_CONFIGURATION
			a_       :			Tuple												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_       :			Optional[int]												=			SqueezeBertTokenizer
			def __init__(	self				,   A=None				,   A=None				,   A=True				,   A="[UNK]"				,   A="[SEP]"				,   A="[PAD]"				,   A="[CLS]"				,   A="[MASK]"				,   A=True				,   A=None				,   **A				,   ):
						super().__init__(
						    A				,   tokenizer_file=A				,   do_lower_case=A				,   unk_token=A				,   sep_token=A				,   pad_token=A				,   cls_token=A				,   mask_token=A				,   tokenize_chinese_chars=A				,   strip_accents=A				,   **A				,   )

						_lowerCamelCase					:					Union[str, Any]	       =    json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
						if (
						    normalizer_state.get('lowercase'				,   A							) != do_lower_case
						    or normalizer_state.get('strip_accents'				,   A							) != strip_accents
						    or normalizer_state.get('handle_chinese_chars'				,   A							) != tokenize_chinese_chars
						):
									_lowerCamelCase					:					List[Any]	       =    getattr(A				,   normalizer_state.pop('type'							)							)
									_lowerCamelCase					:					Optional[Any]	       =    do_lower_case
									_lowerCamelCase					:					str	       =    strip_accents
									_lowerCamelCase					:					Any	       =    tokenize_chinese_chars
									_lowerCamelCase					:					int	       =    normalizer_class(**A							)

						_lowerCamelCase					:					Dict	       =    do_lower_case
			def    _lowerCAmelCase		(	self				,   A				,   A=None							):
						_lowerCamelCase					:					Dict	       =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output
			def    _lowerCAmelCase		(	self				,   A				,   A = None							):
						_lowerCamelCase					:					Union[str, Any]	       =    [self.sep_token_id]
						_lowerCamelCase					:					Optional[Any]	       =    [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]

			def    _lowerCAmelCase		(	self				,   A				,   A = None							):
						_lowerCamelCase					:					str	       =    self._tokenizer.model.save(A				,   name=A							)
						return tuple(A							)





