
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


_lowercase			:  Union[str, Any]   							=	logging.get_logger(__name__)

_lowercase			:  Union[str, Any]   							=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowercase			:  Dict   							=	{
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

_lowercase			:  Union[str, Any]   							=	{
    "squeezebert/squeezebert-uncased": 5_1_2,
    "squeezebert/squeezebert-mnli": 5_1_2,
    "squeezebert/squeezebert-mnli-headless": 5_1_2,
}


_lowercase			:  Any   							=	{
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					SqueezeBertTokenizer






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE="[UNK]"    ,							__SCREAMING_SNAKE_CASE="[SEP]"    ,							__SCREAMING_SNAKE_CASE="[PAD]"    ,							__SCREAMING_SNAKE_CASE="[CLS]"    ,							__SCREAMING_SNAKE_CASE="[MASK]"    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							do_lower_case=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							mask_token=__SCREAMING_SNAKE_CASE    ,							tokenize_chinese_chars=__SCREAMING_SNAKE_CASE    ,							strip_accents=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Optional[Any]													=	json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
											if (
											    normalizer_state.get('''lowercase'''    ,							__SCREAMING_SNAKE_CASE		) != do_lower_case
											    or normalizer_state.get('''strip_accents'''    ,							__SCREAMING_SNAKE_CASE		) != strip_accents
											    or normalizer_state.get('''handle_chinese_chars'''    ,							__SCREAMING_SNAKE_CASE		) != tokenize_chinese_chars
											):
																	lowercase_     :					List[Any]													=	getattr(__SCREAMING_SNAKE_CASE    ,							normalizer_state.pop('''type'''		)		)
																	lowercase_     :					Any													=	do_lower_case
																	lowercase_     :					str													=	strip_accents
																	lowercase_     :					Union[str, Any]													=	tokenize_chinese_chars
																	lowercase_     :					Tuple													=	normalizer_class(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[Any]													=	do_lower_case






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

											return output






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Any													=	[self.sep_token_id]
											lowercase_     :					int													=	[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					str													=	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,							name=__SCREAMING_SNAKE_CASE		)
											return tuple(__SCREAMING_SNAKE_CASE		)





'''simple docstring'''







from typing import Optional
from urllib.parse import quote

import huggingface_hub as hfh
from packaging import version
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			Optional[str] = None							):



						"""simple docstring"""





						if version.parse(hfh.__version__							).release < version.parse('''0.11.0'''							).release:
												# old versions of hfh don't url-encode the file path
												lowercase_     :					int													=	quote(__SCREAMING_SNAKE_CASE							)
						return hfh.hf_hub_url(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				repo_type='''dataset'''				,				revision=__SCREAMING_SNAKE_CASE							)



