





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


__A     =    logging.get_logger(__name__)

__A     =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A     =    {
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

__A     =    {
    "squeezebert/squeezebert-uncased": 512,
    "squeezebert/squeezebert-mnli": 512,
    "squeezebert/squeezebert-mnli-headless": 512,
}


__A     =    {
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      VOCAB_FILES_NAMES
 lowercase_		=      PRETRAINED_VOCAB_FILES_MAP
 lowercase_		=      PRETRAINED_INIT_CONFIGURATION
 lowercase_		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_		=      SqueezeBertTokenizer

 def __init__(self    :       Optional[Any]	,			UpperCAmelCase_    :       Optional[int]=None	,			UpperCAmelCase_    :       Optional[Any]=None	,			UpperCAmelCase_    :       List[str]=True	,			UpperCAmelCase_    :       Tuple="[UNK]"	,			UpperCAmelCase_    :       List[Any]="[SEP]"	,			UpperCAmelCase_    :       List[str]="[PAD]"	,			UpperCAmelCase_    :       Any="[CLS]"	,			UpperCAmelCase_    :       List[Any]="[MASK]"	,			UpperCAmelCase_    :       int=True	,			UpperCAmelCase_    :       str=None	,			**UpperCAmelCase_    :       Optional[int]	,			)	->str:



   '''simple docstring'''



   super().__init__(
       UpperCAmelCase_	,			tokenizer_file=UpperCAmelCase_	,			do_lower_case=UpperCAmelCase_	,			unk_token=UpperCAmelCase_	,			sep_token=UpperCAmelCase_	,			pad_token=UpperCAmelCase_	,			cls_token=UpperCAmelCase_	,			mask_token=UpperCAmelCase_	,			tokenize_chinese_chars=UpperCAmelCase_	,			strip_accents=UpperCAmelCase_	,			**UpperCAmelCase_	,			)

   lowerCamelCase__:				int        =json.loads(self.backend_tokenizer.normalizer.__getstate__())
   if (
       normalizer_state.get("lowercase"	,			UpperCAmelCase_) != do_lower_case
       or normalizer_state.get("strip_accents"	,			UpperCAmelCase_) != strip_accents
       or normalizer_state.get("handle_chinese_chars"	,			UpperCAmelCase_) != tokenize_chinese_chars
   ):
     lowerCamelCase__:				Union[str, Any]        =getattr(UpperCAmelCase_	,			normalizer_state.pop("type"))
     lowerCamelCase__:				str        =do_lower_case
     lowerCamelCase__:				Any        =strip_accents
     lowerCamelCase__:				int        =tokenize_chinese_chars
     lowerCamelCase__:				str        =normalizer_class(**UpperCAmelCase_)

   lowerCamelCase__:				Optional[Any]        =do_lower_case

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any	,			UpperCAmelCase_    :       List[Any]	,			UpperCAmelCase_    :       List[Any]=None)	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				int        =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       List[int]	,			UpperCAmelCase_    :       Optional[List[int]] = None)	->List[int]:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =[self.sep_token_id]
   lowerCamelCase__:				Dict        =[self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep) * [0]
   return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







 def 							SCREAMING_SNAKE_CASE_ (self    :       int	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       Optional[str] = None)	->Tuple[str]:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =self._tokenizer.model.save(UpperCAmelCase_	,			name=UpperCAmelCase_)
   return tuple(UpperCAmelCase_)








from ..utils import DummyObject, requires_backends



class        _SCREAMING_SNAKE_CASE						(     metaclass=__SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      ["torch", "torchsde"]

 def __init__(self    :       Union[str, Any]	,			*UpperCAmelCase_    :       Dict	,			**UpperCAmelCase_    :       Tuple)	->Any:



   '''simple docstring'''



   requires_backends(self	,			["torch", "torchsde"])

 @classmethod
 def 							SCREAMING_SNAKE_CASE_ (cls    :       Dict	,			*UpperCAmelCase_    :       Optional[int]	,			**UpperCAmelCase_    :       Optional[int])	->List[Any]:



   '''simple docstring'''



   requires_backends(cls	,			["torch", "torchsde"])







 @classmethod
 def 							SCREAMING_SNAKE_CASE_ (cls    :       Tuple	,			*UpperCAmelCase_    :       Tuple	,			**UpperCAmelCase_    :       Any)	->Any:



   '''simple docstring'''



   requires_backends(cls	,			["torch", "torchsde"])

