



'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


A__    :   int           =					logging.get_logger(__name__)

A__    :   Optional[Any]           =					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__    :   str           =					{
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

A__    :   Union[str, Any]           =					{'''mobilebert-uncased''': 5_1_2}


A__    :   Dict           =					{}



class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 A__				= VOCAB_FILES_NAMES
 A__				= PRETRAINED_VOCAB_FILES_MAP
 A__				= PRETRAINED_INIT_CONFIGURATION
 A__				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 A__				= MobileBertTokenizer
 def __init__(   self						:							Tuple      ,     __a						:							Any=None      ,     __a						:							str=None      ,     __a						:							Any=True      ,     __a						:							Tuple="[UNK]"      ,     __a						:							Optional[int]="[SEP]"      ,     __a						:							int="[PAD]"      ,     __a						:							Optional[int]="[CLS]"      ,     __a						:							Optional[Any]="[MASK]"      ,     __a						:							List[Any]=True      ,     __a						:							Optional[Any]=None      ,     **__a						:							str      ,     )      ->   Dict:







       '''simple docstring'''
       super().__init__(
           __a      ,     tokenizer_file=__a      ,     do_lower_case=__a      ,     unk_token=__a      ,     sep_token=__a      ,     pad_token=__a      ,     cls_token=__a      ,     mask_token=__a      ,     tokenize_chinese_chars=__a      ,     strip_accents=__a      ,     **__a      ,     )

       __snake_case :   Tuple													=    json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get('lowercase'      ,     __a  ) != do_lower_case
           or normalizer_state.get('strip_accents'      ,     __a  ) != strip_accents
           or normalizer_state.get('handle_chinese_chars'      ,     __a  ) != tokenize_chinese_chars
       ):
             __snake_case :   str													=    getattr(__a      ,     normalizer_state.pop('type'  )  )
             __snake_case :   List[Any]													=    do_lower_case
             __snake_case :   str													=    strip_accents
             __snake_case :   str													=    tokenize_chinese_chars
             __snake_case :   List[Any]													=    normalizer_class(**__a  )

       __snake_case :   Optional[int]													=    do_lower_case
 def A_   (   self						:							Optional[int]      ,     __a						:							Dict      ,     __a						:							Optional[int]=None  )      ->   str:







       '''simple docstring'''
       __snake_case :   Dict													=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

       return output
 def A_   (   self						:							List[Any]      ,     __a						:							List[int]      ,     __a						:							Optional[List[int]] = None  )      ->   List[int]:







       '''simple docstring'''
       __snake_case :   Tuple													=    [self.sep_token_id]
       __snake_case :   Tuple													=    [self.cls_token_id]
       if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]





 def A_   (   self						:							int      ,     __a						:							str      ,     __a						:							Optional[str] = None  )      ->   Tuple[str]:







       '''simple docstring'''
       __snake_case :   Any													=    self._tokenizer.model.save(__a      ,     name=__a  )
       return tuple(__a  )








'''simple docstring'''





from __future__ import annotations





def   a_   (		_UpperCAmelCase    :      list[int]				)    ->  bool:
      return len(set(_UpperCAmelCase				)				) == len(_UpperCAmelCase				)


if __name__ == "__main__":
    import doctest

    doctest.testmod()



