
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


lowercase       :				List[Any] =   logging.get_logger(__name__)

lowercase       :				str =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase       :				Optional[Any] =   {
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

lowercase       :				Optional[int] =   {'''mobilebert-uncased''': 512}


lowercase       :				Optional[int] =   {}


class 		__UpperCAmelCase (   lowerCamelCase_		):
							__lowercase 						=			VOCAB_FILES_NAMES
							__lowercase 						=			PRETRAINED_VOCAB_FILES_MAP
							__lowercase 						=			PRETRAINED_INIT_CONFIGURATION
							__lowercase 						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase 						=			MobileBertTokenizer


							def __init__(			self       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=True       ,  lowerCAmelCase_="[UNK]"       ,  lowerCAmelCase_="[SEP]"       ,  lowerCAmelCase_="[PAD]"       ,  lowerCAmelCase_="[CLS]"       ,  lowerCAmelCase_="[MASK]"       ,  lowerCAmelCase_=True       ,  lowerCAmelCase_=None       ,  **lowerCAmelCase_       ,  ):





								"""simple docstring"""






								super().__init__(
								    snake_case__       ,  tokenizer_file=snake_case__       ,  do_lower_case=snake_case__       ,  unk_token=snake_case__       ,  sep_token=snake_case__       ,  pad_token=snake_case__       ,  cls_token=snake_case__       ,  mask_token=snake_case__       ,  tokenize_chinese_chars=snake_case__       ,  strip_accents=snake_case__       ,  **snake_case__       ,  )

								_snake_case  		=						json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
								if (
								    normalizer_state.get('lowercase'       ,  snake_case__    ) != do_lower_case
								    or normalizer_state.get('strip_accents'       ,  snake_case__    ) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'       ,  snake_case__    ) != tokenize_chinese_chars
								):
									_snake_case  		=						getattr(snake_case__       ,  normalizer_state.pop('type'    )    )
									_snake_case  		=						do_lower_case
									_snake_case  		=						strip_accents
									_snake_case  		=						tokenize_chinese_chars
									_snake_case  		=						normalizer_class(**snake_case__    )

								_snake_case  		=						do_lower_case


							def 					lowerCamelCase							(			self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=None    ):





								"""simple docstring"""






								_snake_case  		=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output


							def 					lowerCamelCase							(			self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None    ):





								"""simple docstring"""






								_snake_case  		=						[self.sep_token_id]
								_snake_case  		=						[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




							def 					lowerCamelCase							(			self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None    ):





								"""simple docstring"""






								_snake_case  		=						self._tokenizer.model.save(snake_case__       ,  name=snake_case__    )
								return tuple(snake_case__    )







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCamelCase__ :   Optional[int]						=							{
    '''configuration_bloom''': ['''BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BloomConfig''', '''BloomOnnxConfig'''],
}
try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   Tuple						=							['''BloomTokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   Any						=							[
						    '''BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''BloomForCausalLM''',
						    '''BloomModel''',
						    '''BloomPreTrainedModel''',
						    '''BloomForSequenceClassification''',
						    '''BloomForTokenClassification''',
						    '''BloomForQuestionAnswering''',
						]

if TYPE_CHECKING:
						from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_bloom_fast import BloomTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_bloom import (
												    BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    BloomForCausalLM,
												    BloomForQuestionAnswering,
												    BloomForSequenceClassification,
												    BloomForTokenClassification,
												    BloomModel,
												    BloomPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase__ :   Tuple						=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



