
lowercase        =  '''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'''



def      __lowerCAmelCase						(							UpperCAmelCase__				:      bytes       )  ->    bytes:
							# Make sure the supplied data is a bytes-like object
							if not isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					F'''a bytes-like object is required, not \'{data.__class__.__name__}\''''
														raise TypeError(UpperCAmelCase__       )

							lowerCamelCase_							  =					"""""".join(bin(UpperCAmelCase__       )[2:].zfill(8       ) for byte in data       )

							lowerCamelCase_							  =					len(UpperCAmelCase__       ) % 6 != 0

							if padding_needed:
														# The padding that will be added later
														lowerCamelCase_							  =					b"""=""" * ((6 - len(UpperCAmelCase__       ) % 6) // 2)

														# Append binary_stream with arbitrary binary digits (0's by default) to make its
														# length a multiple of 6.
														binary_stream += "0" * (6 - len(UpperCAmelCase__       ) % 6)
							else:
														lowerCamelCase_							  =					b""""""

							# Encode every 6 binary digits to their corresponding Base64 character
							return (
							    "".join(
							        B64_CHARSET[int(binary_stream[index : index + 6]			,				2       )]
							        for index in range(0			,				len(UpperCAmelCase__       )			,				6       )       ).encode()
							    + padding
							)



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str       )  ->    bytes:
							# Make sure encoded_data is either a string or a bytes-like object
							if not isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ) and not isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					(
														    """argument should be a bytes-like object or ASCII string, """
														    F'''not \'{encoded_data.__class__.__name__}\''''
														)
														raise TypeError(UpperCAmelCase__       )

							# In case encoded_data is a bytes-like object, make sure it contains only
							# ASCII characters so we convert it to a string object
							if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														try:
																					lowerCamelCase_							  =					encoded_data.decode("""utf-8"""       )
														except UnicodeDecodeError:
																					raise ValueError("""base64 encoded data should only contain ASCII characters"""       )

							lowerCamelCase_							  =					encoded_data.count("""="""       )

							# Check if the encoded string contains non base64 characters
							if padding:
														assert all(
														    char in B64_CHARSET for char in encoded_data[:-padding]       ), "Invalid base64 character(s) found."
							else:
														assert all(
														    char in B64_CHARSET for char in encoded_data       ), "Invalid base64 character(s) found."

							# Check the padding
							assert len(UpperCAmelCase__       ) % 4 == 0 and padding < 3, "Incorrect padding"

							if padding:
														# Remove padding if there is one
														lowerCamelCase_							  =					encoded_data[:-padding]

														lowerCamelCase_							  =					"""""".join(
														    bin(B64_CHARSET.index(UpperCAmelCase__       )       )[2:].zfill(6       ) for char in encoded_data       )[: -padding * 2]
							else:
														lowerCamelCase_							  =					"""""".join(
														    bin(B64_CHARSET.index(UpperCAmelCase__       )       )[2:].zfill(6       ) for char in encoded_data       )

							lowerCamelCase_							  =					[
							    int(binary_stream[index : index + 8]			,				2       )
							    for index in range(0			,				len(UpperCAmelCase__       )			,				8       )
							]

							return bytes(UpperCAmelCase__       )


if __name__ == "__main__":
		import doctest

		doctest.testmod()







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase        =  {
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

lowercase        =  {'''mobilebert-uncased''': 5_1_2}


lowercase        =  {}





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    MobileBertTokenizer

					def __init__(	self					:     Union[str, Any]   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     Dict=True   ,					__UpperCamelCase					:     Any="[UNK]"   ,					__UpperCamelCase					:     str="[SEP]"   ,					__UpperCamelCase					:     Dict="[PAD]"   ,					__UpperCamelCase					:     List[str]="[CLS]"   ,					__UpperCamelCase					:     Any="[MASK]"   ,					__UpperCamelCase					:     Any=True   ,					__UpperCamelCase					:     int=None   ,					**__UpperCamelCase					:     Dict   ,					):
												super().__init__(
												    __UpperCamelCase   ,					tokenizer_file=__UpperCamelCase   ,					do_lower_case=__UpperCamelCase   ,					unk_token=__UpperCamelCase   ,					sep_token=__UpperCamelCase   ,					pad_token=__UpperCamelCase   ,					cls_token=__UpperCamelCase   ,					mask_token=__UpperCamelCase   ,					tokenize_chinese_chars=__UpperCamelCase   ,					strip_accents=__UpperCamelCase   ,					**__UpperCamelCase   ,					)

												lowerCamelCase_							  =					json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
												if (
												    normalizer_state.get("""lowercase"""   ,					__UpperCamelCase   ) != do_lower_case
												    or normalizer_state.get("""strip_accents"""   ,					__UpperCamelCase   ) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""   ,					__UpperCamelCase   ) != tokenize_chinese_chars
												):
																			lowerCamelCase_							  =					getattr(__UpperCamelCase   ,					normalizer_state.pop("""type"""   )   )
																			lowerCamelCase_							  =					do_lower_case
																			lowerCamelCase_							  =					strip_accents
																			lowerCamelCase_							  =					tokenize_chinese_chars
																			lowerCamelCase_							  =					normalizer_class(**__UpperCamelCase   )

												lowerCamelCase_							  =					do_lower_case

					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     Union[str, Any]   ,					__UpperCamelCase					:     Dict=None   ):
												lowerCamelCase_							  =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																			output += token_ids_a + [self.sep_token_id]

												return output

					def    lowercase__      (	self					:     Tuple   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     Optional[List[int]] = None   ):
												lowerCamelCase_							  =					[self.sep_token_id]
												lowerCamelCase_							  =					[self.cls_token_id]
												if token_ids_a is None:
																			return len(cls + token_ids_a + sep   ) * [0]
												return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[str] = None   ):
												lowerCamelCase_							  =					self._tokenizer.model.save(__UpperCamelCase   ,					name=__UpperCamelCase   )
												return tuple(__UpperCamelCase   )





