
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase   =	logging.get_logger(__name__)

lowercase   =	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase   =	{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase   =	{
    '''google/realm-cc-news-pretrained-embedder''': 5_1_2,
    '''google/realm-cc-news-pretrained-encoder''': 5_1_2,
    '''google/realm-cc-news-pretrained-scorer''': 5_1_2,
    '''google/realm-cc-news-pretrained-openqa''': 5_1_2,
    '''google/realm-orqa-nq-openqa''': 5_1_2,
    '''google/realm-orqa-nq-reader''': 5_1_2,
    '''google/realm-orqa-wq-openqa''': 5_1_2,
    '''google/realm-orqa-wq-reader''': 5_1_2,
}

lowercase   =	{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}





class    __A(					UpperCAmelCase							):
					SCREAMING_SNAKE_CASE           =     VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE           =     PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE           =     PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE           =     RealmTokenizer

					def __init__(						self       :  Union[str, Any] ,       __UpperCamelCase       :  Any=None ,       __UpperCamelCase       :  str=None ,       __UpperCamelCase       :  List[str]=True ,       __UpperCamelCase       :  Optional[Any]="[UNK]" ,       __UpperCamelCase       :  Optional[Any]="[SEP]" ,       __UpperCamelCase       :  List[str]="[PAD]" ,       __UpperCamelCase       :  int="[CLS]" ,       __UpperCamelCase       :  Optional[int]="[MASK]" ,       __UpperCamelCase       :  Optional[int]=True ,       __UpperCamelCase       :  Any=None ,       **__UpperCamelCase       :  int ,       ):
												super().__init__(
												    __UpperCamelCase ,       tokenizer_file=__UpperCamelCase ,       do_lower_case=__UpperCamelCase ,       unk_token=__UpperCamelCase ,       sep_token=__UpperCamelCase ,       pad_token=__UpperCamelCase ,       cls_token=__UpperCamelCase ,       mask_token=__UpperCamelCase ,       tokenize_chinese_chars=__UpperCamelCase ,       strip_accents=__UpperCamelCase ,       **__UpperCamelCase ,       )

												lowerCamelCase_					=       json.loads(self.backend_tokenizer.normalizer.__getstate__() )
												if (
												    normalizer_state.get("""lowercase""" ,       __UpperCamelCase ) != do_lower_case
												    or normalizer_state.get("""strip_accents""" ,       __UpperCamelCase ) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars""" ,       __UpperCamelCase ) != tokenize_chinese_chars
												):
																			lowerCamelCase_					=       getattr(__UpperCamelCase ,       normalizer_state.pop("""type""" ) )
																			lowerCamelCase_					=       do_lower_case
																			lowerCamelCase_					=       strip_accents
																			lowerCamelCase_					=       tokenize_chinese_chars
																			lowerCamelCase_					=       normalizer_class(**__UpperCamelCase )

												lowerCamelCase_					=       do_lower_case

					def  lowercase__   (						self       :  Any ,       __UpperCamelCase       :  Dict ,       **__UpperCamelCase       :  Optional[Any] ):
												lowerCamelCase_					=       PaddingStrategy.MAX_LENGTH

												lowerCamelCase_					=       text
												lowerCamelCase_					=       kwargs.pop("""text_pair""" ,       __UpperCamelCase )
												lowerCamelCase_					=       kwargs.pop("""return_tensors""" ,       __UpperCamelCase )

												lowerCamelCase_					=       {
												    """input_ids""": [],
												    """attention_mask""": [],
												    """token_type_ids""": [],
												}

												for idx, candidate_text in enumerate(__UpperCamelCase ):
																			if batch_text_pair is not None:
																										lowerCamelCase_					=       batch_text_pair[idx]
																			else:
																										lowerCamelCase_					=       None

																			lowerCamelCase_					=       super().__call__(__UpperCamelCase ,       __UpperCamelCase ,       return_tensors=__UpperCamelCase ,       **__UpperCamelCase )

																			lowerCamelCase_					=       encoded_candidates.get("""input_ids""" )
																			lowerCamelCase_					=       encoded_candidates.get("""attention_mask""" )
																			lowerCamelCase_					=       encoded_candidates.get("""token_type_ids""" )

																			if encoded_input_ids is not None:
																										output_data["input_ids"].append(__UpperCamelCase )
																			if encoded_attention_mask is not None:
																										output_data["attention_mask"].append(__UpperCamelCase )
																			if encoded_token_type_ids is not None:
																										output_data["token_type_ids"].append(__UpperCamelCase )

												lowerCamelCase_					=       {key: item for key, item in output_data.items() if len(__UpperCamelCase ) != 0}

												return BatchEncoding(__UpperCamelCase ,       tensor_type=__UpperCamelCase )

					def  lowercase__   (						self       :  Dict ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  Any=None ):
												lowerCamelCase_					=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																			output += token_ids_a + [self.sep_token_id]

												return output

					def  lowercase__   (						self       :  Union[str, Any] ,       __UpperCamelCase       :  List[int] ,       __UpperCamelCase       :  Optional[List[int]] = None ):
												lowerCamelCase_					=       [self.sep_token_id]
												lowerCamelCase_					=       [self.cls_token_id]
												if token_ids_a is None:
																			return len(cls + token_ids_a + sep ) * [0]
												return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]



					def  lowercase__   (						self       :  List[str] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Optional[str] = None ):
												lowerCamelCase_					=       self._tokenizer.model.save(__UpperCamelCase ,       name=__UpperCamelCase )
												return tuple(__UpperCamelCase )







def   __lowerCAmelCase    (				UpperCAmelCase__					:   int							)	->     int:

							if not isinstance(UpperCAmelCase__      ,					UpperCAmelCase__							):
														raise ValueError("""multiplicative_persistence() only accepts integral values"""							)
							if num < 0:
														raise ValueError("""multiplicative_persistence() does not accept negative values"""							)

							lowerCamelCase_					=       0
							lowerCamelCase_					=       str(UpperCAmelCase__							)

							while len(UpperCAmelCase__							) != 1:
														lowerCamelCase_					=       [int(UpperCAmelCase__							) for i in num_string]

														lowerCamelCase_					=       1
														for i in range(0      ,					len(UpperCAmelCase__							)							):
																					total *= numbers[i]

														lowerCamelCase_					=       str(UpperCAmelCase__							)

														steps += 1
							return steps



def   __lowerCAmelCase    (				UpperCAmelCase__					:   int							)	->     int:

							if not isinstance(UpperCAmelCase__      ,					UpperCAmelCase__							):
														raise ValueError("""additive_persistence() only accepts integral values"""							)
							if num < 0:
														raise ValueError("""additive_persistence() does not accept negative values"""							)

							lowerCamelCase_					=       0
							lowerCamelCase_					=       str(UpperCAmelCase__							)

							while len(UpperCAmelCase__							) != 1:
														lowerCamelCase_					=       [int(UpperCAmelCase__							) for i in num_string]

														lowerCamelCase_					=       0
														for i in range(0      ,					len(UpperCAmelCase__							)							):
																					total += numbers[i]

														lowerCamelCase_					=       str(UpperCAmelCase__							)

														steps += 1
							return steps


if __name__ == "__main__":
		import doctest

		doctest.testmod()





