


'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE_      =			{
    'google/realm-cc-news-pretrained-embedder': 5_12,
    'google/realm-cc-news-pretrained-encoder': 5_12,
    'google/realm-cc-news-pretrained-scorer': 5_12,
    'google/realm-cc-news-pretrained-openqa': 5_12,
    'google/realm-orqa-nq-openqa': 5_12,
    'google/realm-orqa-nq-reader': 5_12,
    'google/realm-orqa-wq-openqa': 5_12,
    'google/realm-orqa-wq-reader': 5_12,
}

SCREAMING_SNAKE_CASE_      =			{
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :str												=VOCAB_FILES_NAMES
  a_       :List[Any]												=PRETRAINED_VOCAB_FILES_MAP
  a_       :Tuple												=PRETRAINED_INIT_CONFIGURATION
  a_       :List[str]												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_       :Any												=RealmTokenizer

  def __init__(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							int=None				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=None				,							SCREAMING_SNAKE_CASE__     :							Dict=True				,							SCREAMING_SNAKE_CASE__     :							int="[UNK]"				,							SCREAMING_SNAKE_CASE__     :							Any="[SEP]"				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]="[PAD]"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="[CLS]"				,							SCREAMING_SNAKE_CASE__     :							List[str]="[MASK]"				,							SCREAMING_SNAKE_CASE__     :							List[str]=True				,							SCREAMING_SNAKE_CASE__     :							Dict=None				,							**SCREAMING_SNAKE_CASE__     :							Dict				,							):





       '''simple docstring'''

       super().__init__(
           SCREAMING_SNAKE_CASE__				,							tokenizer_file=SCREAMING_SNAKE_CASE__				,							do_lower_case=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							tokenize_chinese_chars=SCREAMING_SNAKE_CASE__				,							strip_accents=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
       if (
           normalizer_state.get("""lowercase"""				,							SCREAMING_SNAKE_CASE__						) != do_lower_case
           or normalizer_state.get("""strip_accents"""				,							SCREAMING_SNAKE_CASE__						) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""				,							SCREAMING_SNAKE_CASE__						) != tokenize_chinese_chars
       ):
            __a       =   getattr(SCREAMING_SNAKE_CASE__				,							normalizer_state.pop("""type"""						)						)
            __a       =   do_lower_case
            __a       =   strip_accents
            __a       =   tokenize_chinese_chars
            __a       =   normalizer_class(**SCREAMING_SNAKE_CASE__						)

       __a       =   do_lower_case

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							**SCREAMING_SNAKE_CASE__     :							Optional[int]						):





       '''simple docstring'''

       __a       =   PaddingStrategy.MAX_LENGTH

       __a       =   text
       __a       =   kwargs.pop("""text_pair"""				,							SCREAMING_SNAKE_CASE__						)
       __a       =   kwargs.pop("""return_tensors"""				,							SCREAMING_SNAKE_CASE__						)

       __a       =   {
           """input_ids""": [],
           """attention_mask""": [],
           """token_type_ids""": [],
       }

       for idx, candidate_text in enumerate(SCREAMING_SNAKE_CASE__						):
            if batch_text_pair is not None:
                 __a       =   batch_text_pair[idx]
            else:
                 __a       =   None

            __a       =   super().__call__(SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__				,							return_tensors=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__						)

            __a       =   encoded_candidates.get("""input_ids"""						)
            __a       =   encoded_candidates.get("""attention_mask"""						)
            __a       =   encoded_candidates.get("""token_type_ids"""						)

            if encoded_input_ids is not None:
                 output_data["input_ids"].append(SCREAMING_SNAKE_CASE__						)
            if encoded_attention_mask is not None:
                 output_data["attention_mask"].append(SCREAMING_SNAKE_CASE__						)
            if encoded_token_type_ids is not None:
                 output_data["token_type_ids"].append(SCREAMING_SNAKE_CASE__						)

       __a       =   {key: item for key, item in output_data.items() if len(SCREAMING_SNAKE_CASE__						) != 0}

       return BatchEncoding(SCREAMING_SNAKE_CASE__				,							tensor_type=SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							int				,							SCREAMING_SNAKE_CASE__     :							int=None						):





       '''simple docstring'''

       __a       =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output

  def __a						(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       __a       =   self._tokenizer.model.save(SCREAMING_SNAKE_CASE__				,							name=SCREAMING_SNAKE_CASE__						)
       return tuple(SCREAMING_SNAKE_CASE__						)










'''simple docstring'''







def        __lowercase						(      __SCREAMING_SNAKE_CASE		)				->					str:
     """simple docstring"""




     return " ".join(input_str.split()[::-1]		)


if __name__ == "__main__":
 import doctest

 doctest.testmod()






