


'''simple docstring'''
import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_			:  Dict										=   get_tests_dir('fixtures/spiece.model')

@require_sentencepiece
@require_tokenizers
class     _lowerCamelCase       (     snake_case_     ,		unittest.TestCase					):




 '''simple docstring'''
 __lowercase					:		Union[str, Any]     				=    DebertaVaTokenizer
 __lowercase					:		str     				=    DebertaVaTokenizerFast
 __lowercase					:		Any     				=    True
 __lowercase					:		Union[str, Any]     				=    True


 def 			snake_case__ (   self    ):

  """simple docstring"""







  super().setUp()

  # We have a SentencePiece fixture for testing
  __A	:		Optional[int]	  =      DebertaVaTokenizer(__lowercase   ,   unk_token='<unk>'    )
  tokenizer.save_pretrained(self.tmpdirname    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		Dict	  =      'this is a test'
  __A	:		List[str]	  =      'this is a test'
  return input_text, output_text


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Tuple	  =      '<pad>'
  __A	:		Optional[int]	  =      0

  self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase    )   ,   __lowercase    )
  self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase    )   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Dict	  =      list(self.get_tokenizer().get_vocab().keys()    )
  self.assertEqual(vocab_keys[0]   ,   '<pad>'    )
  self.assertEqual(vocab_keys[1]   ,   '<unk>'    )
  self.assertEqual(vocab_keys[-1]   ,   '[PAD]'    )
  self.assertEqual(len(__lowercase    )   ,   30_001    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  self.assertEqual(self.get_tokenizer().vocab_size   ,   30_000    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Tuple	  =      ' \tHeLLo!how  \n Are yoU?  '
  __A	:		Optional[Any]	  =      ['▁hello', '!', 'how', '▁are', '▁you', '?']
  # fmt: on

  __A	:		Any	  =      DebertaVaTokenizer(__lowercase   ,   do_lower_case=__lowercase    )
  __A	:		Optional[Any]	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Tuple	  =      DebertaVaTokenizerFast(__lowercase   ,   do_lower_case=__lowercase    )
  __A	:		str	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )


 @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'    )
 def 			snake_case__ (   self    ):

  """simple docstring"""







  pass


 @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'    )
 def 			snake_case__ (   self    ):

  """simple docstring"""







  pass


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Tuple	  =      'I was born in 92000, and this is falsé.'
  __A	:		Optional[Any]	  =      ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
  # fmt: on

  __A	:		List[Any]	  =      DebertaVaTokenizer(__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Dict	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Tuple	  =      DebertaVaTokenizerFast(__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Union[str, Any]	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		List[Any]	  =      'I was born in 92000, and this is falsé.'
  __A	:		Dict	  =      ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
  # fmt: on

  __A	:		Dict	  =      DebertaVaTokenizer(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Optional[int]	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Dict	  =      DebertaVaTokenizerFast(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Any	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )
  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Dict	  =      'I was born in 92000, and this is falsé.'
  __A	:		Optional[Any]	  =      ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
  # fmt: on

  __A	:		Any	  =      DebertaVaTokenizer(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Dict	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		List[Any]	  =      DebertaVaTokenizerFast(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Union[str, Any]	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Optional[Any]	  =      'I was born in 92000, and this is falsé.'
  __A	:		Union[str, Any]	  =      ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
  # fmt: on

  __A	:		Dict	  =      DebertaVaTokenizer(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		int	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Optional[Any]	  =      DebertaVaTokenizerFast(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Optional[int]	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Optional[int]	  =      ' \tHeLLo!how  \n Are yoU?  '
  __A	:		Dict	  =      ['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
  # fmt: on

  __A	:		str	  =      DebertaVaTokenizer(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		List[Any]	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Tuple	  =      DebertaVaTokenizerFast(__lowercase   ,   do_lower_case=__lowercase   ,   split_by_punct=__lowercase    )
  __A	:		Union[str, Any]	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )

  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Dict	  =      self.get_tokenizer()
  __A	:		int	  =      self.get_rust_tokenizer()

  __A	:		Any	  =      'I was born in 92000, and this is falsé.'

  __A	:		int	  =      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )
  __A	:		int	  =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		List[str]	  =      tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  __A	:		List[str]	  =      rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		str	  =      self.get_rust_tokenizer()
  __A	:		Optional[int]	  =      tokenizer.encode(__lowercase    )
  __A	:		Optional[int]	  =      rust_tokenizer.encode(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		List[Any]	  =      'This is a test'
  __A	:		str	  =      [13, 1, 4_398, 25, 21, 1_289]
  __A	:		List[Any]	  =      ['▁', 'T', 'his', '▁is', '▁a', '▁test']
  __A	:		int	  =      ['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

  __A	:		Optional[Any]	  =      DebertaVaTokenizer(__lowercase   ,   keep_accents=__lowercase    )
  __A	:		Optional[Any]	  =      DebertaVaTokenizerFast(__lowercase   ,   keep_accents=__lowercase    )

  __A	:		Optional[Any]	  =      tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		str	  =      tokenizer.tokenize(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Tuple	  =      tokenizer.convert_ids_to_tokens(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		Optional[Any]	  =      rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Any	  =      rust_tokenizer.tokenize(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Dict	  =      rust_tokenizer.convert_ids_to_tokens(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  # fmt: off
  __A	:		Any	  =      'I was born in 92000, and this is falsé.'
  __A	:		Union[str, Any]	  =      [13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
  __A	:		Optional[int]	  =      ['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
  __A	:		List[str]	  =      ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
  # fmt: on

  __A	:		int	  =      tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Tuple	  =      tokenizer.tokenize(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Dict	  =      tokenizer.convert_ids_to_tokens(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )

  __A	:		str	  =      rust_tokenizer.encode(__lowercase   ,   add_special_tokens=__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Tuple	  =      rust_tokenizer.tokenize(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )
  __A	:		Tuple	  =      rust_tokenizer.convert_ids_to_tokens(__lowercase    )
  self.assertListEqual(__lowercase   ,   __lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		int	  =      DebertaVaTokenizer(__lowercase    )

  __A	:		List[Any]	  =      tokenizer.encode('sequence builders'    )
  __A	:		str	  =      tokenizer.encode('multi-sequence build'    )

  __A	:		Tuple	  =      tokenizer.build_inputs_with_special_tokens(__lowercase    )
  __A	:		List[Any]	  =      tokenizer.build_inputs_with_special_tokens(__lowercase   ,   __lowercase    )

  self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,   __lowercase    )
  self.assertEqual(
      [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,   __lowercase   ,   )







 @slow
 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		List[str]	  =      {'input_ids': [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
  # fmt: on

  self.tokenizer_integration_test_util(
      expected_encoding=__lowercase   ,   model_name='microsoft/deberta-v2-xlarge'   ,   revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'   ,   )










'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase_			:  int										=   logging.get_logger(__name__)

UpperCAmelCase_			:  int										=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_			:  List[str]										=   {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}

class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Union[str, Any]     				=    VOCAB_FILES_NAMES
 __lowercase					:		int     				=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_INIT_CONFIGURATION
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase					:		List[str]     				=    RealmTokenizer


 def __init__(   self   ,   __lowercase=None   ,   __lowercase=None   ,   __lowercase=True   ,   __lowercase="[UNK]"   ,   __lowercase="[SEP]"   ,   __lowercase="[PAD]"   ,   __lowercase="[CLS]"   ,   __lowercase="[MASK]"   ,   __lowercase=True   ,   __lowercase=None   ,   **__lowercase   ,   ):

  """simple docstring"""







  super().__init__(
      __lowercase   ,   tokenizer_file=__lowercase   ,   do_lower_case=__lowercase   ,   unk_token=__lowercase   ,   sep_token=__lowercase   ,   pad_token=__lowercase   ,   cls_token=__lowercase   ,   mask_token=__lowercase   ,   tokenize_chinese_chars=__lowercase   ,   strip_accents=__lowercase   ,   **__lowercase   ,   )

  __A	:		Optional[int]	  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
  if (
      normalizer_state.get('lowercase'   ,   __lowercase    ) != do_lower_case
      or normalizer_state.get('strip_accents'   ,   __lowercase    ) != strip_accents
      or normalizer_state.get('handle_chinese_chars'   ,   __lowercase    ) != tokenize_chinese_chars
  ):
   __A	:		Tuple	  =      getattr(__lowercase   ,   normalizer_state.pop('type'    )    )
   __A	:		Optional[int]	  =      do_lower_case
   __A	:		List[str]	  =      strip_accents
   __A	:		Dict	  =      tokenize_chinese_chars
   __A	:		List[Any]	  =      normalizer_class(**__lowercase    )

  __A	:		int	  =      do_lower_case


 def 			snake_case__ (   self   ,   __lowercase   ,   **__lowercase    ):

  """simple docstring"""







  __A	:		Dict	  =      PaddingStrategy.MAX_LENGTH

  __A	:		Optional[int]	  =      text
  __A	:		Union[str, Any]	  =      kwargs.pop('text_pair'   ,   __lowercase    )
  __A	:		List[str]	  =      kwargs.pop('return_tensors'   ,   __lowercase    )

  __A	:		int	  =      {
      'input_ids': [],
      'attention_mask': [],
      'token_type_ids': [],
  }

  for idx, candidate_text in enumerate(__lowercase    ):
   if batch_text_pair is not None:
    __A	:		Tuple	  =      batch_text_pair[idx]
   else:
    __A	:		Union[str, Any]	  =      None

   __A	:		Optional[int]	  =      super().__call__(__lowercase   ,   __lowercase   ,   return_tensors=__lowercase   ,   **__lowercase    )

   __A	:		str	  =      encoded_candidates.get('input_ids'    )
   __A	:		Union[str, Any]	  =      encoded_candidates.get('attention_mask'    )
   __A	:		Optional[int]	  =      encoded_candidates.get('token_type_ids'    )

   if encoded_input_ids is not None:
    output_data["input_ids"].append(__lowercase    )
   if encoded_attention_mask is not None:
    output_data["attention_mask"].append(__lowercase    )
   if encoded_token_type_ids is not None:
    output_data["token_type_ids"].append(__lowercase    )

  __A	:		List[str]	  =      {key: item for key, item in output_data.items() if len(__lowercase    ) != 0}

  return BatchEncoding(__lowercase   ,   tensor_type=__lowercase    )


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

  if token_ids_a:
   output += token_ids_a + [self.sep_token_id]

  return output


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.sep_token_id]
  __A	:		Tuple	  =      [self.cls_token_id]
  if token_ids_a is None:
   return len(cls + token_ids_a + sep    ) * [0]
  return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		int	  =      self._tokenizer.model.save(__lowercase   ,   name=__lowercase    )
  return tuple(__lowercase    )






