





"""simple docstring"""





import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin






class    _lowerCAmelCase  (	a			):
       """simple docstring"""




       __magic_name__	:List[Any]						    =   None
       __magic_name__	:Union[str, Any]						    =   None






       @property
       def 	snake_case		(       self ):



         '''simple docstring'''

         return self.feat_extract_tester.prepare_feat_extract_dict()






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[int]											=				self.feature_extraction_class(**self.feat_extract_dict )
         self.assertTrue(hasattr(__UpperCAmelCase       ,					'feature_size' ) )
         self.assertTrue(hasattr(__UpperCAmelCase       ,					'sampling_rate' ) )
         self.assertTrue(hasattr(__UpperCAmelCase       ,					'padding_value' ) )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[Any]											=				self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__     :Optional[int]											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :List[Any]											=				BatchFeature({input_name: speech_inputs} )

         self.assertTrue(all(len(__UpperCAmelCase ) == len(__UpperCAmelCase ) for x, y in zip(__UpperCAmelCase       ,					processed_features[input_name] ) ) )

         lowerCAmelCase__     :Optional[Any]											=				self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[Any]											=				BatchFeature({input_name: speech_inputs}       ,					tensor_type='np' )

         lowerCAmelCase__     :Tuple											=				processed_features[input_name]

         if len(batch_features_input.shape ) < 3:
           lowerCAmelCase__     :Tuple											=				batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0] ), self.feat_extract_tester.feature_size) )






       @require_torch
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Dict											=				self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCAmelCase )
         lowerCAmelCase__     :Dict											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :Tuple											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :int											=				BatchFeature({input_name: speech_inputs}       ,					tensor_type='pt' )

         lowerCAmelCase__     :List[Any]											=				processed_features[input_name]

         if len(batch_features_input.shape ) < 3:
           lowerCAmelCase__     :Dict											=				batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0] ), self.feat_extract_tester.feature_size) )






       @require_tf
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :List[str]											=				self.feat_extract_tester.prepare_inputs_for_common(equal_length=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[Any]											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :List[Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :str											=				BatchFeature({input_name: speech_inputs}       ,					tensor_type='tf' )

         lowerCAmelCase__     :Any											=				processed_features[input_name]

         if len(batch_features_input.shape ) < 3:
           lowerCAmelCase__     :Union[str, Any]											=				batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0] ), self.feat_extract_tester.feature_size) )






       def 	snake_case		(       self       ,					__UpperCAmelCase=False ):



         '''simple docstring'''

         def _inputs_have_equal_length(__UpperCAmelCase ):
           lowerCAmelCase__     :int											=				len(input[0] )
           for input_slice in input[1:]:
             if len(__UpperCAmelCase ) != length:
               return False
           return True

         def _inputs_are_equal(__UpperCAmelCase       ,					__UpperCAmelCase ):
           if len(__UpperCAmelCase ) != len(__UpperCAmelCase ):
             return False

           for input_slice_a, input_slice_a in zip(__UpperCAmelCase       ,					__UpperCAmelCase ):
             if not np.allclose(np.asarray(__UpperCAmelCase )       ,					np.asarray(__UpperCAmelCase )       ,					atol=1E-3 ):
               return False
           return True

         lowerCAmelCase__     :Union[str, Any]											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :int											=				self.feat_extract_tester.prepare_inputs_for_common(numpify=__UpperCAmelCase )
         lowerCAmelCase__     :List[Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :List[Any]											=				BatchFeature({input_name: speech_inputs} )

         lowerCAmelCase__     :List[Any]											=				self.feat_extract_tester.seq_length_diff
         lowerCAmelCase__     :List[str]											=				self.feat_extract_tester.max_seq_length + pad_diff
         lowerCAmelCase__     :Tuple											=				self.feat_extract_tester.min_seq_length
         lowerCAmelCase__     :Optional[Any]											=				self.feat_extract_tester.batch_size
         lowerCAmelCase__     :List[str]											=				self.feat_extract_tester.feature_size

         # test padding for List[int] + numpy
         lowerCAmelCase__     :Tuple											=				feat_extract.pad(__UpperCAmelCase       ,					padding=__UpperCAmelCase )
         lowerCAmelCase__     :Tuple											=				input_a[input_name]

         lowerCAmelCase__     :List[str]											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest' )
         lowerCAmelCase__     :Dict											=				input_a[input_name]

         lowerCAmelCase__     :Optional[int]											=				feat_extract.pad(__UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[-1] ) )
         lowerCAmelCase__     :Union[str, Any]											=				input_a[input_name]

         lowerCAmelCase__     :Any											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='np' )
         lowerCAmelCase__     :Dict											=				input_a[input_name]

         # max_length parameter has to be provided when setting `padding="max_length"`
         with self.assertRaises(__UpperCAmelCase ):
           feat_extract.pad(__UpperCAmelCase       ,					padding='max_length' )[input_name]

         lowerCAmelCase__     :Any											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=__UpperCAmelCase       ,					return_tensors='np' )
         lowerCAmelCase__     :str											=				input_a[input_name]

         self.assertFalse(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(_inputs_are_equal(__UpperCAmelCase       ,					__UpperCAmelCase ) )
         self.assertTrue(len(input_a[0] ) == pad_min_length )
         self.assertTrue(len(input_a[1] ) == pad_min_length + pad_diff )
         self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0] )) )
         self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length) )

         if feature_size > 1:
           self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size )

         # test padding for `pad_to_multiple_of` for List[int] + numpy
         lowerCAmelCase__     :Any											=				feat_extract.pad(__UpperCAmelCase       ,					pad_to_multiple_of=1_0 )
         lowerCAmelCase__     :List[str]											=				input_a[input_name]

         lowerCAmelCase__     :str											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					pad_to_multiple_of=1_0 )
         lowerCAmelCase__     :Tuple											=				input_a[input_name]

         lowerCAmelCase__     :List[str]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					pad_to_multiple_of=1_0       ,					max_length=__UpperCAmelCase )
         lowerCAmelCase__     :Any											=				input_a[input_name]

         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					pad_to_multiple_of=1_0       ,					max_length=__UpperCAmelCase       ,					return_tensors='np'       ,					)
         lowerCAmelCase__     :Union[str, Any]											=				input_a[input_name]

         self.assertTrue(all(len(__UpperCAmelCase ) % 1_0 == 0 for x in input_a ) )
         self.assertTrue(_inputs_are_equal(__UpperCAmelCase       ,					__UpperCAmelCase ) )

         lowerCAmelCase__     :Any											=				pad_max_length if pad_max_length % 1_0 == 0 else (pad_max_length // 1_0 + 1) * 1_0
         self.assertTrue(all(len(__UpperCAmelCase ) == expected_mult_pad_length for x in input_a ) )
         self.assertEqual(input_a.shape[:2]       ,					(batch_size, expected_mult_pad_length) )

         if feature_size > 1:
           self.assertTrue(input_a.shape[2] == feature_size )

         # Check padding value is correct
         lowerCAmelCase__     :List[Any]											=				(np.ones(self.feat_extract_tester.feature_size ) * feat_extract.padding_value).sum()
         self.assertTrue(
             abs(np.asarray(input_a[0] )[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length) )
             < 1E-3 )
         self.assertTrue(
             abs(
                 np.asarray(input_a[1] )[pad_min_length + pad_diff :].sum()
                 - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff) )
             < 1E-3 )
         self.assertTrue(
             abs(
                 np.asarray(input_a[2] )[pad_min_length + 2 * pad_diff :].sum()
                 - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff) )
             < 1E-3 )
         self.assertTrue(
             abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length) ) < 1E-3 )
         self.assertTrue(
             abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length) )
             < 1E-3 )






       def 	snake_case		(       self       ,					__UpperCAmelCase=False ):



         '''simple docstring'''

         def _inputs_have_equal_length(__UpperCAmelCase ):
           lowerCAmelCase__     :List[str]											=				len(input[0] )
           for input_slice in input[1:]:
             if len(__UpperCAmelCase ) != length:
               return False
           return True

         def _inputs_are_equal(__UpperCAmelCase       ,					__UpperCAmelCase ):
           if len(__UpperCAmelCase ) != len(__UpperCAmelCase ):
             return False

           for input_slice_a, input_slice_a in zip(__UpperCAmelCase       ,					__UpperCAmelCase ):
             if not np.allclose(np.asarray(__UpperCAmelCase )       ,					np.asarray(__UpperCAmelCase )       ,					atol=1E-3 ):
               return False
           return True

         lowerCAmelCase__     :Any											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :Union[str, Any]											=				self.feat_extract_tester.prepare_inputs_for_common(numpify=__UpperCAmelCase )
         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :str											=				BatchFeature({input_name: speech_inputs} )

         # truncate to smallest
         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] )       ,					truncation=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[int]											=				input_a[input_name]

         lowerCAmelCase__     :Optional[Any]											=				feat_extract.pad(__UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] ) )
         lowerCAmelCase__     :Any											=				input_a[input_name]

         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertFalse(_inputs_have_equal_length(__UpperCAmelCase ) )

         # truncate to smallest with np
         lowerCAmelCase__     :Any											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] )       ,					return_tensors='np'       ,					truncation=__UpperCAmelCase       ,					)
         lowerCAmelCase__     :List[Any]											=				input_a[input_name]

         lowerCAmelCase__     :Any											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] )       ,					return_tensors='np' )
         lowerCAmelCase__     :Tuple											=				input_a[input_name]

         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(input_a.shape[1] == len(speech_inputs[0] ) )

         # since truncation forces padding to be smaller than longest input
         # function can't return `np.ndarray`, but has to return list
         self.assertFalse(_inputs_have_equal_length(__UpperCAmelCase ) )

         # truncate to middle
         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[1] )       ,					truncation=__UpperCAmelCase       ,					return_tensors='np'       ,					)
         lowerCAmelCase__     :str											=				input_a[input_name]

         lowerCAmelCase__     :Dict											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[1] )       ,					truncation=__UpperCAmelCase )
         lowerCAmelCase__     :Optional[int]											=				input_a[input_name]

         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[1] )       ,					return_tensors='np' )
         lowerCAmelCase__     :Union[str, Any]											=				input_a[input_name]

         self.assertTrue(input_a.shape[1] == len(speech_inputs[1] ) )
         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(_inputs_are_equal(__UpperCAmelCase       ,					__UpperCAmelCase ) )

         # since truncation forces padding to be smaller than longest input
         # function can't return `np.ndarray`, but has to return list
         self.assertFalse(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertTrue(len(input_a[-1] ) == len(speech_inputs[-1] ) )

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(__UpperCAmelCase ):
           feat_extract.pad(__UpperCAmelCase       ,					truncation=__UpperCAmelCase )[input_name]

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(__UpperCAmelCase ):
           feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					truncation=__UpperCAmelCase )[input_name]

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(__UpperCAmelCase ):
           feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					truncation=__UpperCAmelCase )[input_name]

         # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
         with self.assertRaises(__UpperCAmelCase ):
           feat_extract.pad(__UpperCAmelCase       ,					padding='max_length'       ,					truncation=__UpperCAmelCase )[input_name]

         # test truncation for `pad_to_multiple_of` for List[int] + numpy
         lowerCAmelCase__     :Dict											=				1_2
         lowerCAmelCase__     :Any											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] )       ,					pad_to_multiple_of=__UpperCAmelCase       ,					truncation=__UpperCAmelCase       ,					)
         lowerCAmelCase__     :Dict											=				input_a[input_name]

         lowerCAmelCase__     :Dict											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=len(speech_inputs[0] )       ,					pad_to_multiple_of=__UpperCAmelCase       ,					)
         lowerCAmelCase__     :List[Any]											=				input_a[input_name]

         # retrieve expected_length as multiple of pad_to_multiple_of
         lowerCAmelCase__     :Optional[Any]											=				len(speech_inputs[0] )
         if expected_length % pad_to_multiple_of != 0:
           lowerCAmelCase__     :Optional[int]											=				((len(speech_inputs[0] ) // pad_to_multiple_of) + 1) * pad_to_multiple_of

         self.assertTrue(len(input_a[0] ) == expected_length )
         self.assertTrue(_inputs_have_equal_length(__UpperCAmelCase ) )
         self.assertFalse(_inputs_have_equal_length(__UpperCAmelCase ) )






       def 	snake_case		(       self ):



         '''simple docstring'''

         self._check_padding(numpify=__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         self._check_padding(numpify=__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         self._check_truncation(numpify=__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         self._check_truncation(numpify=__UpperCAmelCase )






       @require_torch
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :int											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :Optional[Any]											=				self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__     :Optional[int]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :Optional[Any]											=				BatchFeature({input_name: speech_inputs} )

         lowerCAmelCase__     :int											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='np' )[input_name]
         lowerCAmelCase__     :Tuple											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='pt' )[input_name]

         self.assertTrue(abs(input_np.astype(np.floataa ).sum() - input_pt.numpy().astype(np.floataa ).sum() ) < 1E-2 )






       @require_tf
       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Tuple											=				self.feature_extraction_class(**self.feat_extract_dict )
         lowerCAmelCase__     :Optional[Any]											=				self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__     :Union[str, Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :int											=				BatchFeature({input_name: speech_inputs} )

         lowerCAmelCase__     :Optional[int]											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='np' )[input_name]
         lowerCAmelCase__     :Tuple											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='tf' )[input_name]

         self.assertTrue(abs(input_np.astype(np.floataa ).sum() - input_tf.numpy().astype(np.floataa ).sum() ) < 1E-2 )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :List[Any]											=				self.feat_extract_dict
         lowerCAmelCase__     :Optional[int]											=				True
         lowerCAmelCase__     :List[str]											=				self.feature_extraction_class(**__UpperCAmelCase )
         lowerCAmelCase__     :Dict											=				self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__     :int											=				[len(__UpperCAmelCase ) for x in speech_inputs]
         lowerCAmelCase__     :List[Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :Optional[int]											=				BatchFeature({input_name: speech_inputs} )

         lowerCAmelCase__     :Dict											=				feat_extract.pad(__UpperCAmelCase       ,					padding='longest'       ,					return_tensors='np' )
         self.assertIn('attention_mask'       ,					__UpperCAmelCase )
         self.assertListEqual(list(processed.attention_mask.shape )       ,					list(processed[input_name].shape[:2] ) )
         self.assertListEqual(processed.attention_mask.sum(-1 ).tolist()       ,					__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Tuple											=				self.feat_extract_dict
         lowerCAmelCase__     :int											=				True
         lowerCAmelCase__     :Any											=				self.feature_extraction_class(**__UpperCAmelCase )
         lowerCAmelCase__     :Any											=				self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__     :List[str]											=				[len(__UpperCAmelCase ) for x in speech_inputs]
         lowerCAmelCase__     :List[Any]											=				feat_extract.model_input_names[0]

         lowerCAmelCase__     :int											=				BatchFeature({input_name: speech_inputs} )
         lowerCAmelCase__     :int											=				min(__UpperCAmelCase )

         lowerCAmelCase__     :List[Any]											=				feat_extract.pad(
             __UpperCAmelCase       ,					padding='max_length'       ,					max_length=__UpperCAmelCase       ,					truncation=__UpperCAmelCase       ,					return_tensors='np' )
         self.assertIn('attention_mask'       ,					__UpperCAmelCase )
         self.assertListEqual(
             list(processed_pad.attention_mask.shape )       ,					[processed_pad[input_name].shape[0], max_length] )
         self.assertListEqual(
             processed_pad.attention_mask[:, :max_length].sum(-1 ).tolist()       ,					[max_length for x in speech_inputs] )




'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase_			:  int										=   logging.get_logger(__name__)

UpperCAmelCase_			:  int										=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_			:  List[str]										=   {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}

class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Union[str, Any]     				=    VOCAB_FILES_NAMES
 __lowercase					:		int     				=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_INIT_CONFIGURATION
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase					:		List[str]     				=    RealmTokenizer


 def __init__(   self   ,   __lowercase=None   ,   __lowercase=None   ,   __lowercase=True   ,   __lowercase="[UNK]"   ,   __lowercase="[SEP]"   ,   __lowercase="[PAD]"   ,   __lowercase="[CLS]"   ,   __lowercase="[MASK]"   ,   __lowercase=True   ,   __lowercase=None   ,   **__lowercase   ,   ):

  """simple docstring"""







  super().__init__(
      __lowercase   ,   tokenizer_file=__lowercase   ,   do_lower_case=__lowercase   ,   unk_token=__lowercase   ,   sep_token=__lowercase   ,   pad_token=__lowercase   ,   cls_token=__lowercase   ,   mask_token=__lowercase   ,   tokenize_chinese_chars=__lowercase   ,   strip_accents=__lowercase   ,   **__lowercase   ,   )

  __A	:		Optional[int]	  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
  if (
      normalizer_state.get('lowercase'   ,   __lowercase    ) != do_lower_case
      or normalizer_state.get('strip_accents'   ,   __lowercase    ) != strip_accents
      or normalizer_state.get('handle_chinese_chars'   ,   __lowercase    ) != tokenize_chinese_chars
  ):
   __A	:		Tuple	  =      getattr(__lowercase   ,   normalizer_state.pop('type'    )    )
   __A	:		Optional[int]	  =      do_lower_case
   __A	:		List[str]	  =      strip_accents
   __A	:		Dict	  =      tokenize_chinese_chars
   __A	:		List[Any]	  =      normalizer_class(**__lowercase    )

  __A	:		int	  =      do_lower_case


 def 			snake_case__ (   self   ,   __lowercase   ,   **__lowercase    ):

  """simple docstring"""







  __A	:		Dict	  =      PaddingStrategy.MAX_LENGTH

  __A	:		Optional[int]	  =      text
  __A	:		Union[str, Any]	  =      kwargs.pop('text_pair'   ,   __lowercase    )
  __A	:		List[str]	  =      kwargs.pop('return_tensors'   ,   __lowercase    )

  __A	:		int	  =      {
      'input_ids': [],
      'attention_mask': [],
      'token_type_ids': [],
  }

  for idx, candidate_text in enumerate(__lowercase    ):
   if batch_text_pair is not None:
    __A	:		Tuple	  =      batch_text_pair[idx]
   else:
    __A	:		Union[str, Any]	  =      None

   __A	:		Optional[int]	  =      super().__call__(__lowercase   ,   __lowercase   ,   return_tensors=__lowercase   ,   **__lowercase    )

   __A	:		str	  =      encoded_candidates.get('input_ids'    )
   __A	:		Union[str, Any]	  =      encoded_candidates.get('attention_mask'    )
   __A	:		Optional[int]	  =      encoded_candidates.get('token_type_ids'    )

   if encoded_input_ids is not None:
    output_data["input_ids"].append(__lowercase    )
   if encoded_attention_mask is not None:
    output_data["attention_mask"].append(__lowercase    )
   if encoded_token_type_ids is not None:
    output_data["token_type_ids"].append(__lowercase    )

  __A	:		List[str]	  =      {key: item for key, item in output_data.items() if len(__lowercase    ) != 0}

  return BatchEncoding(__lowercase   ,   tensor_type=__lowercase    )


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

  if token_ids_a:
   output += token_ids_a + [self.sep_token_id]

  return output


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.sep_token_id]
  __A	:		Tuple	  =      [self.cls_token_id]
  if token_ids_a is None:
   return len(cls + token_ids_a + sep    ) * [0]
  return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		int	  =      self._tokenizer.model.save(__lowercase   ,   name=__lowercase    )
  return tuple(__lowercase    )






