


"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__				:Optional[Any]			   =	logging.get_logger(__name__)

UpperCamelCase__				:Dict			   =	{
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}






class      A(  lowerCamelCase__		):

		"""simple docstring"""





		A    =							"realm"
		def __init__( self					,				SCREAMING_SNAKE_CASE__=3_05_22					,				SCREAMING_SNAKE_CASE__=7_68					,				SCREAMING_SNAKE_CASE__=1_28					,				SCREAMING_SNAKE_CASE__=12					,				SCREAMING_SNAKE_CASE__=12					,				SCREAMING_SNAKE_CASE__=8					,				SCREAMING_SNAKE_CASE__=30_72					,				SCREAMING_SNAKE_CASE__="gelu_new"					,				SCREAMING_SNAKE_CASE__=0.1					,				SCREAMING_SNAKE_CASE__=0.1					,				SCREAMING_SNAKE_CASE__=5_12					,				SCREAMING_SNAKE_CASE__=2					,				SCREAMING_SNAKE_CASE__=0.0_2					,				SCREAMING_SNAKE_CASE__=1E-12					,				SCREAMING_SNAKE_CASE__=2_56					,				SCREAMING_SNAKE_CASE__=10					,				SCREAMING_SNAKE_CASE__=1E-3					,				SCREAMING_SNAKE_CASE__=5					,				SCREAMING_SNAKE_CASE__=3_20					,				SCREAMING_SNAKE_CASE__=13_35_37_18					,				SCREAMING_SNAKE_CASE__=50_00					,				SCREAMING_SNAKE_CASE__=1					,				SCREAMING_SNAKE_CASE__=0					,				SCREAMING_SNAKE_CASE__=2					,				**SCREAMING_SNAKE_CASE__					,				)      ->	Any:

					"""simple docstring"""



					super().__init__(pad_token_id=SCREAMING_SNAKE_CASE__					,				bos_token_id=SCREAMING_SNAKE_CASE__					,				eos_token_id=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					# Common config
					_UpperCamelCase					:int					     =	vocab_size
					_UpperCamelCase					:Optional[Any]					     =	max_position_embeddings
					_UpperCamelCase					:Optional[Any]					     =	hidden_size
					_UpperCamelCase					:List[Any]					     =	retriever_proj_size
					_UpperCamelCase					:Optional[int]					     =	num_hidden_layers
					_UpperCamelCase					:List[Any]					     =	num_attention_heads
					_UpperCamelCase					:List[str]					     =	num_candidates
					_UpperCamelCase					:int					     =	intermediate_size
					_UpperCamelCase					:str					     =	hidden_act
					_UpperCamelCase					:List[str]					     =	hidden_dropout_prob
					_UpperCamelCase					:int					     =	attention_probs_dropout_prob
					_UpperCamelCase					:Tuple					     =	initializer_range
					_UpperCamelCase					:Optional[int]					     =	type_vocab_size
					_UpperCamelCase					:Union[str, Any]					     =	layer_norm_eps

					# Reader config
					_UpperCamelCase					:Optional[int]					     =	span_hidden_size
					_UpperCamelCase					:str					     =	max_span_width
					_UpperCamelCase					:Tuple					     =	reader_layer_norm_eps
					_UpperCamelCase					:Any					     =	reader_beam_size
					_UpperCamelCase					:Dict					     =	reader_seq_len

					# Retrieval config
					_UpperCamelCase					:str					     =	num_block_records
					_UpperCamelCase					:int					     =	searcher_beam_size





"""simple docstring"""






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class      A(  lowerCamelCase__				,							unittest.TestCase		):

		"""simple docstring"""





		A    =							CpmAntTokenizer
		A    =							False
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					super().setUp()

					_UpperCamelCase					:str					     =	[
					    '''<d>''',
					    '''</d>''',
					    '''<s>''',
					    '''</s>''',
					    '''</_>''',
					    '''<unk>''',
					    '''<pad>''',
					    '''</n>''',
					    '''我''',
					    '''是''',
					    '''C''',
					    '''P''',
					    '''M''',
					    '''A''',
					    '''n''',
					    '''t''',
					]
					_UpperCamelCase					:List[str]					     =	os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES['''vocab_file''']   )
					with open(self.vocab_file					,				'''w'''					,				encoding='''utf-8'''   ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )





		@tooslow
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					_UpperCamelCase					:Any					     =	CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''   )
					_UpperCamelCase					:List[str]					     =	'''今天天气真好！'''
					_UpperCamelCase					:Tuple					     =	['''今天''', '''天气''', '''真''', '''好''', '''！''']
					_UpperCamelCase					:Tuple					     =	tokenizer.tokenize(SCREAMING_SNAKE_CASE__   )
					self.assertListEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Dict					     =	'''今天天气真好！'''
					_UpperCamelCase					:Optional[int]					     =	[tokenizer.bos_token] + tokens

					_UpperCamelCase					:List[str]					     =	[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__   )					,				SCREAMING_SNAKE_CASE__   )

					_UpperCamelCase					:str					     =	tokenizer.decode(SCREAMING_SNAKE_CASE__   )
					self.assertEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )

