





from ... import PretrainedConfig


_A					:				Any				=			{
    """sijunhe/nezha-cn-base""": """https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json""",
}



class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    lowerCamelCase__    :		str				=	NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
    lowerCamelCase__    :		Optional[Any]				=	"""nezha"""



    def __init__(       self   ,				A_=2_11_28   ,				A_=7_68   ,				A_=12   ,				A_=12   ,				A_=30_72   ,				A_="gelu"   ,				A_=0.1   ,				A_=0.1   ,				A_=5_12   ,				A_=64   ,				A_=2   ,				A_=0.02   ,				A_=1E-12   ,				A_=0.1   ,				A_=0   ,				A_=2   ,				A_=3   ,				A_=True   ,				**A_   ,				):





        '''simple docstring'''

        super().__init__(pad_token_id=A_   ,				bos_token_id=A_   ,				eos_token_id=A_   ,				**A_     )

        SCREAMING_SNAKE_CASE__											=			vocab_size
        SCREAMING_SNAKE_CASE__											=			hidden_size
        SCREAMING_SNAKE_CASE__											=			num_hidden_layers
        SCREAMING_SNAKE_CASE__											=			num_attention_heads
        SCREAMING_SNAKE_CASE__											=			hidden_act
        SCREAMING_SNAKE_CASE__											=			intermediate_size
        SCREAMING_SNAKE_CASE__											=			hidden_dropout_prob
        SCREAMING_SNAKE_CASE__											=			attention_probs_dropout_prob
        SCREAMING_SNAKE_CASE__											=			max_position_embeddings
        SCREAMING_SNAKE_CASE__											=			max_relative_position
        SCREAMING_SNAKE_CASE__											=			type_vocab_size
        SCREAMING_SNAKE_CASE__											=			initializer_range
        SCREAMING_SNAKE_CASE__											=			layer_norm_eps
        SCREAMING_SNAKE_CASE__											=			classifier_dropout
        SCREAMING_SNAKE_CASE__											=			use_cache




"""simple docstring"""






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class      A(  lowerCamelCase__				,							unittest.TestCase		):

		"""simple docstring"""





		A    =							CpmAntTokenizer
		A    =							False
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					super().setUp()

					_UpperCamelCase					:str					     =	[
					    '''<d>''',
					    '''</d>''',
					    '''<s>''',
					    '''</s>''',
					    '''</_>''',
					    '''<unk>''',
					    '''<pad>''',
					    '''</n>''',
					    '''我''',
					    '''是''',
					    '''C''',
					    '''P''',
					    '''M''',
					    '''A''',
					    '''n''',
					    '''t''',
					]
					_UpperCamelCase					:List[str]					     =	os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES['''vocab_file''']   )
					with open(self.vocab_file					,				'''w'''					,				encoding='''utf-8'''   ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )





		@tooslow
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					_UpperCamelCase					:Any					     =	CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''   )
					_UpperCamelCase					:List[str]					     =	'''今天天气真好！'''
					_UpperCamelCase					:Tuple					     =	['''今天''', '''天气''', '''真''', '''好''', '''！''']
					_UpperCamelCase					:Tuple					     =	tokenizer.tokenize(SCREAMING_SNAKE_CASE__   )
					self.assertListEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Dict					     =	'''今天天气真好！'''
					_UpperCamelCase					:Optional[int]					     =	[tokenizer.bos_token] + tokens

					_UpperCamelCase					:List[str]					     =	[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__   )					,				SCREAMING_SNAKE_CASE__   )

					_UpperCamelCase					:str					     =	tokenizer.decode(SCREAMING_SNAKE_CASE__   )
					self.assertEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )

