



import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Optional[Any]     =			OpenAIGPTTokenizer
			UpperCamelCase					:Optional[Any]     =			OpenAIGPTTokenizerFast
			UpperCamelCase					:str     =			True
			UpperCamelCase					:Any     =			False





			def        _snake_case    (self				):
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						lowerCamelCase__ :      Optional[Any]								=		[
						    """l""",
						    """o""",
						    """w""",
						    """e""",
						    """r""",
						    """s""",
						    """t""",
						    """i""",
						    """d""",
						    """n""",
						    """w</w>""",
						    """r</w>""",
						    """t</w>""",
						    """lo""",
						    """low""",
						    """er</w>""",
						    """low</w>""",
						    """lowest</w>""",
						    """newer</w>""",
						    """wider</w>""",
						    """<unk>""",
						]
						lowerCamelCase__ :      Any								=		dict(zip(__magic_name__							,    range(len(__magic_name__				)				)				)				)
						lowerCamelCase__ :      Union[str, Any]								=		["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

						lowerCamelCase__ :      Any								=		os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""vocab_file"""]				)
						lowerCamelCase__ :      Union[str, Any]								=		os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES["""merges_file"""]				)
						with open(self.vocab_file							,    """w"""				) as fp:
									fp.write(json.dumps(__magic_name__				)				)
						with open(self.merges_file							,    """w"""				) as fp:
									fp.write("""\n""".join(__magic_name__				)				)





			def        _snake_case    (self							,    __magic_name__				):
						return "lower newer", "lower newer"





			def        _snake_case    (self				):
						lowerCamelCase__ :      str								=		OpenAIGPTTokenizer(self.vocab_file							,    self.merges_file				)

						lowerCamelCase__ :      int								=		"""lower"""
						lowerCamelCase__ :      Optional[Any]								=		["""low""", """er</w>"""]
						lowerCamelCase__ :      int								=		tokenizer.tokenize(__magic_name__				)
						self.assertListEqual(__magic_name__							,    __magic_name__				)

						lowerCamelCase__ :      Dict								=		tokens + ["""<unk>"""]
						lowerCamelCase__ :      Dict								=		[14, 15, 20]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)							,    __magic_name__				)





			def        _snake_case    (self							,    __magic_name__=15				):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
												lowerCamelCase__ :      Optional[int]								=		self.rust_tokenizer_class.from_pretrained(__magic_name__							,    **__magic_name__				)

												# Simple input
												lowerCamelCase__ :      Tuple								=		"""This is a simple input"""
												lowerCamelCase__ :      Optional[Any]								=		["""This is a simple input 1""", """This is a simple input 2"""]
												lowerCamelCase__ :      List[str]								=		("""This is a simple input""", """This is a pair""")
												lowerCamelCase__ :      Any								=		[
												    ("""This is a simple input 1""", """This is a simple input 2"""),
												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
												]

												# Simple input tests
												self.assertRaises(__magic_name__							,    tokenizer_r.encode							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""				)

												# Simple input
												self.assertRaises(__magic_name__							,    tokenizer_r.encode_plus							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""				)

												# Simple input
												self.assertRaises(
												    __magic_name__							,    tokenizer_r.batch_encode_plus							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""							,    )

												# Pair input
												self.assertRaises(__magic_name__							,    tokenizer_r.encode							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""				)

												# Pair input
												self.assertRaises(__magic_name__							,    tokenizer_r.encode_plus							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""				)

												# Pair input
												self.assertRaises(
												    __magic_name__							,    tokenizer_r.batch_encode_plus							,    __magic_name__							,    max_length=__magic_name__							,    padding="""max_length"""							,    )





			def        _snake_case    (self				):
						pass







@require_ftfy
@require_spacy
@require_tokenizers
class 				__A					( A_      ):
			pass








from ... import PretrainedConfig


_lowercase     							=						{
    '''sijunhe/nezha-cn-base''': '''https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json''',
}
class 				__A					( A_      ):
			UpperCamelCase					:Optional[Any]     =			NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
			UpperCamelCase					:int     =			'''nezha'''





			def __init__(self							,    __magic_name__=21128							,    __magic_name__=768							,    __magic_name__=12							,    __magic_name__=12							,    __magic_name__=3072							,    __magic_name__="gelu"							,    __magic_name__=0.1							,    __magic_name__=0.1							,    __magic_name__=512							,    __magic_name__=64							,    __magic_name__=2							,    __magic_name__=0.02							,    __magic_name__=1E-12							,    __magic_name__=0.1							,    __magic_name__=0							,    __magic_name__=2							,    __magic_name__=3							,    __magic_name__=True							,    **__magic_name__							,    ):
						super().__init__(pad_token_id=__magic_name__							,    bos_token_id=__magic_name__							,    eos_token_id=__magic_name__							,    **__magic_name__				)

						lowerCamelCase__ :      List[str]								=		vocab_size
						lowerCamelCase__ :      int								=		hidden_size
						lowerCamelCase__ :      int								=		num_hidden_layers
						lowerCamelCase__ :      Optional[int]								=		num_attention_heads
						lowerCamelCase__ :      Dict								=		hidden_act
						lowerCamelCase__ :      str								=		intermediate_size
						lowerCamelCase__ :      Optional[Any]								=		hidden_dropout_prob
						lowerCamelCase__ :      Dict								=		attention_probs_dropout_prob
						lowerCamelCase__ :      str								=		max_position_embeddings
						lowerCamelCase__ :      Optional[Any]								=		max_relative_position
						lowerCamelCase__ :      Optional[int]								=		type_vocab_size
						lowerCamelCase__ :      Optional[Any]								=		initializer_range
						lowerCamelCase__ :      Any								=		layer_norm_eps
						lowerCamelCase__ :      Union[str, Any]								=		classifier_dropout
						lowerCamelCase__ :      Union[str, Any]								=		use_cache



