


'''simple docstring'''






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class 						__UpperCamelCase	(						_UpperCAmelCase						,  unittest.TestCase							):
							lowercase			:		int   		=  DebertaTokenizer
							lowercase			:		Optional[int]   		=  True
							lowercase			:		Tuple   		=  DebertaTokenizerFast





							def 		a__   (  self		:List[str]			):
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									snake_case_		:							Optional[int]			       =						[
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """\u0120""",
									    """\u0120l""",
									    """\u0120n""",
									    """\u0120lo""",
									    """\u0120low""",
									    """er""",
									    """\u0120lowest""",
									    """\u0120newer""",
									    """\u0120wider""",
									    """[UNK]""",
									]
									snake_case_		:							Optional[Any]			       =						dict(zip(lowercase__  ,range(len(lowercase__			)			)			)			)
									snake_case_		:							Tuple			       =						["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
									snake_case_		:							int			       =						{"""unk_token""": """[UNK]"""}

									snake_case_		:							Union[str, Any]			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]			)
									snake_case_		:							str			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""merges_file"""]			)
									with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""			) as fp:
											fp.write(json.dumps(lowercase__			) + """\n"""			)
									with open(self.merges_file  ,"""w"""  ,encoding="""utf-8"""			) as fp:
											fp.write("""\n""".join(lowercase__			)			)





							def 		a__   (  self		:List[str]  ,**_UpperCamelCase		:Any			):
									kwargs.update(self.special_tokens_map			)
									return self.tokenizer_class.from_pretrained(self.tmpdirname  ,**lowercase__			)





							def 		a__   (  self		:str  ,_UpperCamelCase		:List[Any]			):
									snake_case_		:							Dict			       =						"""lower newer"""
									snake_case_		:							Tuple			       =						"""lower newer"""
									return input_text, output_text





							def 		a__   (  self		:List[str]			):
									snake_case_		:							int			       =						self.get_tokenizer()
									snake_case_		:							List[Any]			       =						"""lower newer"""
									snake_case_		:							str			       =						["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
									snake_case_		:							Tuple			       =						tokenizer.tokenize(lowercase__			)
									self.assertListEqual(lowercase__  ,lowercase__			)

									snake_case_		:							List[str]			       =						tokens + [tokenizer.unk_token]
									snake_case_		:							Any			       =						[0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__			)  ,lowercase__			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							str			       =						self.get_tokenizer()
									snake_case_		:							Optional[int]			       =						tokenizer("""Hello"""  ,"""World"""			)
									snake_case_		:							Dict			       =						[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
									self.assertListEqual(tokd["""token_type_ids"""]  ,lowercase__			)





							@slow
							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							Tuple			       =						self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""			)

									snake_case_		:							Dict			       =						tokenizer.encode("""sequence builders"""  ,add_special_tokens=lowercase__			)
									snake_case_		:							Optional[int]			       =						tokenizer.encode("""multi-sequence build"""  ,add_special_tokens=lowercase__			)

									snake_case_		:							Union[str, Any]			       =						tokenizer.encode(
									    """sequence builders"""  ,add_special_tokens=lowercase__  ,add_prefix_space=lowercase__			)
									snake_case_		:							int			       =						tokenizer.encode(
									    """sequence builders"""  ,"""multi-sequence build"""  ,add_special_tokens=lowercase__  ,add_prefix_space=lowercase__			)

									snake_case_		:							Union[str, Any]			       =						tokenizer.build_inputs_with_special_tokens(lowercase__			)
									snake_case_		:							Tuple			       =						tokenizer.build_inputs_with_special_tokens(lowercase__  ,lowercase__			)

									assert encoded_sentence == encoded_text_from_decode
									assert encoded_pair == encoded_pair_from_decode





							@slow
							def 		a__   (  self		:List[Any]			):
									snake_case_		:							str			       =						[self.tokenizer_class]
									if self.test_rust_tokenizer:
											tokenizer_classes.append(self.rust_tokenizer_class			)

									for tokenizer_class in tokenizer_classes:
											snake_case_		:							str			       =						tokenizer_class.from_pretrained("""microsoft/deberta-base"""			)

											snake_case_		:							Optional[Any]			       =						[
											    """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
											    """ALBERT incorporates two parameter reduction techniques""",
											    """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
											    """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
											    """ vocabulary embedding.""",
											]

											snake_case_		:							Optional[Any]			       =						tokenizer(lowercase__  ,padding=lowercase__			)
											snake_case_		:							List[str]			       =						[tokenizer.decode(lowercase__  ,skip_special_tokens=lowercase__			) for seq in encoding["""input_ids"""]]

											# fmt: off
											snake_case_		:							Optional[Any]			       =						{
											    """input_ids""": [
											        [1, 2_1_1_8, 1_1_1_2_6, 5_6_5, 3_5, 8_3, 2_5_1_9_1, 1_6_3, 1_8_8_5_4, 1_3, 1_2_1_5_6, 1_2, 1_6_1_0_1, 2_5_3_7_6, 1_3_8_0_7, 9, 2_2_2_0_5, 2_7_8_9_3, 1_6_3_5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 2_1_1_8, 1_1_1_2_6, 5_6_5, 2_4_5_3_6, 8_0, 4_3_7_9_7, 4_8_7_8, 7_3_7_3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 1_3_3, 7_8, 6_5, 1_6, 1_0, 3_7_2_4, 1_5_3_8, 3_3_1_8_3, 1_1_3_0_3, 4_3_7_9_7, 1_9_3_8, 4, 8_7_0, 2_4_1_6_5, 2_9_1_0_5, 5, 7_3_9, 3_2_6_4_4, 3_3_1_8_3, 1_1_3_0_3, 3_6_1_7_3, 8_8, 8_0, 6_5_0, 7_8_2_1, 4_5_9_4_0, 6, 5_2, 2_5_5_9, 5, 1_8_3_6, 9, 5, 7_3_9_7, 1_3_1_7_1, 3_1, 5, 1_8_3_6, 9, 3_2_6_4_4, 3_3_1_8_3, 1_1_3_0_3, 4, 2]
											    ],
											    """token_type_ids""": [
											        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
											    ],
											    """attention_mask""": [
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
											    ]
											}
											# fmt: on

											snake_case_		:							str			       =						[
											    """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
											    """ALBERT incorporates two parameter reduction techniques""",
											    """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
											    """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
											    """ vocabulary embedding.""",
											]

											self.assertDictEqual(encoding.data  ,lowercase__			)

											for expected, decoded in zip(lowercase__  ,lowercase__			):
													self.assertEqual(lowercase__  ,lowercase__			)




'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A	:	Optional[int]     =     logging.get_logger(__name__)

__A	:	List[Any]     =     {
    'studio-ousia/luke-base': 'https://huggingface.co/studio-ousia/luke-base/resolve/main/config.json',
    'studio-ousia/luke-large': 'https://huggingface.co/studio-ousia/luke-large/resolve/main/config.json',
}
class 						__UpperCamelCase	(						lowercase__							):
							lowercase			:		Union[str, Any]   		=  'luke'





							def __init__(  self		:List[str]  ,_UpperCamelCase		:Tuple=5_0_2_6_7  ,_UpperCamelCase		:Tuple=5_0_0_0_0_0  ,_UpperCamelCase		:str=7_6_8  ,_UpperCamelCase		:List[Any]=2_5_6  ,_UpperCamelCase		:List[str]=1_2  ,_UpperCamelCase		:str=1_2  ,_UpperCamelCase		:Union[str, Any]=3_0_7_2  ,_UpperCamelCase		:int="gelu"  ,_UpperCamelCase		:int=0.1  ,_UpperCamelCase		:Dict=0.1  ,_UpperCamelCase		:Dict=5_1_2  ,_UpperCamelCase		:Optional[Any]=2  ,_UpperCamelCase		:int=0.02  ,_UpperCamelCase		:Any=1E-1_2  ,_UpperCamelCase		:Tuple=True  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:Any=1  ,_UpperCamelCase		:int=0  ,_UpperCamelCase		:List[Any]=2  ,**_UpperCamelCase		:str  ,):
									super().__init__(pad_token_id=_UpperCamelCase  ,bos_token_id=_UpperCamelCase  ,eos_token_id=_UpperCamelCase  ,**_UpperCamelCase			)

									snake_case_		:							Any			       =						vocab_size
									snake_case_		:							List[Any]			       =						entity_vocab_size
									snake_case_		:							Optional[Any]			       =						hidden_size
									snake_case_		:							List[Any]			       =						entity_emb_size
									snake_case_		:							Dict			       =						num_hidden_layers
									snake_case_		:							Union[str, Any]			       =						num_attention_heads
									snake_case_		:							Dict			       =						hidden_act
									snake_case_		:							Dict			       =						intermediate_size
									snake_case_		:							Any			       =						hidden_dropout_prob
									snake_case_		:							List[Any]			       =						attention_probs_dropout_prob
									snake_case_		:							Optional[Any]			       =						max_position_embeddings
									snake_case_		:							Any			       =						type_vocab_size
									snake_case_		:							int			       =						initializer_range
									snake_case_		:							Union[str, Any]			       =						layer_norm_eps
									snake_case_		:							Tuple			       =						use_entity_aware_attention
									snake_case_		:							Tuple			       =						classifier_dropout