






import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							_UpperCamelCase			(a_							,		unittest.TestCase			):
      snake_case_  =							LxmertTokenizer
      snake_case_  =							LxmertTokenizerFast
      snake_case_  =							True
      snake_case_  =							True


      def  __UpperCAmelCase		(	self	)->	Tuple:
           super().setUp()

           __lowerCAmelCase								=					[
               "[UNK]",
               "[CLS]",
               "[SEP]",
               "want",
               "##want",
               "##ed",
               "wa",
               "un",
               "runn",
               "##ing",
               ",",
               "low",
               "lowest",
           ]
           __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]	)
           with open(self.vocab_file					,						"w"					,						encoding="utf-8"	) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	str:
           __lowerCAmelCase								=					"UNwant\u00E9d,running"
           __lowerCAmelCase								=					"unwanted, running"
           return input_text, output_text


      def  __UpperCAmelCase		(	self	)->	Dict:
           __lowerCAmelCase								=					self.tokenizer_class(self.vocab_file	)

           __lowerCAmelCase								=					tokenizer.tokenize("UNwant\u00E9d,running"	)
           self.assertListEqual(__UpperCamelCase					,						["un", "##want", "##ed", ",", "runn", "##ing"]	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase	)					,						[7, 4, 5, 1_0, 8, 9]	)


      def  __UpperCAmelCase		(	self	)->	int:
           if not self.test_rust_tokenizer:
                return

           __lowerCAmelCase								=					self.get_tokenizer()
           __lowerCAmelCase								=					self.get_rust_tokenizer()

           __lowerCAmelCase								=					"I was born in 92000, and this is falsé."

           __lowerCAmelCase								=					tokenizer.tokenize(__UpperCamelCase	)
           __lowerCAmelCase								=					rust_tokenizer.tokenize(__UpperCamelCase	)
           self.assertListEqual(__UpperCamelCase					,						__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
           __lowerCAmelCase								=					rust_tokenizer.encode(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
           self.assertListEqual(__UpperCamelCase					,						__UpperCamelCase	)

           __lowerCAmelCase								=					self.get_rust_tokenizer()
           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase	)
           __lowerCAmelCase								=					rust_tokenizer.encode(__UpperCamelCase	)
           self.assertListEqual(__UpperCamelCase					,						__UpperCamelCase	)














from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase			:					Any         =	logging.get_logger(__name__)

lowerCamelCase			:					Dict         =	{
    '''microsoft/markuplm-base''': '''https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json''',
    '''microsoft/markuplm-large''': '''https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json''',
}



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""markuplm"""


      def __init__(	self					,						__UpperCamelCase=3_0_5_2_2					,						__UpperCamelCase=7_6_8					,						__UpperCamelCase=1_2					,						__UpperCamelCase=1_2					,						__UpperCamelCase=3_0_7_2					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=0.1					,						__UpperCamelCase=0.1					,						__UpperCamelCase=5_1_2					,						__UpperCamelCase=2					,						__UpperCamelCase=0.0_2					,						__UpperCamelCase=1e-12					,						__UpperCamelCase=0					,						__UpperCamelCase=0					,						__UpperCamelCase=2					,						__UpperCamelCase=2_5_6					,						__UpperCamelCase=1_0_2_4					,						__UpperCamelCase=2_1_6					,						__UpperCamelCase=1_0_0_1					,						__UpperCamelCase=3_2					,						__UpperCamelCase=5_0					,						__UpperCamelCase="absolute"					,						__UpperCamelCase=True					,						__UpperCamelCase=None					,						**__UpperCamelCase					,						)->	Dict:
           super().__init__(
               pad_token_id=__UpperCamelCase					,						bos_token_id=__UpperCamelCase					,						eos_token_id=__UpperCamelCase					,						**__UpperCamelCase					,						)
           __lowerCAmelCase								=					vocab_size
           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					hidden_dropout_prob
           __lowerCAmelCase								=					attention_probs_dropout_prob
           __lowerCAmelCase								=					max_position_embeddings
           __lowerCAmelCase								=					type_vocab_size
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					position_embedding_type
           __lowerCAmelCase								=					use_cache
           __lowerCAmelCase								=					classifier_dropout
           # additional properties
           __lowerCAmelCase								=					max_depth
           __lowerCAmelCase								=					max_xpath_tag_unit_embeddings
           __lowerCAmelCase								=					max_xpath_subs_unit_embeddings
           __lowerCAmelCase								=					tag_pad_id
           __lowerCAmelCase								=					subs_pad_id
           __lowerCAmelCase								=					xpath_unit_hidden_size






