


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class       __magic_name__	(     __lowerCAmelCase					,				unittest.TestCase):
     A:		List[str]			=      BertTokenizer
     A:		Any			=      BertTokenizerFast
     A:		List[Any]			=      True
     A:		str			=      True
     A:		List[str]			=      filter_non_english


     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      str:





      '''simple docstring'''






      super().setUp()

      UpperCamelCase__					:    int          =   [
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      UpperCamelCase__					:    Tuple          =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES['''vocab_file''']			)
      with open(self.vocab_file							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)


     def      UpperCAmelCase__	(      self							:     int							,	lowerCamelCase__							:     Optional[int]			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   '''UNwant\u00E9d,running'''
      UpperCamelCase__					:    Dict          =   '''unwanted, running'''
      return input_text, output_text


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   self.tokenizer_class(self.vocab_file			)

      UpperCamelCase__					:    Optional[Any]          =   tokenizer.tokenize('''UNwant\u00E9d,running'''			)
      self.assertListEqual(lowerCamelCase__							,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__			)							,	[9, 6, 7, 12, 10, 11]			)


     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      Union[str, Any]:





      '''simple docstring'''






      if not self.test_rust_tokenizer:
       return

      UpperCamelCase__					:    str          =   self.get_tokenizer()
      UpperCamelCase__					:    Dict          =   self.get_rust_tokenizer()

      UpperCamelCase__					:    List[str]          =   '''UNwant\u00E9d,running'''

      UpperCamelCase__					:    Optional[Any]          =   tokenizer.tokenize(lowerCamelCase__			)
      UpperCamelCase__					:    Any          =   rust_tokenizer.tokenize(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      UpperCamelCase__					:    Any          =   tokenizer.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
      UpperCamelCase__					:    Dict          =   rust_tokenizer.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      UpperCamelCase__					:    Optional[int]          =   self.get_rust_tokenizer()
      UpperCamelCase__					:    List[Any]          =   tokenizer.encode(lowerCamelCase__			)
      UpperCamelCase__					:    List[str]          =   rust_tokenizer.encode(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      # With lower casing
      UpperCamelCase__					:    Tuple          =   self.get_tokenizer(do_lower_case=lowerCamelCase__			)
      UpperCamelCase__					:    str          =   self.get_rust_tokenizer(do_lower_case=lowerCamelCase__			)

      UpperCamelCase__					:    List[Any]          =   '''UNwant\u00E9d,running'''

      UpperCamelCase__					:    Union[str, Any]          =   tokenizer.tokenize(lowerCamelCase__			)
      UpperCamelCase__					:    Dict          =   rust_tokenizer.tokenize(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      UpperCamelCase__					:    List[Any]          =   tokenizer.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
      UpperCamelCase__					:    Any          =   rust_tokenizer.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      UpperCamelCase__					:    Dict          =   self.get_rust_tokenizer()
      UpperCamelCase__					:    str          =   tokenizer.encode(lowerCamelCase__			)
      UpperCamelCase__					:    List[Any]          =   rust_tokenizer.encode(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Tuple          =   BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)							,	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   BasicTokenizer(do_lower_case=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)							,	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)							,	['''hello''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   BasicTokenizer(do_lower_case=lowerCamelCase__							,	strip_accents=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)							,	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)							,	['''h\u00E9llo''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   BasicTokenizer(do_lower_case=lowerCamelCase__							,	strip_accents=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)							,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)							,	['''hello''']			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   BasicTokenizer(do_lower_case=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)							,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)							,	['''hello''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   BasicTokenizer(do_lower_case=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)							,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Tuple:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   BasicTokenizer(do_lower_case=lowerCamelCase__							,	strip_accents=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)							,	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   BasicTokenizer(do_lower_case=lowerCamelCase__							,	strip_accents=lowerCamelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)							,	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   BasicTokenizer(do_lower_case=lowerCamelCase__							,	never_split=['''[UNK]''']			)

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)							,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)


     def      UpperCAmelCase__	(      self							:     List[Any]			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   BasicTokenizer()
      UpperCamelCase__					:    List[str]          =   '''a\n\'ll !!to?\'d of, can\'t.'''
      UpperCamelCase__					:    Optional[Any]          =   ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
      self.assertListEqual(tokenizer.tokenize(lowerCamelCase__			)							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      UpperCamelCase__					:    List[Any]          =   {}
      for i, token in enumerate(lowerCamelCase__			):
       UpperCamelCase__					:    str          =   i
      UpperCamelCase__					:    Union[str, Any]          =   WordpieceTokenizer(vocab=lowerCamelCase__							,	unk_token='''[UNK]'''			)

      self.assertListEqual(tokenizer.tokenize(''''''			)							,	[]			)

      self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)							,	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

      self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)							,	['''[UNK]''', '''runn''', '''##ing''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      int:





      '''simple docstring'''






      self.assertTrue(_is_whitespace(''' '''			)			)
      self.assertTrue(_is_whitespace('''\t'''			)			)
      self.assertTrue(_is_whitespace('''\r'''			)			)
      self.assertTrue(_is_whitespace('''\n'''			)			)
      self.assertTrue(_is_whitespace('''\u00A0'''			)			)

      self.assertFalse(_is_whitespace('''A'''			)			)
      self.assertFalse(_is_whitespace('''-'''			)			)


     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      List[str]:





      '''simple docstring'''






      self.assertTrue(_is_control('''\u0005'''			)			)

      self.assertFalse(_is_control('''A'''			)			)
      self.assertFalse(_is_control(''' '''			)			)
      self.assertFalse(_is_control('''\t'''			)			)
      self.assertFalse(_is_control('''\r'''			)			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      int:





      '''simple docstring'''






      self.assertTrue(_is_punctuation('''-'''			)			)
      self.assertTrue(_is_punctuation('''$'''			)			)
      self.assertTrue(_is_punctuation('''`'''			)			)
      self.assertTrue(_is_punctuation('''.'''			)			)

      self.assertFalse(_is_punctuation('''A'''			)			)
      self.assertFalse(_is_punctuation(''' '''			)			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[Any]          =   self.get_tokenizer()
      UpperCamelCase__					:    Optional[Any]          =   self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(lowerCamelCase__			) for t in ['''Test''', '''\xad''', '''test''']]							,	[['''[UNK]'''], [], ['''[UNK]''']]			)

      self.assertListEqual(
          [rust_tokenizer.tokenize(lowerCamelCase__			) for t in ['''Test''', '''\xad''', '''test''']]							,	[['''[UNK]'''], [], ['''[UNK]''']]			)


     @slow
     def      UpperCAmelCase__	(      self							:     List[Any]			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   self.tokenizer_class.from_pretrained('''bert-base-uncased'''			)

      UpperCamelCase__					:    List[str]          =   tokenizer.encode('''sequence builders'''							,	add_special_tokens=lowerCamelCase__			)
      UpperCamelCase__					:    str          =   tokenizer.encode('''multi-sequence build'''							,	add_special_tokens=lowerCamelCase__			)

      UpperCamelCase__					:    Union[str, Any]          =   tokenizer.build_inputs_with_special_tokens(lowerCamelCase__			)
      UpperCamelCase__					:    Optional[Any]          =   tokenizer.build_inputs_with_special_tokens(lowerCamelCase__							,	lowerCamelCase__			)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]


     def      UpperCAmelCase__	(      self							:     int			)  ->      Union[str, Any]:





      '''simple docstring'''






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"			):
        UpperCamelCase__					:    int          =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__							,	**lowerCamelCase__			)

        UpperCamelCase__					:    Dict          =   F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
        UpperCamelCase__					:    Union[str, Any]          =   tokenizer_r.encode_plus(
            lowerCamelCase__							,	return_attention_mask=lowerCamelCase__							,	return_token_type_ids=lowerCamelCase__							,	return_offsets_mapping=lowerCamelCase__							,	add_special_tokens=lowerCamelCase__							,	)

        UpperCamelCase__					:    List[Any]          =   tokenizer_r.do_lower_case if hasattr(lowerCamelCase__							,	'''do_lower_case'''			) else False
        UpperCamelCase__					:    Optional[int]          =   (
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''A'''),
                ((1, 2), ''','''),
                ((3, 5), '''na'''),
                ((5, 6), '''##ï'''),
                ((6, 8), '''##ve'''),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), '''Allen'''),
                ((21, 23), '''##NL'''),
                ((23, 24), '''##P'''),
                ((25, 33), '''sentence'''),
                ((33, 34), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''a'''),
                ((1, 2), ''','''),
                ((3, 8), '''naive'''),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), '''allen'''),
                ((21, 23), '''##nl'''),
                ((23, 24), '''##p'''),
                ((25, 33), '''sentence'''),
                ((33, 34), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )

        self.assertEqual(
            [e[1] for e in expected_results]							,	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
        self.assertEqual([e[0] for e in expected_results]							,	tokens['''offset_mapping''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   ['''的''', '''人''', '''有''']
      UpperCamelCase__					:    Optional[int]          =   ''''''.join(lowerCamelCase__			)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"			):
        UpperCamelCase__					:    Dict          =   True
        UpperCamelCase__					:    Any          =   self.tokenizer_class.from_pretrained(lowerCamelCase__							,	**lowerCamelCase__			)
        UpperCamelCase__					:    Optional[int]          =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__							,	**lowerCamelCase__			)

        UpperCamelCase__					:    Optional[int]          =   tokenizer_p.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
        UpperCamelCase__					:    List[str]          =   tokenizer_r.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)

        UpperCamelCase__					:    str          =   tokenizer_r.convert_ids_to_tokens(lowerCamelCase__			)
        UpperCamelCase__					:    str          =   tokenizer_p.convert_ids_to_tokens(lowerCamelCase__			)

        # it is expected that each Chinese character is not preceded by "##"
        self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)
        self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

        UpperCamelCase__					:    List[Any]          =   False
        UpperCamelCase__					:    Dict          =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__							,	**lowerCamelCase__			)
        UpperCamelCase__					:    Optional[Any]          =   self.tokenizer_class.from_pretrained(lowerCamelCase__							,	**lowerCamelCase__			)

        UpperCamelCase__					:    Dict          =   tokenizer_r.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
        UpperCamelCase__					:    Optional[Any]          =   tokenizer_p.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)

        UpperCamelCase__					:    List[str]          =   tokenizer_r.convert_ids_to_tokens(lowerCamelCase__			)
        UpperCamelCase__					:    Tuple          =   tokenizer_p.convert_ids_to_tokens(lowerCamelCase__			)

        # it is expected that only the first Chinese character is not preceded by "##".
        UpperCamelCase__					:    Union[str, Any]          =   [
            F"##{token}" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__			)
        ]
        self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)
        self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase       :       Optional[int]						       =				logging.get_logger(__name__)

__UpperCamelCase       :       Optional[int]						       =				{
    "microsoft/markuplm-base": "https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json",
    "microsoft/markuplm-large": "https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json",
}

class       __magic_name__	(     __lowerCAmelCase):
     A:		Optional[Any]			=      "markuplm"


     def __init__(      self							:     List[Any]							,	lowerCamelCase__							:     Optional[int]=30522							,	lowerCamelCase__							:     Dict=768							,	lowerCamelCase__							:     List[str]=12							,	lowerCamelCase__							:     List[Any]=12							,	lowerCamelCase__							:     Optional[Any]=3072							,	lowerCamelCase__							:     int="gelu"							,	lowerCamelCase__							:     Union[str, Any]=0.1							,	lowerCamelCase__							:     Optional[int]=0.1							,	lowerCamelCase__							:     Tuple=512							,	lowerCamelCase__							:     List[str]=2							,	lowerCamelCase__							:     Union[str, Any]=0.02							,	lowerCamelCase__							:     Optional[Any]=1E-1_2							,	lowerCamelCase__							:     List[str]=0							,	lowerCamelCase__							:     Tuple=0							,	lowerCamelCase__							:     List[str]=2							,	lowerCamelCase__							:     List[Any]=256							,	lowerCamelCase__							:     List[str]=1024							,	lowerCamelCase__							:     Optional[int]=216							,	lowerCamelCase__							:     Any=1001							,	lowerCamelCase__							:     int=32							,	lowerCamelCase__							:     Tuple=50							,	lowerCamelCase__							:     str="absolute"							,	lowerCamelCase__							:     List[str]=True							,	lowerCamelCase__							:     List[Any]=None							,	**lowerCamelCase__							:     List[str]							,	)  ->      Union[str, Any]:





      '''simple docstring'''






      super().__init__(
          pad_token_id=lowerCamelCase__							,	bos_token_id=lowerCamelCase__							,	eos_token_id=lowerCamelCase__							,	**lowerCamelCase__							,	)
      UpperCamelCase__					:    Optional[int]          =   vocab_size
      UpperCamelCase__					:    List[Any]          =   hidden_size
      UpperCamelCase__					:    Any          =   num_hidden_layers
      UpperCamelCase__					:    Tuple          =   num_attention_heads
      UpperCamelCase__					:    Any          =   hidden_act
      UpperCamelCase__					:    Dict          =   intermediate_size
      UpperCamelCase__					:    Optional[Any]          =   hidden_dropout_prob
      UpperCamelCase__					:    Optional[int]          =   attention_probs_dropout_prob
      UpperCamelCase__					:    Tuple          =   max_position_embeddings
      UpperCamelCase__					:    List[str]          =   type_vocab_size
      UpperCamelCase__					:    Any          =   initializer_range
      UpperCamelCase__					:    Dict          =   layer_norm_eps
      UpperCamelCase__					:    Any          =   position_embedding_type
      UpperCamelCase__					:    Optional[Any]          =   use_cache
      UpperCamelCase__					:    int          =   classifier_dropout
      # additional properties
      UpperCamelCase__					:    int          =   max_depth
      UpperCamelCase__					:    Optional[Any]          =   max_xpath_tag_unit_embeddings
      UpperCamelCase__					:    List[Any]          =   max_xpath_subs_unit_embeddings
      UpperCamelCase__					:    List[Any]          =   tag_pad_id
      UpperCamelCase__					:    Optional[int]          =   subs_pad_id
      UpperCamelCase__					:    Optional[Any]          =   xpath_unit_hidden_size


