'''simple docstring'''




import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MarkupLMForQuestionAnswering,
      MarkupLMForSequenceClassification,
      MarkupLMForTokenClassification,
      MarkupLMModel,
  )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer




class 	__UpperCAmelCase				:





       '''simple docstring'''



       def __init__(      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE=13    ,     _SCREAMING_SNAKE_CASE=7    ,     _SCREAMING_SNAKE_CASE=True    ,     _SCREAMING_SNAKE_CASE=True    ,     _SCREAMING_SNAKE_CASE=True    ,     _SCREAMING_SNAKE_CASE=True    ,     _SCREAMING_SNAKE_CASE=99    ,     _SCREAMING_SNAKE_CASE=32    ,     _SCREAMING_SNAKE_CASE=5    ,     _SCREAMING_SNAKE_CASE=4    ,     _SCREAMING_SNAKE_CASE=37    ,     _SCREAMING_SNAKE_CASE="gelu"    ,     _SCREAMING_SNAKE_CASE=0.1    ,     _SCREAMING_SNAKE_CASE=0.1    ,     _SCREAMING_SNAKE_CASE=512    ,     _SCREAMING_SNAKE_CASE=16    ,     _SCREAMING_SNAKE_CASE=2    ,     _SCREAMING_SNAKE_CASE=0.02    ,     _SCREAMING_SNAKE_CASE=3    ,     _SCREAMING_SNAKE_CASE=None    ,     _SCREAMING_SNAKE_CASE=20    ,     _SCREAMING_SNAKE_CASE=30    ,     _SCREAMING_SNAKE_CASE=2    ,     _SCREAMING_SNAKE_CASE=2    ,     _SCREAMING_SNAKE_CASE=10    ,     )		->     str:
           A_     =   parent
           A_     =   batch_size
           A_     =   seq_length
           A_     =   is_training
           A_     =   use_input_mask
           A_     =   use_token_type_ids
           A_     =   use_labels
           A_     =   vocab_size
           A_     =   hidden_size
           A_     =   num_hidden_layers
           A_     =   num_attention_heads
           A_     =   intermediate_size
           A_     =   hidden_act
           A_     =   hidden_dropout_prob
           A_     =   attention_probs_dropout_prob
           A_     =   max_position_embeddings
           A_     =   type_vocab_size
           A_     =   type_sequence_label_size
           A_     =   initializer_range
           A_     =   num_labels
           A_     =   scope
           A_     =   max_xpath_tag_unit_embeddings
           A_     =   max_xpath_subs_unit_embeddings
           A_     =   tag_pad_id
           A_     =   subs_pad_id
           A_     =   max_depth

       def 					__A       (      self )		->     str:
           A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.vocab_size )

           A_     =   ids_tensor(
               [self.batch_size, self.seq_length, self.max_depth]    ,     self.max_xpath_tag_unit_embeddings )

           A_     =   ids_tensor(
               [self.batch_size, self.seq_length, self.max_depth]    ,     self.max_xpath_subs_unit_embeddings )

           A_     =   None
           if self.use_input_mask:
               A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     vocab_size=2 )

           A_     =   None
           if self.use_token_type_ids:
               A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.type_vocab_size )

           A_     =   None
           A_     =   None
           if self.use_labels:
               A_     =   ids_tensor([self.batch_size]    ,     self.type_sequence_label_size )
               A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.num_labels )

           A_     =   self.get_config()

           return (
               config,
               input_ids,
               xpath_tags_seq,
               xpath_subs_seq,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
           )

       def 					__A       (      self )		->     Optional[int]:
           return MarkupLMConfig(
               vocab_size=self.vocab_size    ,     hidden_size=self.hidden_size    ,     num_hidden_layers=self.num_hidden_layers    ,     num_attention_heads=self.num_attention_heads    ,     intermediate_size=self.intermediate_size    ,     hidden_act=self.hidden_act    ,     hidden_dropout_prob=self.hidden_dropout_prob    ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,     max_position_embeddings=self.max_position_embeddings    ,     type_vocab_size=self.type_vocab_size    ,     initializer_range=self.initializer_range    ,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings    ,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings    ,     tag_pad_id=self.tag_pad_id    ,     subs_pad_id=self.subs_pad_id    ,     max_depth=self.max_depth    ,     )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     int:
           A_     =   MarkupLMModel(config=_SCREAMING_SNAKE_CASE )
           model.to(_SCREAMING_SNAKE_CASE )
           model.eval()
           print('''Configs:'''    ,     model.config.tag_pad_id    ,     model.config.subs_pad_id )
           A_     =   model(_SCREAMING_SNAKE_CASE    ,     attention_mask=_SCREAMING_SNAKE_CASE    ,     token_type_ids=_SCREAMING_SNAKE_CASE )
           A_     =   model(_SCREAMING_SNAKE_CASE    ,     token_type_ids=_SCREAMING_SNAKE_CASE )
           A_     =   model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.hidden_size) )
           self.parent.assertEqual(result.pooler_output.shape    ,     (self.batch_size, self.hidden_size) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     Tuple:
           A_     =   self.num_labels
           A_     =   MarkupLMForSequenceClassification(_SCREAMING_SNAKE_CASE )
           model.to(_SCREAMING_SNAKE_CASE )
           model.eval()
           A_     =   model(
               _SCREAMING_SNAKE_CASE    ,     xpath_tags_seq=_SCREAMING_SNAKE_CASE    ,     xpath_subs_seq=_SCREAMING_SNAKE_CASE    ,     attention_mask=_SCREAMING_SNAKE_CASE    ,     token_type_ids=_SCREAMING_SNAKE_CASE    ,     labels=_SCREAMING_SNAKE_CASE    ,     )
           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_labels) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     str:
           A_     =   self.num_labels
           A_     =   MarkupLMForTokenClassification(config=_SCREAMING_SNAKE_CASE )
           model.to(_SCREAMING_SNAKE_CASE )
           model.eval()
           A_     =   model(
               _SCREAMING_SNAKE_CASE    ,     xpath_tags_seq=_SCREAMING_SNAKE_CASE    ,     xpath_subs_seq=_SCREAMING_SNAKE_CASE    ,     attention_mask=_SCREAMING_SNAKE_CASE    ,     token_type_ids=_SCREAMING_SNAKE_CASE    ,     labels=_SCREAMING_SNAKE_CASE    ,     )
           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.num_labels) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     Tuple:
           A_     =   MarkupLMForQuestionAnswering(config=_SCREAMING_SNAKE_CASE )
           model.to(_SCREAMING_SNAKE_CASE )
           model.eval()
           A_     =   model(
               _SCREAMING_SNAKE_CASE    ,     xpath_tags_seq=_SCREAMING_SNAKE_CASE    ,     xpath_subs_seq=_SCREAMING_SNAKE_CASE    ,     attention_mask=_SCREAMING_SNAKE_CASE    ,     token_type_ids=_SCREAMING_SNAKE_CASE    ,     start_positions=_SCREAMING_SNAKE_CASE    ,     end_positions=_SCREAMING_SNAKE_CASE    ,     )
           self.parent.assertEqual(result.start_logits.shape    ,     (self.batch_size, self.seq_length) )
           self.parent.assertEqual(result.end_logits.shape    ,     (self.batch_size, self.seq_length) )

       def 					__A       (      self )		->     Optional[int]:
           A_     =   self.prepare_config_and_inputs()
           (
               (
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,
           )     =   config_and_inputs
           A_     =   {
               '''input_ids''': input_ids,
               '''xpath_tags_seq''': xpath_tags_seq,
               '''xpath_subs_seq''': xpath_subs_seq,
               '''token_type_ids''': token_type_ids,
               '''attention_mask''': input_mask,
           }
           return config, inputs_dict




@require_torch
class 	__UpperCAmelCase				(				_UpperCamelCase						,      _UpperCamelCase						,      unittest.TestCase     ):





       '''simple docstring'''



       __lowercase					:					Optional[Any]    =				(
           (
               MarkupLMModel,
               MarkupLMForSequenceClassification,
               MarkupLMForTokenClassification,
               MarkupLMForQuestionAnswering,
           )
           if is_torch_available()
           else None
       )
       __lowercase					:					List[Any]    =				(
           {
               'feature-extraction': MarkupLMModel,
               'question-answering': MarkupLMForQuestionAnswering,
               'text-classification': MarkupLMForSequenceClassification,
               'token-classification': MarkupLMForTokenClassification,
               'zero-shot': MarkupLMForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE )		->     List[str]:
           # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
           # (batch of pretokenized examples).
           return True

       def 					__A       (      self )		->     Optional[Any]:
           A_     =   MarkupLMModelTester(self )
           A_     =   ConfigTester(self    ,     config_class=_SCREAMING_SNAKE_CASE    ,     hidden_size=37 )

       def 					__A       (      self )		->     Dict:
           self.config_tester.run_common_tests()

       def 					__A       (      self )		->     Optional[int]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     List[Any]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     Union[str, Any]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     Tuple:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*_SCREAMING_SNAKE_CASE )





def      _UpperCAmelCase    (							)	->      List[str]:
    A_     =   '''
    <!DOCTYPE html>
    <html>
    <head>
    <title>Page Title</title>
    </head>
    <body>

    <h1>This is a Heading</h1>
    <p>This is a paragraph.</p>

    </body>
    </html>
    '''

    return html_string




@require_torch
class 	__UpperCAmelCase				(				unittest.TestCase     ):





       '''simple docstring'''



       @cached_property
       def 					__A       (      self )		->     Dict:
           # TODO use from_pretrained here
           A_     =   MarkupLMFeatureExtractor()
           A_     =   MarkupLMTokenizer.from_pretrained('''microsoft/markuplm-base''' )

           return MarkupLMProcessor(_SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE )

       @slow
       def 					__A       (      self )		->     Any:
           A_     =   MarkupLMModel.from_pretrained('''microsoft/markuplm-base''' ).to(_SCREAMING_SNAKE_CASE )

           A_     =   self.default_processor

           A_     =   processor(prepare_html_string()    ,     return_tensors='''pt''' )
           A_     =   inputs.to(_SCREAMING_SNAKE_CASE )

           # forward pass
           with torch.no_grad():
               A_     =   model(**_SCREAMING_SNAKE_CASE )

           # verify the last hidden states
           A_     =   torch.Size([1, 14, 768] )
           self.assertEqual(outputs.last_hidden_state.shape    ,     _SCREAMING_SNAKE_CASE )

           A_     =   torch.tensor(
               [[0.0_675, -0.0_052, 0.5_001], [-0.2_281, 0.0_802, 0.2_192], [-0.0_583, -0.3_311, 0.1_185]] ).to(_SCREAMING_SNAKE_CASE )

           self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]    ,     _SCREAMING_SNAKE_CASE    ,     atol=1E-4 ) )







'''simple docstring'''




import bisect
import copy
from collections import defaultdict

import numpy as np
from torch.utils.data import BatchSampler, Sampler

from utils import logger





def      _UpperCAmelCase    (							_UpperCamelCase    :	Optional[Any],   _UpperCamelCase    :	int   )	->      Union[str, Any]:
    A_     =   copy.deepcopy(_UpperCamelCase   )
    A_     =   sorted(_UpperCamelCase   )
    A_     =   [bisect.bisect_right(_UpperCamelCase,   _UpperCamelCase   ) for y in x]
    return quantized





def      _UpperCAmelCase    (							_UpperCamelCase    :	str,   _UpperCamelCase    :	Tuple=0   )	->      str:
    A_     =   np.arange(start=3,   stop=_UpperCamelCase,   step=4   ).tolist() if k > 0 else [10]
    A_     =   _quantize(_UpperCamelCase,   _UpperCamelCase   )
    # count number of elements per group
    A_     =   np.unique(_UpperCamelCase,   return_counts=_UpperCamelCase   )[1]
    A_     =   [0] + bins + [np.inf]
    logger.info('''Using {} as bins for aspect lengths quantization'''.format(_UpperCamelCase   )   )
    logger.info('''Count of instances per bin: {}'''.format(_UpperCamelCase   )   )
    return groups




class 	__UpperCAmelCase				(				_UpperCamelCase     ):





       '''simple docstring'''



       def __init__(      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE )		->     Dict:
           if not isinstance(_SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE ):
               raise ValueError(
                   '''sampler should be an instance of torch.utils.data.Sampler, but got sampler={}'''.format(_SCREAMING_SNAKE_CASE ) )
           A_     =   sampler
           A_     =   group_ids
           A_     =   batch_size

       def __iter__(      self )		->     Tuple:
           A_     =   defaultdict(_SCREAMING_SNAKE_CASE )
           A_     =   defaultdict(_SCREAMING_SNAKE_CASE )

           A_     =   0
           for idx in self.sampler:
               A_     =   self.group_ids[idx]
               buffer_per_group[group_id].append(_SCREAMING_SNAKE_CASE )
               samples_per_group[group_id].append(_SCREAMING_SNAKE_CASE )
               if len(buffer_per_group[group_id] ) == self.batch_size:
                   yield buffer_per_group[group_id]  # TODO
                   num_batches += 1
                   del buffer_per_group[group_id]
               assert len(buffer_per_group[group_id] ) < self.batch_size

           # now we have run out of elements that satisfy
           # the group criteria, let's return the remaining
           # elements so that the size of the sampler is
           # deterministic
           A_     =   len(self )
           A_     =   expected_num_batches - num_batches
           if num_remaining > 0:
               # for the remaining batches, group the batches by similar lengths
               A_     =   []
               for group_id, idxs in sorted(buffer_per_group.items()    ,     key=lambda _SCREAMING_SNAKE_CASE : x[0] ):
                   batch_idx.extend(_SCREAMING_SNAKE_CASE )
                   if len(_SCREAMING_SNAKE_CASE ) >= self.batch_size:
                       yield batch_idx[: self.batch_size]
                       A_     =   batch_idx[self.batch_size :]
                       num_remaining -= 1
               if len(_SCREAMING_SNAKE_CASE ) > 0:
                   yield batch_idx
                   num_remaining -= 1
           assert num_remaining == 0

       def __len__(      self )		->     Optional[int]:
           return (len(self.sampler ) + self.batch_size - 1) // self.batch_size






