





"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase     :						Optional[int]						       =  logging.get_logger(__name__)

__UpperCamelCase     :						Dict						       =  {
    '''microsoft/markuplm-base''': '''https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json''',
    '''microsoft/markuplm-large''': '''https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json''',
}







class       a							(   a__  ):
 snake_case__             =       '''markuplm'''



 def __init__(     self				,       _snake_case=3_05_22				,       _snake_case=7_68				,       _snake_case=12				,       _snake_case=12				,       _snake_case=30_72				,       _snake_case="gelu"				,       _snake_case=0.1				,       _snake_case=0.1				,       _snake_case=5_12				,       _snake_case=2				,       _snake_case=0.02				,       _snake_case=1E-12				,       _snake_case=0				,       _snake_case=0				,       _snake_case=2				,       _snake_case=2_56				,       _snake_case=10_24				,       _snake_case=2_16				,       _snake_case=10_01				,       _snake_case=32				,       _snake_case=50				,       _snake_case="absolute"				,       _snake_case=True				,       _snake_case=None				,       **_snake_case				,       ):







  """simple docstring"""






  super().__init__(
      pad_token_id=_snake_case				,       bos_token_id=_snake_case				,       eos_token_id=_snake_case				,       **_snake_case				,       )
  lowerCAmelCase    				= vocab_size
  lowerCAmelCase    				= hidden_size
  lowerCAmelCase    				= num_hidden_layers
  lowerCAmelCase    				= num_attention_heads
  lowerCAmelCase    				= hidden_act
  lowerCAmelCase    				= intermediate_size
  lowerCAmelCase    				= hidden_dropout_prob
  lowerCAmelCase    				= attention_probs_dropout_prob
  lowerCAmelCase    				= max_position_embeddings
  lowerCAmelCase    				= type_vocab_size
  lowerCAmelCase    				= initializer_range
  lowerCAmelCase    				= layer_norm_eps
  lowerCAmelCase    				= position_embedding_type
  lowerCAmelCase    				= use_cache
  lowerCAmelCase    				= classifier_dropout
  # additional properties
  lowerCAmelCase    				= max_depth
  lowerCAmelCase    				= max_xpath_tag_unit_embeddings
  lowerCAmelCase    				= max_xpath_subs_unit_embeddings
  lowerCAmelCase    				= tag_pad_id
  lowerCAmelCase    				= subs_pad_id
  lowerCAmelCase    				= xpath_unit_hidden_size












"""simple docstring"""
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class       a							:



 def __init__(     self				,       _snake_case				,       _snake_case=13				,       _snake_case=7				,       _snake_case=True				,       _snake_case=True				,       _snake_case=False				,       _snake_case=True				,       _snake_case=99				,       _snake_case=32				,       _snake_case=5				,       _snake_case=4				,       _snake_case=37				,       _snake_case="gelu"				,       _snake_case=0.1				,       _snake_case=0.1				,       _snake_case=5_12				,       _snake_case=16				,       _snake_case=2				,       _snake_case=0.02				,       _snake_case=3				,       _snake_case=4				,       _snake_case=None				,       ):







  """simple docstring"""






  lowerCAmelCase    				= parent
  lowerCAmelCase    				= batch_size
  lowerCAmelCase    				= seq_length
  lowerCAmelCase    				= is_training
  lowerCAmelCase    				= use_input_mask
  lowerCAmelCase    				= use_token_type_ids
  lowerCAmelCase    				= use_labels
  lowerCAmelCase    				= vocab_size
  lowerCAmelCase    				= hidden_size
  lowerCAmelCase    				= num_hidden_layers
  lowerCAmelCase    				= num_attention_heads
  lowerCAmelCase    				= intermediate_size
  lowerCAmelCase    				= hidden_act
  lowerCAmelCase    				= hidden_dropout_prob
  lowerCAmelCase    				= attention_probs_dropout_prob
  lowerCAmelCase    				= max_position_embeddings
  lowerCAmelCase    				= type_vocab_size
  lowerCAmelCase    				= type_sequence_label_size
  lowerCAmelCase    				= initializer_range
  lowerCAmelCase    				= num_labels
  lowerCAmelCase    				= num_choices
  lowerCAmelCase    				= scope



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= ids_tensor([self.batch_size, self.seq_length]				,       self.vocab_size     )

  lowerCAmelCase    				= None
  if self.use_input_mask:
   lowerCAmelCase    				= random_attention_mask([self.batch_size, self.seq_length]     )

  lowerCAmelCase    				= None
  if self.use_token_type_ids:
   lowerCAmelCase    				= ids_tensor([self.batch_size, self.seq_length]				,       self.type_vocab_size     )

  lowerCAmelCase    				= None
  lowerCAmelCase    				= None
  lowerCAmelCase    				= None
  if self.use_labels:
   lowerCAmelCase    				= ids_tensor([self.batch_size]				,       self.type_sequence_label_size     )
   lowerCAmelCase    				= ids_tensor([self.batch_size, self.seq_length]				,       self.num_labels     )
   lowerCAmelCase    				= ids_tensor([self.batch_size]				,       self.num_choices     )

  lowerCAmelCase    				= self.get_config()

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return OpenLlamaConfig(
      vocab_size=self.vocab_size				,       hidden_size=self.hidden_size				,       num_hidden_layers=self.num_hidden_layers				,       num_attention_heads=self.num_attention_heads				,       intermediate_size=self.intermediate_size				,       hidden_act=self.hidden_act				,       hidden_dropout_prob=self.hidden_dropout_prob				,       attention_probs_dropout_prob=self.attention_probs_dropout_prob				,       max_position_embeddings=self.max_position_embeddings				,       type_vocab_size=self.type_vocab_size				,       is_decoder=_snake_case				,       initializer_range=self.initializer_range				,       use_stable_embedding=_snake_case				,       )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= OpenLlamaModel(config=_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case     )
  lowerCAmelCase    				= model(_snake_case     )
  self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.hidden_size)     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       ):







  """simple docstring"""






  lowerCAmelCase    				= True
  lowerCAmelCase    				= OpenLlamaModel(_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(
      _snake_case				,       attention_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       encoder_attention_mask=_snake_case				,       )
  lowerCAmelCase    				= model(
      _snake_case				,       attention_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       )
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case     )
  self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.hidden_size)     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       ):







  """simple docstring"""






  lowerCAmelCase    				= OpenLlamaForCausalLM(config=_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case				,       labels=_snake_case     )
  self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.vocab_size)     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       _snake_case				,       ):







  """simple docstring"""






  lowerCAmelCase    				= True
  lowerCAmelCase    				= True
  lowerCAmelCase    				= OpenLlamaForCausalLM(config=_snake_case     )
  model.to(_snake_case     )
  model.eval()

  # first forward pass
  lowerCAmelCase    				= model(
      _snake_case				,       attention_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       encoder_attention_mask=_snake_case				,       use_cache=_snake_case				,       )
  lowerCAmelCase    				= outputs.past_key_values

  # create hypothetical multiple next token and extent to next_input_ids
  lowerCAmelCase    				= ids_tensor((self.batch_size, 3)				,       config.vocab_size     )
  lowerCAmelCase    				= ids_tensor((self.batch_size, 3)				,       vocab_size=2     )

  # append to next input_ids and
  lowerCAmelCase    				= torch.cat([input_ids, next_tokens]				,       dim=-1     )
  lowerCAmelCase    				= torch.cat([input_mask, next_mask]				,       dim=-1     )

  lowerCAmelCase    				= model(
      _snake_case				,       attention_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       encoder_attention_mask=_snake_case				,       output_hidden_states=_snake_case				,       )['hidden_states'][0]
  lowerCAmelCase    				= model(
      _snake_case				,       attention_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       encoder_attention_mask=_snake_case				,       past_key_values=_snake_case				,       output_hidden_states=_snake_case				,       )['hidden_states'][0]

  # select random slice
  lowerCAmelCase    				= ids_tensor((1,)				,       output_from_past.shape[-1]     ).item()
  lowerCAmelCase    				= output_from_no_past[:, -3:, random_slice_idx].detach()
  lowerCAmelCase    				= output_from_past[:, :, random_slice_idx].detach()

  self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

  # test that outputs are equal for slice
  self.parent.assertTrue(torch.allclose(_snake_case				,       _snake_case				,       atol=1E-3     )     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.prepare_config_and_inputs()
  (
      (
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,(
      lowerCAmelCase
  )				,
  )    				= config_and_inputs
  lowerCAmelCase    				= {'input_ids': input_ids, 'attention_mask': input_mask}
  return config, inputs_dict







@require_torch
class       a							(   a__   , a__   , a__   , unittest.TestCase  ):
 snake_case__             =       (
     (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
 )
 snake_case__             =       (OpenLlamaForCausalLM,) if is_torch_available() else ()
 snake_case__             =       (
     {
         '''feature-extraction''': OpenLlamaModel,
         '''text-classification''': OpenLlamaForSequenceClassification,
         '''text-generation''': OpenLlamaForCausalLM,
         '''zero-shot''': OpenLlamaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 snake_case__             =       False
 snake_case__             =       False



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= OpenLlamaModelTester(self     )
  lowerCAmelCase    				= ConfigTester(self				,       config_class=_snake_case				,       hidden_size=37     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  self.config_tester.run_common_tests()



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*_snake_case     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs()
  for type in ["absolute", "relative_key", "relative_key_query"]:
   lowerCAmelCase    				= type
   self.model_tester.create_and_check_model(*_snake_case     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase				,lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs_for_common()
  lowerCAmelCase    				= 3
  lowerCAmelCase    				= input_dict['input_ids']
  lowerCAmelCase    				= input_ids.ne(1     ).to(_snake_case     )
  lowerCAmelCase    				= ids_tensor([self.model_tester.batch_size]				,       self.model_tester.type_sequence_label_size     )
  lowerCAmelCase    				= OpenLlamaForSequenceClassification(_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case				,       labels=_snake_case     )
  self.assertEqual(result.logits.shape				,       (self.model_tester.batch_size, self.model_tester.num_labels)     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase				,lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs_for_common()
  lowerCAmelCase    				= 3
  lowerCAmelCase    				= 'single_label_classification'
  lowerCAmelCase    				= input_dict['input_ids']
  lowerCAmelCase    				= input_ids.ne(1     ).to(_snake_case     )
  lowerCAmelCase    				= ids_tensor([self.model_tester.batch_size]				,       self.model_tester.type_sequence_label_size     )
  lowerCAmelCase    				= OpenLlamaForSequenceClassification(_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case				,       labels=_snake_case     )
  self.assertEqual(result.logits.shape				,       (self.model_tester.batch_size, self.model_tester.num_labels)     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase				,lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs_for_common()
  lowerCAmelCase    				= 3
  lowerCAmelCase    				= 'multi_label_classification'
  lowerCAmelCase    				= input_dict['input_ids']
  lowerCAmelCase    				= input_ids.ne(1     ).to(_snake_case     )
  lowerCAmelCase    				= ids_tensor(
      [self.model_tester.batch_size, config.num_labels]				,       self.model_tester.type_sequence_label_size     ).to(torch.float     )
  lowerCAmelCase    				= OpenLlamaForSequenceClassification(_snake_case     )
  model.to(_snake_case     )
  model.eval()
  lowerCAmelCase    				= model(_snake_case				,       attention_mask=_snake_case				,       labels=_snake_case     )
  self.assertEqual(result.logits.shape				,       (self.model_tester.batch_size, self.model_tester.num_labels)     )



 @unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'     )
 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  pass



 @parameterized.expand([('linear',), ('dynamic',)]     )
 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase				,lowerCAmelCase    				= self.model_tester.prepare_config_and_inputs_for_common()
  lowerCAmelCase    				= ids_tensor([1, 10]				,       config.vocab_size     )
  lowerCAmelCase    				= ids_tensor([1, int(config.max_position_embeddings * 1.5     )]				,       config.vocab_size     )

  set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
  lowerCAmelCase    				= OpenLlamaModel(_snake_case     )
  original_model.to(_snake_case     )
  original_model.eval()
  lowerCAmelCase    				= original_model(_snake_case     ).last_hidden_state
  lowerCAmelCase    				= original_model(_snake_case     ).last_hidden_state

  set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
  lowerCAmelCase    				= {'type': scaling_type, 'factor': 10.0}
  lowerCAmelCase    				= OpenLlamaModel(_snake_case     )
  scaled_model.to(_snake_case     )
  scaled_model.eval()
  lowerCAmelCase    				= scaled_model(_snake_case     ).last_hidden_state
  lowerCAmelCase    				= scaled_model(_snake_case     ).last_hidden_state

  # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
  # maximum sequence length, so the outputs for the short input should match.
  if scaling_type == "dynamic":
   self.assertTrue(torch.allclose(_snake_case				,       _snake_case				,       atol=1E-5     )     )
  else:
   self.assertFalse(torch.allclose(_snake_case				,       _snake_case				,       atol=1E-5     )     )

  # The output should be different for long inputs
  self.assertFalse(torch.allclose(_snake_case				,       _snake_case				,       atol=1E-5     )     )





