






import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
 import tensorflow as tf
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops

 from transformers import GradientAccumulator, create_optimizer



@require_tf
class 							_UpperCamelCase			(unittest.TestCase			):


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase	)->	Dict:
           self.assertEqual(len(__UpperCamelCase	)					,						len(__UpperCamelCase	)	)
           for a, b in zip(__UpperCamelCase					,						__UpperCamelCase	):
                self.assertAlmostEqual(__UpperCamelCase					,						__UpperCamelCase					,						delta=__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	int:
           __lowerCAmelCase								=					GradientAccumulator()
           accumulator([tf.constant([1.0, 2.0]	)]	)
           accumulator([tf.constant([-2.0, 1.0]	)]	)
           accumulator([tf.constant([-1.0, 2.0]	)]	)
           with self.assertRaises(__UpperCamelCase	):
                accumulator([tf.constant([1.0, 1.0]	), tf.constant([2.0, 2.0]	)]	)
           self.assertEqual(accumulator.step					,						3	)
           self.assertEqual(len(accumulator.gradients	)					,						1	)
           self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()					,						[-2.0, 5.0]					,						tol=1e-2	)
           accumulator.reset()
           self.assertEqual(accumulator.step					,						0	)
           self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()					,						[0.0, 0.0]					,						tol=1e-2	)


      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           __lowerCAmelCase								=					None
           ops.enable_eager_execution_internal()
           __lowerCAmelCase								=					tf.config.list_physical_devices("CPU"	)
           if len(__UpperCamelCase	) == 1:
                tf.config.set_logical_device_configuration(
                    physical_devices[0]					,						[tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]	)
           __lowerCAmelCase								=					tf.config.list_logical_devices(device_type="CPU"	)
           __lowerCAmelCase								=					tf.distribute.MirroredStrategy(devices=devices[:2]	)

           with strategy.scope():
                __lowerCAmelCase								=					GradientAccumulator()
                __lowerCAmelCase								=					tf.Variable([4.0, 3.0]	)
                __lowerCAmelCase			, __lowerCAmelCase								=					create_optimizer(5e-5					,						1_0					,						5	)
                __lowerCAmelCase								=					tf.Variable([0.0, 0.0]					,						trainable=__UpperCamelCase	)

           def accumulate_on_replica(__UpperCamelCase	):
                accumulator([gradient]	)

           def apply_on_replica():
                optimizer.apply_gradients(list(zip(accumulator.gradients					,						[variable]	)	)	)

           @tf.function
           def accumulate(__UpperCamelCase					,						__UpperCamelCase	):
                with strategy.scope():
                     __lowerCAmelCase								=					strategy.experimental_local_results(__UpperCamelCase	)
                     local_variables[0].assign(__UpperCamelCase	)
                     local_variables[1].assign(__UpperCamelCase	)
                     strategy.run(__UpperCamelCase					,						args=(gradient_placeholder,)	)

           @tf.function
           def apply_grad():
                with strategy.scope():
                     strategy.run(__UpperCamelCase	)

           def _check_local_values(__UpperCamelCase					,						__UpperCamelCase	):
                __lowerCAmelCase								=					strategy.experimental_local_results(accumulator._gradients[0]	)
                self.assertListAlmostEqual(values[0].value()					,						__UpperCamelCase					,						tol=1e-2	)
                self.assertListAlmostEqual(values[1].value()					,						__UpperCamelCase					,						tol=1e-2	)

           accumulate([1.0, 2.0]					,						[-1.0, 1.0]	)
           accumulate([3.0, -1.0]					,						[-1.0, -1.0]	)
           accumulate([-2.0, 2.0]					,						[3.0, -2.0]	)
           self.assertEqual(accumulator.step					,						3	)
           _check_local_values([2.0, 3.0]					,						[1.0, -2.0]	)
           apply_grad()
           self.assertListAlmostEqual(variable.value()					,						[4.0, 3.0]					,						tol=1e-2	)
           accumulator.reset()
           self.assertEqual(accumulator.step					,						0	)
           _check_local_values([0.0, 0.0]					,						[0.0, 0.0]	)














from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase			:					Any         =	logging.get_logger(__name__)

lowerCamelCase			:					Dict         =	{
    '''microsoft/markuplm-base''': '''https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json''',
    '''microsoft/markuplm-large''': '''https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json''',
}



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""markuplm"""


      def __init__(	self					,						__UpperCamelCase=3_0_5_2_2					,						__UpperCamelCase=7_6_8					,						__UpperCamelCase=1_2					,						__UpperCamelCase=1_2					,						__UpperCamelCase=3_0_7_2					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=0.1					,						__UpperCamelCase=0.1					,						__UpperCamelCase=5_1_2					,						__UpperCamelCase=2					,						__UpperCamelCase=0.0_2					,						__UpperCamelCase=1e-12					,						__UpperCamelCase=0					,						__UpperCamelCase=0					,						__UpperCamelCase=2					,						__UpperCamelCase=2_5_6					,						__UpperCamelCase=1_0_2_4					,						__UpperCamelCase=2_1_6					,						__UpperCamelCase=1_0_0_1					,						__UpperCamelCase=3_2					,						__UpperCamelCase=5_0					,						__UpperCamelCase="absolute"					,						__UpperCamelCase=True					,						__UpperCamelCase=None					,						**__UpperCamelCase					,						)->	Dict:
           super().__init__(
               pad_token_id=__UpperCamelCase					,						bos_token_id=__UpperCamelCase					,						eos_token_id=__UpperCamelCase					,						**__UpperCamelCase					,						)
           __lowerCAmelCase								=					vocab_size
           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					hidden_dropout_prob
           __lowerCAmelCase								=					attention_probs_dropout_prob
           __lowerCAmelCase								=					max_position_embeddings
           __lowerCAmelCase								=					type_vocab_size
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					position_embedding_type
           __lowerCAmelCase								=					use_cache
           __lowerCAmelCase								=					classifier_dropout
           # additional properties
           __lowerCAmelCase								=					max_depth
           __lowerCAmelCase								=					max_xpath_tag_unit_embeddings
           __lowerCAmelCase								=					max_xpath_subs_unit_embeddings
           __lowerCAmelCase								=					tag_pad_id
           __lowerCAmelCase								=					subs_pad_id
           __lowerCAmelCase								=					xpath_unit_hidden_size






