




"""simple docstring"""







import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
						import tensorflow as tf
						from tensorflow.python.eager import context
						from tensorflow.python.framework import ops

						from transformers import GradientAccumulator, create_optimizer




@require_tf
class       lowerCAmelCase__       (			unittest.TestCase    ):


				"""simple docstring"""





				def       snake_case    (     self					:	List[str]    ,   lowercase__					:	List[Any]    ,   lowercase__					:	Dict    ,   lowercase__					:	Tuple						):
						self.assertEqual(len(lowercase__						)    ,   len(lowercase__						)						)
						for a, b in zip(lowercase__    ,   lowercase__						):
								self.assertAlmostEqual(lowercase__    ,   lowercase__    ,   delta=lowercase__						)



				def       snake_case    (     self					:	str						):
						__lowercase				:		Optional[int]												=				GradientAccumulator()
						accumulator([tf.constant([1.0, 2.0]						)]						)
						accumulator([tf.constant([-2.0, 1.0]						)]						)
						accumulator([tf.constant([-1.0, 2.0]						)]						)
						with self.assertRaises(lowercase__						):
								accumulator([tf.constant([1.0, 1.0]						), tf.constant([2.0, 2.0]						)]						)
						self.assertEqual(accumulator.step    ,   3						)
						self.assertEqual(len(accumulator.gradients						)    ,   1						)
						self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()    ,   [-2.0, 5.0]    ,   tol=1e-2						)
						accumulator.reset()
						self.assertEqual(accumulator.step    ,   0						)
						self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()    ,   [0.0, 0.0]    ,   tol=1e-2						)



				def       snake_case    (     self					:	Optional[int]						):
						__lowercase				:		Dict												=				None
						ops.enable_eager_execution_internal()
						__lowercase				:		Optional[Any]												=				tf.config.list_physical_devices("CPU"						)
						if len(lowercase__						) == 1:
								tf.config.set_logical_device_configuration(
								    physical_devices[0]    ,   [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]						)
						__lowercase				:		Dict												=				tf.config.list_logical_devices(device_type="CPU"						)
						__lowercase				:		Dict												=				tf.distribute.MirroredStrategy(devices=devices[:2]						)

						with strategy.scope():
								__lowercase				:		Optional[int]												=				GradientAccumulator()
								__lowercase				:		Tuple												=				tf.Variable([4.0, 3.0]						)
								__lowercase						,__lowercase				:		Any												=				create_optimizer(5e-5    ,   1_0    ,   5						)
								__lowercase				:		int												=				tf.Variable([0.0, 0.0]    ,   trainable=lowercase__						)

						def accumulate_on_replica(lowercase__					:	int						):
								accumulator([gradient]						)

						def apply_on_replica():
								optimizer.apply_gradients(list(zip(accumulator.gradients    ,   [variable]						)						)						)

						@tf.function
						def accumulate(lowercase__					:	Any    ,   lowercase__					:	List[Any]						):
								with strategy.scope():
										__lowercase				:		Dict												=				strategy.experimental_local_results(lowercase__						)
										local_variables[0].assign(lowercase__						)
										local_variables[1].assign(lowercase__						)
										strategy.run(lowercase__    ,   args=(gradient_placeholder,)						)

						@tf.function
						def apply_grad():
								with strategy.scope():
										strategy.run(lowercase__						)

						def _check_local_values(lowercase__					:	Dict    ,   lowercase__					:	Optional[int]						):
								__lowercase				:		str												=				strategy.experimental_local_results(accumulator._gradients[0]						)
								self.assertListAlmostEqual(values[0].value()    ,   lowercase__    ,   tol=1e-2						)
								self.assertListAlmostEqual(values[1].value()    ,   lowercase__    ,   tol=1e-2						)

						accumulate([1.0, 2.0]    ,   [-1.0, 1.0]						)
						accumulate([3.0, -1.0]    ,   [-1.0, -1.0]						)
						accumulate([-2.0, 2.0]    ,   [3.0, -2.0]						)
						self.assertEqual(accumulator.step    ,   3						)
						_check_local_values([2.0, 3.0]    ,   [1.0, -2.0]						)
						apply_grad()
						self.assertListAlmostEqual(variable.value()    ,   [4.0, 3.0]    ,   tol=1e-2						)
						accumulator.reset()
						self.assertEqual(accumulator.step    ,   0						)
						_check_local_values([0.0, 0.0]    ,   [0.0, 0.0]						)






"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A   :  str					=      logging.get_logger(__name__)

__A   :  str					=      {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}




class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				__UpperCAmelCase :     Tuple       =      "visual_bert"



				def __init__(     self					:	List[Any]    ,   lowercase__					:	Dict=3_0_5_2_2    ,   lowercase__					:	Any=7_6_8    ,   lowercase__					:	Dict=5_1_2    ,   lowercase__					:	Tuple=1_2    ,   lowercase__					:	Any=1_2    ,   lowercase__					:	Union[str, Any]=3_0_7_2    ,   lowercase__					:	Tuple="gelu"    ,   lowercase__					:	Dict=0.1    ,   lowercase__					:	Tuple=0.1    ,   lowercase__					:	str=5_1_2    ,   lowercase__					:	str=2    ,   lowercase__					:	Any=0.0_2    ,   lowercase__					:	Any=1e-12    ,   lowercase__					:	Any=False    ,   lowercase__					:	List[str]=True    ,   lowercase__					:	Dict=1    ,   lowercase__					:	List[Any]=0    ,   lowercase__					:	Tuple=2    ,   **lowercase__					:	Any    ,   ):
						super().__init__(pad_token_id=lowercase__    ,   bos_token_id=lowercase__    ,   eos_token_id=lowercase__    ,   **lowercase__						)

						__lowercase				:		Optional[int]												=				vocab_size
						__lowercase				:		Any												=				max_position_embeddings
						__lowercase				:		Optional[Any]												=				hidden_size
						__lowercase				:		List[str]												=				visual_embedding_dim
						__lowercase				:		Dict												=				num_hidden_layers
						__lowercase				:		Any												=				num_attention_heads
						__lowercase				:		str												=				intermediate_size
						__lowercase				:		Dict												=				hidden_act
						__lowercase				:		Any												=				hidden_dropout_prob
						__lowercase				:		Optional[int]												=				attention_probs_dropout_prob
						__lowercase				:		Dict												=				initializer_range
						__lowercase				:		str												=				type_vocab_size
						__lowercase				:		List[Any]												=				layer_norm_eps
						__lowercase				:		Tuple												=				bypass_transformer
						__lowercase				:		List[str]												=				special_visual_initialize
