

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_   							=  logging.get_logger(__name__)

lowercase_   							=  {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}





class 					__lowerCAmelCase	(   SCREAMING_SNAKE_CASE				):
  _a		     =    """visual_bert"""




  def __init__(							self				,  lowerCAmelCase=30_522				,  lowerCAmelCase=768				,  lowerCAmelCase=512				,  lowerCAmelCase=12				,  lowerCAmelCase=12				,  lowerCAmelCase=3_072				,  lowerCAmelCase="gelu"				,  lowerCAmelCase=0.1				,  lowerCAmelCase=0.1				,  lowerCAmelCase=512				,  lowerCAmelCase=2				,  lowerCAmelCase=0.02				,  lowerCAmelCase=1e-12				,  lowerCAmelCase=False				,  lowerCAmelCase=True				,  lowerCAmelCase=1				,  lowerCAmelCase=0				,  lowerCAmelCase=2				,  **lowerCAmelCase				,  )    ->					List[str]:
        '''simple docstring'''






        super().__init__(pad_token_id=lowerCAmelCase				,  bos_token_id=lowerCAmelCase				,  eos_token_id=lowerCAmelCase				,  **lowerCAmelCase   )

        _lowercase      			=vocab_size
        _lowercase      			=max_position_embeddings
        _lowercase      			=hidden_size
        _lowercase      			=visual_embedding_dim
        _lowercase      			=num_hidden_layers
        _lowercase      			=num_attention_heads
        _lowercase      			=intermediate_size
        _lowercase      			=hidden_act
        _lowercase      			=hidden_dropout_prob
        _lowercase      			=attention_probs_dropout_prob
        _lowercase      			=initializer_range
        _lowercase      			=type_vocab_size
        _lowercase      			=layer_norm_eps
        _lowercase      			=bypass_transformer
        _lowercase      			=special_visual_initialize





# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings('ignore', category=UserWarning, module='torch.optim.lr_scheduler')





class 					__lowerCAmelCase	:




  def __init__(							self				,  lowerCAmelCase				,  lowerCAmelCase				,  lowerCAmelCase = True				,  lowerCAmelCase = False   )    ->					str:
        '''simple docstring'''






        _lowercase      			=scheduler
        _lowercase      			=optimizers if isinstance(lowerCAmelCase				,  (list, tuple)   ) else [optimizers]
        _lowercase      			=split_batches
        _lowercase      			=step_with_optimizer
        _lowercase      			=GradientState()




  def     A__	(							self				,  *lowerCAmelCase				,  **lowerCAmelCase   )    ->					List[Any]:
        '''simple docstring'''






        if not self.step_with_optimizer:
              # No link between scheduler and optimizer -> just step
              self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
              return

        # Otherwise, first make sure the optimizer was stepped.
        if not self.gradient_state.sync_gradients:
              if self.gradient_state.adjust_scheduler:
                    self.scheduler._step_count += 1
              return

        for opt in self.optimizers:
              if opt.step_was_skipped:
                    return
        if self.split_batches:
              # Split batches -> the training dataloader batch size is not changed so one step per training step
              self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
        else:
              # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
              # num_processes steps per training step
              _lowercase      			=AcceleratorState().num_processes
              for _ in range(lowerCAmelCase   ):
                    # Special case when using OneCycle and `drop_last` was not used
                    if hasattr(self.scheduler				,  'total_steps'   ):
                          if self.scheduler._step_count <= self.scheduler.total_steps:
                                self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
                    else:
                          self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )




  def     A__	(							self   )    ->					str:
        '''simple docstring'''






        return self.scheduler.get_last_lr()




  def     A__	(							self   )    ->					Tuple:
        '''simple docstring'''






        return self.scheduler.state_dict()




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        self.scheduler.load_state_dict(lowerCAmelCase   )




  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return self.scheduler.get_lr()




  def     A__	(							self				,  *lowerCAmelCase				,  **lowerCAmelCase   )    ->					Optional[int]:
        '''simple docstring'''






        return self.scheduler.print_lr(*lowerCAmelCase				,  **lowerCAmelCase   )


