



"""simple docstring"""




import unittest

from transformers import BigBirdConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
 import jax

 from transformers.models.big_bird.modeling_flax_big_bird import (
     FlaxBigBirdForCausalLM,
     FlaxBigBirdForMaskedLM,
     FlaxBigBirdForMultipleChoice,
     FlaxBigBirdForPreTraining,
     FlaxBigBirdForQuestionAnswering,
     FlaxBigBirdForSequenceClassification,
     FlaxBigBirdForTokenClassification,
     FlaxBigBirdModel,
 )






class       UpperCAmelCase_    (							unittest.TestCase		):



      def __init__(	self	:      Tuple	,      A	:      Tuple	,      A	:      int=2	,      A	:      List[Any]=5_6	,      A	:      Optional[int]=True	,      A	:      Any=True	,      A	:      str=True	,      A	:      Any=True	,      A	:      Any=9_9	,      A	:      Tuple=3_2	,      A	:      Optional[int]=2	,      A	:      Dict=2	,      A	:      Tuple=7	,      A	:      List[Any]="gelu_new"	,      A	:      List[str]=0.1	,      A	:      Dict=0.1	,      A	:      Any=5_1_2	,      A	:      Any=1_6	,      A	:      str=2	,      A	:      int=0.02	,      A	:      Optional[int]=4	,      A	:      Tuple="block_sparse"	,      A	:      List[Any]=True	,      A	:      Dict=False	,      A	:      List[Any]=2	,      A	:      Any=3	,      ):
           _UpperCAmelCase     :      Any														=       parent
           _UpperCAmelCase     :      Optional[int]														=       batch_size
           _UpperCAmelCase     :      Dict														=       seq_length
           _UpperCAmelCase     :      List[Any]														=       is_training
           _UpperCAmelCase     :      Dict														=       use_attention_mask
           _UpperCAmelCase     :      Union[str, Any]														=       use_token_type_ids
           _UpperCAmelCase     :      Any														=       use_labels
           _UpperCAmelCase     :      str														=       vocab_size
           _UpperCAmelCase     :      str														=       hidden_size
           _UpperCAmelCase     :      Optional[int]														=       num_hidden_layers
           _UpperCAmelCase     :      Union[str, Any]														=       num_attention_heads
           _UpperCAmelCase     :      str														=       intermediate_size
           _UpperCAmelCase     :      List[Any]														=       hidden_act
           _UpperCAmelCase     :      Optional[int]														=       hidden_dropout_prob
           _UpperCAmelCase     :      Dict														=       attention_probs_dropout_prob
           _UpperCAmelCase     :      List[Any]														=       max_position_embeddings
           _UpperCAmelCase     :      Any														=       type_vocab_size
           _UpperCAmelCase     :      Tuple														=       type_sequence_label_size
           _UpperCAmelCase     :      Tuple														=       initializer_range
           _UpperCAmelCase     :      str														=       num_choices

           _UpperCAmelCase     :      Union[str, Any]														=       rescale_embeddings
           _UpperCAmelCase     :      int														=       attention_type
           _UpperCAmelCase     :      str														=       use_bias
           _UpperCAmelCase     :      List[Any]														=       block_size
           _UpperCAmelCase     :      Union[str, Any]														=       num_random_blocks



      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Dict														=       ids_tensor([self.batch_size, self.seq_length]	,      self.vocab_size		)

           _UpperCAmelCase     :      Dict														=       None
           if self.use_attention_mask:
                _UpperCAmelCase     :      str														=       random_attention_mask([self.batch_size, self.seq_length]		)

           _UpperCAmelCase     :      Dict														=       None
           if self.use_token_type_ids:
                _UpperCAmelCase     :      int														=       ids_tensor([self.batch_size, self.seq_length]	,      self.type_vocab_size		)

           _UpperCAmelCase     :      List[Any]														=       BigBirdConfig(
               vocab_size=self.vocab_size	,      hidden_size=self.hidden_size	,      num_hidden_layers=self.num_hidden_layers	,      num_attention_heads=self.num_attention_heads	,      intermediate_size=self.intermediate_size	,      hidden_act=self.hidden_act	,      hidden_dropout_prob=self.hidden_dropout_prob	,      attention_probs_dropout_prob=self.attention_probs_dropout_prob	,      max_position_embeddings=self.max_position_embeddings	,      type_vocab_size=self.type_vocab_size	,      is_decoder=A	,      initializer_range=self.initializer_range	,      attention_type=self.attention_type	,      block_size=self.block_size	,      num_random_blocks=self.num_random_blocks	,      use_bias=self.use_bias	,      rescale_embeddings=self.rescale_embeddings	,      )

           return config, input_ids, token_type_ids, attention_mask



      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      str														=       self.prepare_config_and_inputs()
           _UpperCAmelCase							,			_UpperCAmelCase							,			_UpperCAmelCase							,			_UpperCAmelCase     :      Dict														=       config_and_inputs
           _UpperCAmelCase     :      Optional[int]														=       {
               "input_ids": input_ids,
               "token_type_ids": token_type_ids,
               "attention_mask": attention_mask,
           }
           return config, inputs_dict







@require_flax
class       UpperCAmelCase_    (							_UpperCamelCase			,     unittest.TestCase		):
      __SCREAMING_SNAKE_CASE		:  int						=					(
          (
              FlaxBigBirdForCausalLM,
              FlaxBigBirdModel,
              FlaxBigBirdForPreTraining,
              FlaxBigBirdForMaskedLM,
              FlaxBigBirdForMultipleChoice,
              FlaxBigBirdForQuestionAnswering,
              FlaxBigBirdForSequenceClassification,
              FlaxBigBirdForTokenClassification,
          )
          if is_flax_available()
          else ()
      )

      __SCREAMING_SNAKE_CASE		:  Any						=					False
      __SCREAMING_SNAKE_CASE		:  List[Any]						=					False



      def 		snake_case_   (	self	:      Tuple		):
           _UpperCAmelCase     :      Union[str, Any]														=       FlaxBigBirdModelTester(self		)



      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 		snake_case_   (	self	:      Dict		):
           super().test_from_pretrained_save_pretrained()



      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 		snake_case_   (	self	:      Tuple		):
           super().test_from_pretrained_with_no_automatic_init()



      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 		snake_case_   (	self	:      Any		):
           super().test_no_automatic_init()



      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 		snake_case_   (	self	:      Tuple		):
           super().test_hidden_states_output()



      @slow
      def 		snake_case_   (	self	:      List[Any]		):
           for model_class_name in self.all_model_classes:
                _UpperCAmelCase     :      int														=       model_class_name.from_pretrained("google/bigbird-roberta-base"		)
                self.assertIsNotNone(A		)



      def 		snake_case_   (	self	:      Dict		):
           if self.test_attn_probs:
                super().test_attention_outputs()



      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 		snake_case_   (	self	:      Tuple		):
           _UpperCAmelCase							,			_UpperCAmelCase     :      Union[str, Any]														=       self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                with self.subTest(model_class.__name__		):
                     _UpperCAmelCase     :      str														=       self._prepare_for_class(A	,      A		)
                     _UpperCAmelCase     :      Dict														=       model_class(A		)

                     @jax.jit
                     def model_jitted(A	:      int	,      A	:      int=None	,      **A	:      List[str]		):
                          return model(input_ids=A	,      attention_mask=A	,      **A		)

                     with self.subTest("JIT Enabled"		):
                          _UpperCAmelCase     :      Any														=       model_jitted(**A		).to_tuple()

                     with self.subTest("JIT Disabled"		):
                          with jax.disable_jit():
                               _UpperCAmelCase     :      Optional[int]														=       model_jitted(**A		).to_tuple()

                     self.assertEqual(len(A		)	,      len(A		)		)
                     for jitted_output, output in zip(A	,      A		):
                          self.assertEqual(jitted_output.shape	,      output.shape		)



      def 		snake_case_   (	self	:      Optional[int]	,      A	:      int	,      A	:      Any	,      A	:      Optional[Any]	,      A	:      Dict=1e-5	,      A	:      List[Any]="outputs"	,      A	:      List[Any]=None		):
           # `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version,
           # an effort was done to return `attention_probs` (yet to be verified).
           if name.startswith("outputs.attentions"		):
                return
           else:
                super().check_pt_flax_outputs(A	,      A	,      A	,      A	,      A	,      A		)





"""simple docstring"""




# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")






class       UpperCAmelCase_    :



      def __init__(	self	:      Optional[Any]	,      A	:      List[str]	,      A	:      List[str]	,      A	:      bool = True	,      A	:      bool = False		):
           _UpperCAmelCase     :      Dict														=       scheduler
           _UpperCAmelCase     :      str														=       optimizers if isinstance(A	,      (list, tuple)		) else [optimizers]
           _UpperCAmelCase     :      List[str]														=       split_batches
           _UpperCAmelCase     :      Dict														=       step_with_optimizer
           _UpperCAmelCase     :      Tuple														=       GradientState()



      def 		snake_case_   (	self	:      Optional[int]	,      *A	:      List[Any]	,      **A	:      Optional[int]		):
           if not self.step_with_optimizer:
                # No link between scheduler and optimizer -> just step
                self.scheduler.step(*A	,      **A		)
                return

           # Otherwise, first make sure the optimizer was stepped.
           if not self.gradient_state.sync_gradients:
                if self.gradient_state.adjust_scheduler:
                     self.scheduler._step_count += 1
                return

           for opt in self.optimizers:
                if opt.step_was_skipped:
                     return
           if self.split_batches:
                # Split batches -> the training dataloader batch size is not changed so one step per training step
                self.scheduler.step(*A	,      **A		)
           else:
                # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
                # num_processes steps per training step
                _UpperCAmelCase     :      int														=       AcceleratorState().num_processes
                for _ in range(A		):
                     # Special case when using OneCycle and `drop_last` was not used
                     if hasattr(self.scheduler	,      "total_steps"		):
                          if self.scheduler._step_count <= self.scheduler.total_steps:
                               self.scheduler.step(*A	,      **A		)
                     else:
                          self.scheduler.step(*A	,      **A		)



      def 		snake_case_   (	self	:      Union[str, Any]		):
           return self.scheduler.get_last_lr()



      def 		snake_case_   (	self	:      str		):
           return self.scheduler.state_dict()



      def 		snake_case_   (	self	:      str	,      A	:      Optional[Any]		):
           self.scheduler.load_state_dict(A		)



      def 		snake_case_   (	self	:      Tuple		):
           return self.scheduler.get_lr()



      def 		snake_case_   (	self	:      Optional[int]	,      *A	:      List[str]	,      **A	:      Optional[Any]		):
           return self.scheduler.print_lr(*A	,      **A		)
