
"""simple docstring"""






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
 import torch

 from transformers import XLMRobertaModel





@require_sentencepiece
@require_tokenizers
@require_torch
class 				a__					(						unittest.TestCase       ):



     @slow
     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=XLMRobertaModel.from_pretrained('xlm-roberta-base'   )
      UpperCamelCase_     :					str   	=torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]]   )
      # The dog is cute and lives in the garden house

      UpperCamelCase_     :					List[str]   	=torch.Size((1, 12, 768)   )  # batch_size, sequence_length, embedding_vector_dim
      UpperCamelCase_     :					Any   	=torch.tensor(
          [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]   )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
       UpperCamelCase_     :					Dict   	=model(_lowerCamelCase   )['last_hidden_state'].detach()
      self.assertEqual(output.shape      ,  _lowerCamelCase   )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]      ,  _lowerCamelCase      ,  atol=1E-3   )   )




     @slow
     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=XLMRobertaModel.from_pretrained('xlm-roberta-large'   )
      UpperCamelCase_     :					Optional[int]   	=torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]]   )
      # The dog is cute and lives in the garden house

      UpperCamelCase_     :					Union[str, Any]   	=torch.Size((1, 12, 1_024)   )  # batch_size, sequence_length, embedding_vector_dim
      UpperCamelCase_     :					int   	=torch.tensor(
          [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]   )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
       UpperCamelCase_     :					int   	=model(_lowerCamelCase   )['last_hidden_state'].detach()
      self.assertEqual(output.shape      ,  _lowerCamelCase   )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]      ,  _lowerCamelCase      ,  atol=1E-3   )   )


"""simple docstring"""






# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings('ignore', category=UserWarning, module='torch.optim.lr_scheduler')





class 				a__					:



     def __init__(							self :List[str]      ,  _lowerCamelCase :Any      ,  _lowerCamelCase :Optional[Any]      ,  _lowerCamelCase :bool = True      ,  _lowerCamelCase :bool = False   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=scheduler
      UpperCamelCase_     :					Any   	=optimizers if isinstance(_lowerCamelCase      ,  (list, tuple)   ) else [optimizers]
      UpperCamelCase_     :					Dict   	=split_batches
      UpperCamelCase_     :					str   	=step_with_optimizer
      UpperCamelCase_     :					Dict   	=GradientState()



     def  lowerCamelCase_						(							self :Optional[int]      ,  *_lowerCamelCase :str      ,  **_lowerCamelCase :Union[str, Any]   ):







      '''simple docstring'''



      if not self.step_with_optimizer:
       # No link between scheduler and optimizer -> just step
       self.scheduler.step(*_lowerCamelCase      ,  **_lowerCamelCase   )
       return

      # Otherwise, first make sure the optimizer was stepped.
      if not self.gradient_state.sync_gradients:
       if self.gradient_state.adjust_scheduler:
        self.scheduler._step_count += 1
       return

      for opt in self.optimizers:
       if opt.step_was_skipped:
        return
      if self.split_batches:
       # Split batches -> the training dataloader batch size is not changed so one step per training step
       self.scheduler.step(*_lowerCamelCase      ,  **_lowerCamelCase   )
      else:
       # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
       # num_processes steps per training step
       UpperCamelCase_     :					Optional[Any]   	=AcceleratorState().num_processes
       for _ in range(_lowerCamelCase   ):
        # Special case when using OneCycle and `drop_last` was not used
        if hasattr(self.scheduler      ,  'total_steps'   ):
         if self.scheduler._step_count <= self.scheduler.total_steps:
          self.scheduler.step(*_lowerCamelCase      ,  **_lowerCamelCase   )
        else:
         self.scheduler.step(*_lowerCamelCase      ,  **_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      return self.scheduler.get_last_lr()



     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      return self.scheduler.state_dict()



     def  lowerCamelCase_						(							self :Any      ,  _lowerCamelCase :Any   ):







      '''simple docstring'''



      self.scheduler.load_state_dict(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      return self.scheduler.get_lr()




     def  lowerCamelCase_						(							self :Optional[int]      ,  *_lowerCamelCase :Union[str, Any]      ,  **_lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      return self.scheduler.print_lr(*_lowerCamelCase      ,  **_lowerCamelCase   )
