




from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


lowerCamelCase_  						=   logging.get_logger(__name__)

lowerCamelCase_  						=   {
    "google/long-t5-local-base": "https://huggingface.co/google/long-t5-local-base/blob/main/config.json",
    "google/long-t5-local-large": "https://huggingface.co/google/long-t5-local-large/blob/main/config.json",
    "google/long-t5-tglobal-base": "https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json",
    "google/long-t5-tglobal-large": "https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json",
}
class       __a					( _UpperCamelCase     ):


     """simple docstring"""


     _A				:    Tuple									=  "longt5"
     _A				:    Any									=  ["past_key_values"]
     _A				:    Dict									=  {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}




     def __init__(	self			: str				,_UpperCamelCase			: List[Any]=3_2_1_2_8				,_UpperCamelCase			: Optional[Any]=5_1_2				,_UpperCamelCase			: str=6_4				,_UpperCamelCase			: int=2_0_4_8				,_UpperCamelCase			: Tuple=6				,_UpperCamelCase			: List[str]=None				,_UpperCamelCase			: Optional[int]=8				,_UpperCamelCase			: Optional[int]=1_2_7				,_UpperCamelCase			: List[str]=1_6				,_UpperCamelCase			: Any=3_2				,_UpperCamelCase			: Dict=1_2_8				,_UpperCamelCase			: str=0.1				,_UpperCamelCase			: List[str]=1e-6				,_UpperCamelCase			: Optional[int]=1.0				,_UpperCamelCase			: Union[str, Any]="relu"				,_UpperCamelCase			: Optional[int]=True				,_UpperCamelCase			: Any="local"				,_UpperCamelCase			: Dict=True				,_UpperCamelCase			: Union[str, Any]=0				,_UpperCamelCase			: int=1				,**_UpperCamelCase			: str				,)		->   Optional[int]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=vocab_size
         SCREAMING_SNAKE_CASE__								=d_model
         SCREAMING_SNAKE_CASE__								=d_kv
         SCREAMING_SNAKE_CASE__								=d_ff
         SCREAMING_SNAKE_CASE__								=num_layers
         # default = symmetry
         SCREAMING_SNAKE_CASE__								=num_decoder_layers if num_decoder_layers is not None else self.num_layers
         SCREAMING_SNAKE_CASE__								=num_heads
         SCREAMING_SNAKE_CASE__								=local_radius
         SCREAMING_SNAKE_CASE__								=global_block_size
         SCREAMING_SNAKE_CASE__								=relative_attention_num_buckets
         SCREAMING_SNAKE_CASE__								=relative_attention_max_distance
         SCREAMING_SNAKE_CASE__								=dropout_rate
         SCREAMING_SNAKE_CASE__								=layer_norm_epsilon
         SCREAMING_SNAKE_CASE__								=initializer_factor
         SCREAMING_SNAKE_CASE__								=feed_forward_proj
         SCREAMING_SNAKE_CASE__								=encoder_attention_type
         SCREAMING_SNAKE_CASE__								=use_cache

         SCREAMING_SNAKE_CASE__								=self.feed_forward_proj.split("""-"""						)
         SCREAMING_SNAKE_CASE__								=act_info[-1]
         SCREAMING_SNAKE_CASE__								=act_info[0] == """gated"""

         if len(__a						) > 1 and act_info[0] != "gated" or len(__a						) > 2:
             raise ValueError(
                 f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
                 """Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. """
                 """\'gated-gelu\' or \'relu\'"""						)

         # for backwards compatibility
         if feed_forward_proj == "gated-gelu":
             SCREAMING_SNAKE_CASE__								="""gelu_new"""

         super().__init__(
             pad_token_id=__a				,eos_token_id=__a				,is_encoder_decoder=__a				,**__a				,)






class       __a					( _UpperCamelCase     ):


     """simple docstring"""


     @property
     def   __A				(	self			: Optional[Any]						)		->   Mapping[str, Mapping[int, str]]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								={
             """input_ids""": {0: """batch""", 1: """encoder_sequence"""},
             """attention_mask""": {0: """batch""", 1: """encoder_sequence"""},
         }
         if self.use_past:
             SCREAMING_SNAKE_CASE__								="""past_encoder_sequence + sequence"""
             SCREAMING_SNAKE_CASE__								={0: """batch"""}
             SCREAMING_SNAKE_CASE__								={0: """batch""", 1: """past_decoder_sequence + sequence"""}
         else:
             SCREAMING_SNAKE_CASE__								={0: """batch""", 1: """decoder_sequence"""}
             SCREAMING_SNAKE_CASE__								={0: """batch""", 1: """decoder_sequence"""}

         if self.use_past:
             self.fill_with_past_key_values_(__a				,direction="""inputs"""						)

         return common_inputs




     @property
     def   __A				(	self			: Tuple						)		->   int:


         '''simple docstring'''





         return 1_3









import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


_UpperCamelCase			       =			namedtuple('''ExampleNamedTuple''', '''a b c''')



class     lowercase						(       unittest.TestCase					):



 '''simple docstring'''


 def   UpperCamelCase__							(self ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				torch.randn(5       ,       2 )
  UpperCAmelCase__							=				torch.device('cuda' ) if torch.cuda.is_available() else torch.device('cpu' )

  UpperCAmelCase__							=				send_to_device(__a       ,       __a )
  self.assertTrue(torch.equal(resulta.cpu()       ,       __a ) )

  UpperCAmelCase__							=				send_to_device((tensor, [tensor, tensor], 1)       ,       __a )
  self.assertIsInstance(__a       ,       __a )
  self.assertTrue(torch.equal(resulta[0].cpu()       ,       __a ) )
  self.assertIsInstance(resulta[1]       ,       __a )
  self.assertTrue(torch.equal(resulta[1][0].cpu()       ,       __a ) )
  self.assertTrue(torch.equal(resulta[1][1].cpu()       ,       __a ) )
  self.assertEqual(resulta[2]       ,       1 )

  UpperCAmelCase__							=				send_to_device({'a': tensor, 'b': [tensor, tensor], 'c': 1}       ,       __a )
  self.assertIsInstance(__a       ,       __a )
  self.assertTrue(torch.equal(resulta['a'].cpu()       ,       __a ) )
  self.assertIsInstance(resulta['b']       ,       __a )
  self.assertTrue(torch.equal(resulta['b'][0].cpu()       ,       __a ) )
  self.assertTrue(torch.equal(resulta['b'][1].cpu()       ,       __a ) )
  self.assertEqual(resulta['c']       ,       1 )

  UpperCAmelCase__							=				send_to_device(ExampleNamedTuple(a=__a       ,       b=[tensor, tensor]       ,       c=1 )       ,       __a )
  self.assertIsInstance(__a       ,       __a )
  self.assertTrue(torch.equal(resulta.a.cpu()       ,       __a ) )
  self.assertIsInstance(resulta.b       ,       __a )
  self.assertTrue(torch.equal(resulta.b[0].cpu()       ,       __a ) )
  self.assertTrue(torch.equal(resulta.b[1].cpu()       ,       __a ) )
  self.assertEqual(resulta.c       ,       1 )

  UpperCAmelCase__							=				send_to_device(UserDict({'a': tensor, 'b': [tensor, tensor], 'c': 1} )       ,       __a )
  self.assertIsInstance(__a       ,       __a )
  self.assertTrue(torch.equal(resulta['a'].cpu()       ,       __a ) )
  self.assertIsInstance(resulta['b']       ,       __a )
  self.assertTrue(torch.equal(resulta['b'][0].cpu()       ,       __a ) )
  self.assertTrue(torch.equal(resulta['b'][1].cpu()       ,       __a ) )
  self.assertEqual(resulta['c']       ,       1 )




 def   UpperCamelCase__							(self ) ->					List[str]:






  """simple docstring"""





  with self.assertRaises(__a ) as cm:
   UpperCAmelCase__							=				recursively_apply(torch.tensor       ,       (torch.tensor(1 ), 1)       ,       error_on_other_type=__a )
  self.assertEqual(
      str(cm.exception )       ,       'Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.'       ,       )




 def   UpperCamelCase__							(self ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				torch.tensor([1, 2, 3, 4, 5] )
  self.assertEqual(listify(__a )       ,       [1, 2, 3, 4, 5] )

  UpperCAmelCase__							=				torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] )
  self.assertEqual(listify(__a )       ,       [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] )

  UpperCAmelCase__							=				torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]] )
  self.assertEqual(
      listify(__a )       ,       [[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]] )




 def   UpperCamelCase__							(self ) ->					int:






  """simple docstring"""





  with patch_environment(aa=1       ,       BB=2 ):
   self.assertEqual(os.environ.get('AA' )       ,       '1' )
   self.assertEqual(os.environ.get('BB' )       ,       '2' )

  self.assertNotIn('AA'       ,       os.environ )
  self.assertNotIn('BB'       ,       os.environ )




 def   UpperCamelCase__							(self ) ->					Dict:






  """simple docstring"""





  UpperCAmelCase__							=				RegressionModel()
  UpperCAmelCase__							=				model.forward
  UpperCAmelCase__							=				convert_outputs_to_fpaa(model.forward )
  UpperCAmelCase__							=				extract_model_from_parallel(__a       ,       keep_fpaa_wrapper=__a )
  UpperCAmelCase__							=				pickle.dumps(__a )




 @require_cuda
 def   UpperCamelCase__							(self ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				RegressionModel()
  UpperCAmelCase__							=				model.forward
  UpperCAmelCase__							=				torch.cuda.amp.autocast(dtype=torch.floataa )(model.forward )
  UpperCAmelCase__							=				convert_outputs_to_fpaa(model.forward )
  UpperCAmelCase__							=				extract_model_from_parallel(__a       ,       keep_fpaa_wrapper=__a )
  UpperCAmelCase__							=				pickle.dumps(__a )




 @require_cuda
 @require_torch_min_version(version='2.0' )
 def   UpperCamelCase__							(self ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				RegressionModel()
  UpperCAmelCase__							=				model.forward
  UpperCAmelCase__							=				torch.cuda.amp.autocast(dtype=torch.floataa )(model.forward )
  UpperCAmelCase__							=				convert_outputs_to_fpaa(model.forward )
  UpperCAmelCase__							=				torch.compile(model.forward       ,       backend='inductor' )
  UpperCAmelCase__							=				torch.randn(4       ,       10 ).cuda()
  UpperCAmelCase__							=				model(__a )




 def   UpperCamelCase__							(self ) ->					Optional[Any]:






  """simple docstring"""





  UpperCAmelCase__							=				RegressionModel()
  # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
  UpperCAmelCase__							=				torch.nn.parallel.DataParallel(__a )
  UpperCAmelCase__							=				extract_model_from_parallel(__a )

  self.assertEqual(__a       ,       __a )




 @require_torch_min_version(version='2.0' )
 def   UpperCamelCase__							(self ) ->					Optional[Any]:






  """simple docstring"""





  UpperCAmelCase__							=				RegressionModel()
  UpperCAmelCase__							=				torch.compile(__a )

  # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
  UpperCAmelCase__							=				torch.nn.parallel.DataParallel(__a )
  UpperCAmelCase__							=				torch.compile(__a )
  UpperCAmelCase__							=				extract_model_from_parallel(__a )

  self.assertEqual(compiled_model._orig_mod       ,       compiled_model_unwrapped._orig_mod )




 def   UpperCamelCase__							(self ) ->					str:






  """simple docstring"""





  self.assertEqual(find_device([1, 'a', torch.tensor([1, 2, 3] )] )       ,       torch.device('cpu' ) )
  self.assertEqual(find_device({'a': 1, 'b': torch.tensor([1, 2, 3] )} )       ,       torch.device('cpu' ) )
  self.assertIsNone(find_device([1, 'a'] ) )



