






'''simple docstring'''







import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class   __SCREAMING_SNAKE_CASE       (__A							,       __A							,       __A       ):


  """simple docstring"""


  @register_to_config
  def __init__(			self   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__   ,	UpperCamelCase__ = False   ,	):






         """simple docstring"""


         super().__init__()

         a_						     =			nn.Embedding(UpperCamelCase__   ,	UpperCamelCase__	)

         a_						     =			nn.Embedding(UpperCamelCase__   ,	UpperCamelCase__	)
         a_						     =			False

         a_						     =			nn.Dropout(p=UpperCamelCase__	)

         a_						     =			TaConfig(
             vocab_size=UpperCamelCase__   ,	d_model=UpperCamelCase__   ,	num_heads=UpperCamelCase__   ,	d_kv=UpperCamelCase__   ,	d_ff=UpperCamelCase__   ,	dropout_rate=UpperCamelCase__   ,	feed_forward_proj=UpperCamelCase__   ,	is_decoder=UpperCamelCase__   ,	is_encoder_decoder=UpperCamelCase__   ,	)

         a_						     =			nn.ModuleList()
         for lyr_num in range(UpperCamelCase__	):
                a_						     =			TaBlock(UpperCamelCase__	)
                self.encoders.append(UpperCamelCase__	)

         a_						     =			TaLayerNorm(UpperCamelCase__	)
         a_						     =			nn.Dropout(p=UpperCamelCase__	)







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__	):






         """simple docstring"""


         a_						     =			self.token_embedder(UpperCamelCase__	)

         a_						     =			encoder_input_tokens.shape[1]
         a_						     =			torch.arange(UpperCamelCase__   ,	device=encoder_input_tokens.device	)
         x += self.position_encoding(UpperCamelCase__	)

         a_						     =			self.dropout_pre(UpperCamelCase__	)

         # inverted the attention mask
         a_						     =			encoder_input_tokens.size()
         a_						     =			self.get_extended_attention_mask(UpperCamelCase__   ,	UpperCamelCase__	)

         for lyr in self.encoders:
                a_						     =			lyr(UpperCamelCase__   ,	UpperCamelCase__	)[0]
         a_						     =			self.layer_norm(UpperCamelCase__	)

         return self.dropout_post(UpperCamelCase__	), encoder_inputs_mask












'''simple docstring'''







import functools
import gc
import inspect

import torch

from .imports import is_npu_available, is_xpu_available


def       __UpperCamelCase					(      *lowercase_       :  Union[str, Any]   ):
       """simple docstring"""




       if not isinstance(lowercase_   ,       lowercase_   ):
              a_						     =			list(lowercase_   )
       for i in range(len(lowercase_   )   ):
              a_						     =			None
       gc.collect()
       if is_xpu_available():
              torch.xpu.empty_cache()
       elif is_npu_available():
              torch.npu.empty_cache()
       else:
              torch.cuda.empty_cache()
       return objects


def       __UpperCamelCase					(      lowercase_       :  Exception   ):
       """simple docstring"""




       a_						     =			[
           'CUDA out of memory.',  # CUDA OOM
           'cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.',  # CUDNN SNAFU
           'DefaultCPUAllocator: can\'t allocate memory',  # CPU OOM
       ]
       if isinstance(lowercase_   ,       lowercase_   ) and len(exception.args   ) == 1:
              return any(err in exception.args[0] for err in _statements   )
       return False


def       __UpperCamelCase					(      lowercase_       :  callable = None   ,       lowercase_       :  int = 128   ):
       """simple docstring"""




       if function is None:
              return functools.partial(lowercase_   ,       starting_batch_size=lowercase_   )

       a_						     =			starting_batch_size

       def decorator(*lowercase_       :  str   ,       **lowercase_       :  Dict   ):
              nonlocal batch_size
              gc.collect()
              if is_xpu_available():
                     torch.xpu.empty_cache()
              elif is_npu_available():
                     torch.npu.empty_cache()
              else:
                     torch.cuda.empty_cache()
              a_						     =			list(inspect.signature(lowercase_   ).parameters.keys()   )
              # Guard against user error
              if len(lowercase_   ) < (len(lowercase_   ) + 1):
                     a_						     =			', '.join([F'{arg}={value}' for arg, value in zip(params[1:]   ,       args[1:]   )]   )
                     raise TypeError(
                         F'Batch size was passed into `{function.__name__}` as the first argument when called.'
                         F'Remove this as the decorator already does so: `{function.__name__}({arg_str})`'   )
              while True:
                     if batch_size == 0:
                            raise RuntimeError('No executable batch size found, reached zero.'   )
                     try:
                            return function(lowercase_   ,       *lowercase_   ,       **lowercase_   )
                     except Exception as e:
                            if should_reduce_batch_size(lowercase_   ):
                                   gc.collect()
                                   if is_xpu_available():
                                          torch.xpu.empty_cache()
                                   elif is_npu_available():
                                          torch.npu.empty_cache()
                                   else:
                                          torch.cuda.empty_cache()
                                   batch_size //= 2
                            else:
                                   raise

       return decorator




