


import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


UpperCAmelCase			=    [nn.LayerNorm]

UpperCAmelCase			=    logging.get_logger(__name__)

UpperCAmelCase			=    version.parse(version.parse(torch.__version__).base_version)

UpperCAmelCase			=    parsed_torch_version_base >= version.parse("""2.0""")
UpperCAmelCase			=    parsed_torch_version_base >= version.parse("""1.12""")
UpperCAmelCase			=    parsed_torch_version_base >= version.parse("""1.11""")
UpperCAmelCase			=    parsed_torch_version_base < version.parse("""1.11""")






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					self					)->  Tuple:




      """simple docstring"""

      from torch import _softmax_backward_data

      if is_torch_less_than_1_11:
            return _softmax_backward_data(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					parent.dim					,					self					)
      else:
            return _softmax_backward_data(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					parent.dim					,					self.dtype					)






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE = 0					)->  nn.Linear:




      """simple docstring"""

      snake_case_       =      index.to(layer.weight.device					)
      snake_case_       =      layer.weight.index_select(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					).clone().detach()
      if layer.bias is not None:
            if dim == 1:
                  snake_case_       =      layer.bias.clone().detach()
            else:
                  snake_case_       =      layer.bias[index].clone().detach()
      snake_case_       =      list(layer.weight.size()					)
      snake_case_       =      len(SCREAMING_SNAKE_CASE					)
      snake_case_       =      nn.Linear(new_size[1]					,					new_size[0]					,					bias=layer.bias is not None					).to(layer.weight.device					)
      snake_case_       =      False
      new_layer.weight.copy_(W.contiguous()					)
      snake_case_       =      True
      if layer.bias is not None:
            snake_case_       =      False
            new_layer.bias.copy_(b.contiguous()					)
            snake_case_       =      True
      return new_layer







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase		,      _UpperCAmelCase					):
       super().__init__()
       snake_case_       =      nf
       snake_case_       =      nn.Parameter(torch.empty(_UpperCAmelCase		,      _UpperCAmelCase					)					)
       snake_case_       =      nn.Parameter(torch.zeros(_UpperCAmelCase					)					)
       nn.init.normal_(self.weight		,      std=0.02					)





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       snake_case_       =      x.size()[:-1] + (self.nf,)
       snake_case_       =      torch.addmm(self.bias		,      x.view(-1		,      x.size(-1					)					)		,      self.weight					)
       snake_case_       =      x.view(_UpperCAmelCase					)
       return x






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE = 1					)->  ConvaD:




      """simple docstring"""

      snake_case_       =      index.to(layer.weight.device					)
      snake_case_       =      layer.weight.index_select(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					).clone().detach()
      if dim == 0:
            snake_case_       =      layer.bias.clone().detach()
      else:
            snake_case_       =      layer.bias[index].clone().detach()
      snake_case_       =      list(layer.weight.size()					)
      snake_case_       =      len(SCREAMING_SNAKE_CASE					)
      snake_case_       =      ConvaD(new_size[1]					,					new_size[0]					).to(layer.weight.device					)
      snake_case_       =      False
      new_layer.weight.copy_(W.contiguous()					)
      snake_case_       =      True
      snake_case_       =      False
      new_layer.bias.copy_(b.contiguous()					)
      snake_case_       =      True
      return new_layer






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE = None					)->  Union[nn.Linear, ConvaD]:




      """simple docstring"""

      if isinstance(SCREAMING_SNAKE_CASE					,					nn.Linear					):
            return prune_linear_layer(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					dim=0 if dim is None else dim					)
      elif isinstance(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					):
            return prune_convad_layer(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					dim=1 if dim is None else dim					)
      else:
            raise ValueError(f'''Can\'t prune layer of class {layer.__class__}'''					)






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					*SCREAMING_SNAKE_CASE					)->  torch.Tensor:




      """simple docstring"""

      assert len(SCREAMING_SNAKE_CASE					) > 0, f'''{input_tensors} has to be a tuple/list of tensors'''

      # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
      snake_case_       =      len(inspect.signature(SCREAMING_SNAKE_CASE					).parameters					)
      if num_args_in_forward_chunk_fn != len(SCREAMING_SNAKE_CASE					):
            raise ValueError(
                f'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(SCREAMING_SNAKE_CASE					)} input '''
                '''tensors are given'''					)

      if chunk_size > 0:
            snake_case_       =      input_tensors[0].shape[chunk_dim]
            for input_tensor in input_tensors:
                  if input_tensor.shape[chunk_dim] != tensor_shape:
                        raise ValueError(
                            f'''All input tenors have to be of the same shape: {tensor_shape}, '''
                            f'''found shape {input_tensor.shape[chunk_dim]}'''					)

            if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
                  raise ValueError(
                      f'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
                      f'''size {chunk_size}'''					)

            snake_case_       =      input_tensors[0].shape[chunk_dim] // chunk_size

            # chunk input tensor into tuples
            snake_case_       =      tuple(input_tensor.chunk(SCREAMING_SNAKE_CASE					,					dim=SCREAMING_SNAKE_CASE					) for input_tensor in input_tensors					)
            # apply forward fn to every tuple
            snake_case_       =      tuple(forward_fn(*SCREAMING_SNAKE_CASE					) for input_tensors_chunk in zip(*SCREAMING_SNAKE_CASE					)					)
            # concatenate output at same dimension
            return torch.cat(SCREAMING_SNAKE_CASE					,					dim=SCREAMING_SNAKE_CASE					)

      return forward_fn(*SCREAMING_SNAKE_CASE					)






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					)->  Tuple[Set[int], torch.LongTensor]:




      """simple docstring"""

      snake_case_       =      torch.ones(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					)
      snake_case_       =      set(SCREAMING_SNAKE_CASE					) - already_pruned_heads  # Convert to set and remove already pruned heads
      for head in heads:
            # Compute how many pruned heads are before the head and move the index accordingly
            snake_case_       =      head - sum(1 if h < head else 0 for h in already_pruned_heads					)
            snake_case_       =      0
      snake_case_       =      mask.view(-1					).contiguous().eq(1					)
      snake_case_       =      torch.arange(len(SCREAMING_SNAKE_CASE					)					)[mask].long()
      return heads, index






def        __lowerCAmelCase    (*SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE = None					)->  Tuple[torch.Tensor, ...]:




      """simple docstring"""

      return torch.meshgrid(*SCREAMING_SNAKE_CASE					,					indexing=SCREAMING_SNAKE_CASE					)






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  Tuple[torch.device, int, int]:




      """simple docstring"""

      return tensor.device, storage_ptr(SCREAMING_SNAKE_CASE					), storage_size(SCREAMING_SNAKE_CASE					)



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase			=    {
    """configuration_big_bird""": ["""BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BigBirdConfig""", """BigBirdOnnxConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """BigBirdForCausalLM""",
       """BigBirdForMaskedLM""",
       """BigBirdForMultipleChoice""",
       """BigBirdForPreTraining""",
       """BigBirdForQuestionAnswering""",
       """BigBirdForSequenceClassification""",
       """BigBirdForTokenClassification""",
       """BigBirdLayer""",
       """BigBirdModel""",
       """BigBirdPreTrainedModel""",
       """load_tf_weights_in_big_bird""",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """FlaxBigBirdForCausalLM""",
       """FlaxBigBirdForMaskedLM""",
       """FlaxBigBirdForMultipleChoice""",
       """FlaxBigBirdForPreTraining""",
       """FlaxBigBirdForQuestionAnswering""",
       """FlaxBigBirdForSequenceClassification""",
       """FlaxBigBirdForTokenClassification""",
       """FlaxBigBirdModel""",
       """FlaxBigBirdPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird import BigBirdTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird_fast import BigBirdTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_big_bird import (
          BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
          BigBirdForCausalLM,
          BigBirdForMaskedLM,
          BigBirdForMultipleChoice,
          BigBirdForPreTraining,
          BigBirdForQuestionAnswering,
          BigBirdForSequenceClassification,
          BigBirdForTokenClassification,
          BigBirdLayer,
          BigBirdModel,
          BigBirdPreTrainedModel,
          load_tf_weights_in_big_bird,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_big_bird import (
          FlaxBigBirdForCausalLM,
          FlaxBigBirdForMaskedLM,
          FlaxBigBirdForMultipleChoice,
          FlaxBigBirdForPreTraining,
          FlaxBigBirdForQuestionAnswering,
          FlaxBigBirdForSequenceClassification,
          FlaxBigBirdForTokenClassification,
          FlaxBigBirdModel,
          FlaxBigBirdPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase			=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)