


import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


UpperCAmelCase			=    logging.get_logger(__name__)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self					):
       super().__init__()
       if version.parse(torch.__version__					) < version.parse('''1.12.0'''					):
             raise ImportError(
                 F'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
                 '''PytorchGELUTanh. Please upgrade torch.'''					)





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return nn.functional.gelu(_UpperCAmelCase		,      approximate='''tanh'''					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi					) * (input + 0.044_715 * torch.pow(_UpperCAmelCase		,      3.0					))					))







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase = False					):
       super().__init__()
       if use_gelu_python:
             snake_case_       =      self._gelu_python
       else:
             snake_case_       =      nn.functional.gelu



 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0					)					))





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return self.act(_UpperCAmelCase					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)					))







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return input * torch.sigmoid(1.702 * input					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase		,      _UpperCAmelCase					):
       if min > max:
             raise ValueError(F'''min should be < max (got min: {min}, max: {max})'''					)

       super().__init__()
       snake_case_       =      min
       snake_case_       =      max





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return torch.clip(gelu(_UpperCAmelCase					)		,      self.min		,      self.max					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self					):
       super().__init__()
       snake_case_       =      math.sqrt(2 / math.pi					)





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(_UpperCAmelCase		,      3					))					))







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return nn.functional.silu(_UpperCAmelCase					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def __init__(	self					):
       super().__init__()
       if version.parse(torch.__version__					) < version.parse('''1.9.0'''					):
             snake_case_       =      self._mish_python
       else:
             snake_case_       =      nn.functional.mish



 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return input * torch.tanh(nn.functional.softplus(_UpperCAmelCase					)					)





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return self.act(_UpperCAmelCase					)







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       return input







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      _UpperCAmelCase=0.707_107		,      _UpperCAmelCase=0.282_095					):
       snake_case_       =      (input - mu).div(sigma * math.sqrt(2.0					)					)
       return 0.5 * (1.0 + torch.erf(_UpperCAmelCase					))







class lowerCAmelCase_						( nn.Module       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       snake_case_       =      nn.functional.relu(_UpperCAmelCase					)
       snake_case_       =      torch.square(_UpperCAmelCase					)
       return squared







class lowerCAmelCase_						( lowerCamelCase__       ):

 '''simple docstring'''

 def __getitem__(	self		,      _UpperCAmelCase					):
       snake_case_       =      super().__getitem__(_UpperCAmelCase					)
       snake_case_       ,    snake_case_       =      content if isinstance(_UpperCAmelCase		,      _UpperCAmelCase					) else (content, {})
       return cls(**_UpperCAmelCase					)





UpperCAmelCase			=    {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -10, """max""": 10}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
UpperCAmelCase			=    ClassInstantier(ACTaCLS)






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  List[str]:




      """simple docstring"""

      if activation_string in ACTaFN:
            return ACTaFN[activation_string]
      else:
            raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()					)}'''					)


# For backwards compatibility with: from activations import gelu_python
UpperCAmelCase			=    get_activation("""gelu_python""")
UpperCAmelCase			=    get_activation("""gelu_new""")
UpperCAmelCase			=    get_activation("""gelu""")
UpperCAmelCase			=    get_activation("""gelu_fast""")
UpperCAmelCase			=    get_activation("""quick_gelu""")
UpperCAmelCase			=    get_activation("""silu""")
UpperCAmelCase			=    get_activation("""mish""")




UpperCAmelCase			=    get_activation("""linear""")



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase			=    {
    """configuration_big_bird""": ["""BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BigBirdConfig""", """BigBirdOnnxConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """BigBirdForCausalLM""",
       """BigBirdForMaskedLM""",
       """BigBirdForMultipleChoice""",
       """BigBirdForPreTraining""",
       """BigBirdForQuestionAnswering""",
       """BigBirdForSequenceClassification""",
       """BigBirdForTokenClassification""",
       """BigBirdLayer""",
       """BigBirdModel""",
       """BigBirdPreTrainedModel""",
       """load_tf_weights_in_big_bird""",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """FlaxBigBirdForCausalLM""",
       """FlaxBigBirdForMaskedLM""",
       """FlaxBigBirdForMultipleChoice""",
       """FlaxBigBirdForPreTraining""",
       """FlaxBigBirdForQuestionAnswering""",
       """FlaxBigBirdForSequenceClassification""",
       """FlaxBigBirdForTokenClassification""",
       """FlaxBigBirdModel""",
       """FlaxBigBirdPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird import BigBirdTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird_fast import BigBirdTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_big_bird import (
          BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
          BigBirdForCausalLM,
          BigBirdForMaskedLM,
          BigBirdForMultipleChoice,
          BigBirdForPreTraining,
          BigBirdForQuestionAnswering,
          BigBirdForSequenceClassification,
          BigBirdForTokenClassification,
          BigBirdLayer,
          BigBirdModel,
          BigBirdPreTrainedModel,
          load_tf_weights_in_big_bird,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_big_bird import (
          FlaxBigBirdForCausalLM,
          FlaxBigBirdForMaskedLM,
          FlaxBigBirdForMultipleChoice,
          FlaxBigBirdForPreTraining,
          FlaxBigBirdForQuestionAnswering,
          FlaxBigBirdForSequenceClassification,
          FlaxBigBirdForTokenClassification,
          FlaxBigBirdModel,
          FlaxBigBirdPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase			=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)