


'''simple docstring'''

import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__lowercase:						Dict  		=   logging.get_logger(__name__)







class        UpperCAmelCase       (				nn.Module):

 def __init__(       self       :		Optional[Any]    ):





    """simple docstring"""



    super().__init__()
    if version.parse(torch.__version__    ) < version.parse("1.12.0"    ):
       raise ImportError(
           f'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
           "PytorchGELUTanh. Please upgrade torch."    )

 def 						lowercase_  (       self       :		Union[str, Any],    a_       :		Tuple    ):





    """simple docstring"""



    return nn.functional.gelu(lowerCAmelCase__,    approximate="tanh"    )







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		Optional[Any],    a_       :		Tuple    ):





    """simple docstring"""



    return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi    ) * (input + 0.044_715 * torch.pow(lowerCAmelCase__,    3.0    ))    ))







class        UpperCAmelCase       (				nn.Module):

 def __init__(       self       :		Any,    a_       :		Tuple = False    ):





    """simple docstring"""



    super().__init__()
    if use_gelu_python:
       UpperCamelCase__         =  self._gelu_python
    else:
       UpperCamelCase__         =  nn.functional.gelu

 def 						lowercase_  (       self       :		List[str],    a_       :		List[Any]    ):





    """simple docstring"""



    return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0    )    ))

 def 						lowercase_  (       self       :		List[str],    a_       :		Optional[Any]    ):





    """simple docstring"""



    return self.act(lowerCAmelCase__    )







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		Any,    a_       :		Union[str, Any]    ):





    """simple docstring"""



    return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)    ))







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		Tuple,    a_       :		Optional[Any]    ):





    """simple docstring"""



    return input * torch.sigmoid(1.702 * input    )







class        UpperCAmelCase       (				nn.Module):

 def __init__(       self       :		Union[str, Any],    a_       :		List[str],    a_       :		Optional[int]    ):





    """simple docstring"""



    if min > max:
       raise ValueError(f'min should be < max (got min: {min}, max: {max})'    )

    super().__init__()
    UpperCamelCase__         =  min
    UpperCamelCase__         =  max

 def 						lowercase_  (       self       :		List[str],    a_       :		Union[str, Any]    ):





    """simple docstring"""



    return torch.clip(gelu(lowerCAmelCase__    ),    self.min,    self.max    )







class        UpperCAmelCase       (				nn.Module):

 def __init__(       self       :		List[str]    ):





    """simple docstring"""



    super().__init__()
    UpperCamelCase__         =  math.sqrt(2 / math.pi    )

 def 						lowercase_  (       self       :		Tuple,    a_       :		str    ):





    """simple docstring"""



    return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(lowerCAmelCase__,    3    ))    ))







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		List[Any],    a_       :		List[Any]    ):





    """simple docstring"""



    return nn.functional.silu(lowerCAmelCase__    )







class        UpperCAmelCase       (				nn.Module):

 def __init__(       self       :		Any    ):





    """simple docstring"""



    super().__init__()
    if version.parse(torch.__version__    ) < version.parse("1.9.0"    ):
       UpperCamelCase__         =  self._mish_python
    else:
       UpperCamelCase__         =  nn.functional.mish

 def 						lowercase_  (       self       :		int,    a_       :		List[Any]    ):





    """simple docstring"""



    return input * torch.tanh(nn.functional.softplus(lowerCAmelCase__    )    )

 def 						lowercase_  (       self       :		Optional[Any],    a_       :		List[Any]    ):





    """simple docstring"""



    return self.act(lowerCAmelCase__    )







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		Any,    a_       :		Optional[Any]    ):





    """simple docstring"""



    return input







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		Any,    a_       :		Union[str, Any],    a_       :		Union[str, Any]=0.707_107,    a_       :		Union[str, Any]=0.282_095    ):





    """simple docstring"""



    UpperCamelCase__         =  (input - mu).div(sigma * math.sqrt(2.0    )    )
    return 0.5 * (1.0 + torch.erf(lowerCAmelCase__    ))







class        UpperCAmelCase       (				nn.Module):

 def 						lowercase_  (       self       :		List[str],    a_       :		str    ):





    """simple docstring"""



    UpperCamelCase__         =  nn.functional.relu(lowerCAmelCase__    )
    UpperCamelCase__         =  torch.square(lowerCAmelCase__    )
    return squared







class        UpperCAmelCase       (				__SCREAMING_SNAKE_CASE):

 def __getitem__(       self       :		Dict,    a_       :		List[Any]    ):





    """simple docstring"""



    UpperCamelCase__         =  super().__getitem__(lowerCAmelCase__    )
    UpperCamelCase__         =  content if isinstance(lowerCAmelCase__,    lowerCAmelCase__    ) else (content, {})
    return cls(**lowerCAmelCase__    )







__lowercase:						List[str]  		=   {
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
__lowercase:						Any  		=   ClassInstantier(ACTaCLS)



def 				SCREAMING_SNAKE_CASE__(						_UpperCamelCase					: Optional[int]      )						->    str:





   '''simple docstring'''







   if activation_string in ACTaFN:
      return ACTaFN[activation_string]
   else:
      raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()      )}'      )


# For backwards compatibility with: from activations import gelu_python
__lowercase:						Optional[Any]  		=   get_activation("gelu_python")
__lowercase:						List[Any]  		=   get_activation("gelu_new")
__lowercase:						str  		=   get_activation("gelu")
__lowercase:						Optional[int]  		=   get_activation("gelu_fast")
__lowercase:						Tuple  		=   get_activation("quick_gelu")
__lowercase:						Optional[int]  		=   get_activation("silu")
__lowercase:						int  		=   get_activation("mish")






__lowercase:						List[str]  		=   get_activation("linear")



from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__       :	Any     				=logging.get_logger(__name__)

lowerCAmelCase__       :	Union[str, Any]     				={
    'google/reformer-crime-and-punishment': (
        'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json'
    ),
    'google/reformer-enwik8': 'https://huggingface.co/google/reformer-enwik8/resolve/main/config.json',
}







class      __lowercase  (__SCREAMING_SNAKE_CASE   ):





							"""simple docstring"""







							_UpperCAmelCase     =		"""reformer"""
							_UpperCAmelCase     =		["""past_buckets_states"""]
							_UpperCAmelCase     =		{}






							def __init__(	self       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=["local", "lsh", "local", "lsh", "local", "lsh"]       ,		lowerCAmelCase__=1.0       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__=[6_4, 6_4]       ,		lowerCAmelCase__=[6_4, 1_9_2]       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=2       ,		lowerCAmelCase__=5_1_2       ,		lowerCAmelCase__=None       ,		lowerCAmelCase__="relu"       ,		lowerCAmelCase__=0.05       ,		lowerCAmelCase__=2_5_6       ,		lowerCAmelCase__=0.02       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=1E-12       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=0.05       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=0.0       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=4_0_9_6       ,		lowerCAmelCase__=1_2       ,		lowerCAmelCase__=None       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=3_2_0       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__=None       ,		**lowerCAmelCase__       ,		):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= hash_seed
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= vocab_size
										SCREAMING_SNAKE_CASE_						:			Any       						= attention_head_size
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= hidden_size
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= num_attention_heads
										SCREAMING_SNAKE_CASE_						:			Any       						= num_hashes
										SCREAMING_SNAKE_CASE_						:			int       						= len(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			str       						= tuple(lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else num_buckets
										SCREAMING_SNAKE_CASE_						:			int       						= lsh_attn_chunk_length
										SCREAMING_SNAKE_CASE_						:			List[str]       						= local_attn_chunk_length
										SCREAMING_SNAKE_CASE_						:			Tuple       						= lsh_num_chunks_after
										SCREAMING_SNAKE_CASE_						:			str       						= lsh_num_chunks_before
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= local_num_chunks_after
										SCREAMING_SNAKE_CASE_						:			Any       						= local_num_chunks_before
										SCREAMING_SNAKE_CASE_						:			Any       						= hidden_act
										SCREAMING_SNAKE_CASE_						:			Dict       						= feed_forward_size
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= hidden_dropout_prob
										SCREAMING_SNAKE_CASE_						:			Tuple       						= lsh_attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE_						:			Any       						= local_attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE_						:			str       						= max_position_embeddings
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= initializer_range
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= layer_norm_eps
										SCREAMING_SNAKE_CASE_						:			List[str]       						= axial_pos_embds
										SCREAMING_SNAKE_CASE_						:			List[str]       						= tuple(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Tuple       						= tuple(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= axial_norm_std
										SCREAMING_SNAKE_CASE_						:			Any       						= chunk_size_lm_head
										SCREAMING_SNAKE_CASE_						:			int       						= attn_layers
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= use_cache
										SCREAMING_SNAKE_CASE_						:			Tuple       						= classifier_dropout
										super().__init__(
										    pad_token_id=lowerCAmelCase__       ,		eos_token_id=lowerCAmelCase__       ,		is_decoder=lowerCAmelCase__       ,		tie_word_embeddings=lowerCAmelCase__       ,		**lowerCAmelCase__       ,		)





