






from typing import Callable, Dict, Optional, Tuple

import torch
from torch import nn
from torch.distributions import (
    AffineTransform,
    Distribution,
    Independent,
    NegativeBinomial,
    Normal,
    StudentT,
    TransformedDistribution,
)

class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				def __init__( self     :str  ,     lowerCamelCase__     :Distribution  ,     lowerCamelCase__     :Any=None  ,     lowerCamelCase__     :List[Any]=None  ,     lowerCamelCase__     :List[str]=0     ):
					UpperCamelCase__ :int						=      1.0 if scale is None else scale
					UpperCamelCase__ :Union[str, Any]						=      0.0 if loc is None else loc

					super().__init__(lowerCamelCase__  ,     [AffineTransform(loc=self.loc  ,     scale=self.scale  ,     event_dim=lowerCamelCase__     )]     )







				@property
				def 				__a   ( self     :Any     ):
					return self.base_dist.mean * self.scale + self.loc







				@property
				def 				__a   ( self     :List[Any]     ):
					return self.base_dist.variance * self.scale**2







				@property
				def 				__a   ( self     :Optional[int]     ):
					return self.variance.sqrt()

class        lowerCAmelCase_				(      nn.Module	):







				"""simple docstring"""




				def __init__( self     :int  ,     lowerCamelCase__     :int  ,     lowerCamelCase__     :Dict[str, int]  ,     lowerCamelCase__     :Callable[..., Tuple[torch.Tensor]]  ,     **lowerCamelCase__     :Any     ):
					super().__init__(**lowerCamelCase__     )
					UpperCamelCase__ :Optional[Any]						=      args_dim
					UpperCamelCase__ :Dict						=      nn.ModuleList([nn.Linear(lowerCamelCase__  ,     lowerCamelCase__     ) for dim in args_dim.values()]     )
					UpperCamelCase__ :List[str]						=      domain_map







				def 				__a   ( self     :Tuple  ,     lowerCamelCase__     :torch.Tensor     ):
					UpperCamelCase__ :Optional[Any]						=      [proj(lowerCamelCase__     ) for proj in self.proj]

					return self.domain_map(*lowerCamelCase__     )

class        lowerCAmelCase_				(      nn.Module	):







				"""simple docstring"""




				def __init__( self     :Dict  ,     lowerCamelCase__     :Optional[int]     ):
					super().__init__()
					UpperCamelCase__ :Union[str, Any]						=      function







				def 				__a   ( self     :int  ,     lowerCamelCase__     :Union[str, Any]  ,     *lowerCamelCase__     :Tuple     ):
					return self.function(lowerCamelCase__  ,     *lowerCamelCase__     )

class        lowerCAmelCase_				:







				"""simple docstring"""




				_snake_case				:   type
				_snake_case				:   int
				_snake_case				:   Dict[str, int]







				def __init__( self     :Tuple  ,     lowerCamelCase__     :int = 1     ):
					UpperCamelCase__ :str						=      dim
					UpperCamelCase__ :List[str]						=      {k: dim * self.args_dim[k] for k in self.args_dim}







				def 				__a   ( self     :Union[str, Any]  ,     lowerCamelCase__     :Union[str, Any]     ):
					if self.dim == 1:
						return self.distribution_class(*lowerCamelCase__     )
					else:
						return Independent(self.distribution_class(*lowerCamelCase__     )  ,     1     )







				def 				__a   ( self     :Optional[int]  ,     lowerCamelCase__     :Optional[Any]  ,     lowerCamelCase__     :Optional[torch.Tensor] = None  ,     lowerCamelCase__     :Optional[torch.Tensor] = None  ,     ):
					UpperCamelCase__ :Union[str, Any]						=      self._base_distribution(lowerCamelCase__     )
					if loc is None and scale is None:
						return distr
					else:
						return AffineTransformed(lowerCamelCase__  ,     loc=lowerCamelCase__  ,     scale=lowerCamelCase__  ,     event_dim=self.event_dim     )







				@property
				def 				__a   ( self     :Optional[Any]     ):
					return () if self.dim == 1 else (self.dim,)







				@property
				def 				__a   ( self     :List[str]     ):
					return len(self.event_shape     )







				@property
				def 				__a   ( self     :Any     ):
					return 0.0







				def 				__a   ( self     :Any  ,     lowerCamelCase__     :int     ):
					return ParameterProjection(
					    in_features=lowerCamelCase__  ,     args_dim=self.args_dim  ,     domain_map=LambdaLayer(self.domain_map     )  ,     )







				def 				__a   ( self     :Union[str, Any]  ,     *lowerCamelCase__     :torch.Tensor     ):
					raise NotImplementedError()







				@staticmethod
				def 				__a   ( lowerCamelCase__     :torch.Tensor     ):
					return (x + torch.sqrt(torch.square(lowerCamelCase__     ) + 4.0     )) / 2.0

class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				_snake_case				:   Dict[str, int]        =			{"df": 1, "loc": 1, "scale": 1}
				_snake_case				:   type        =			StudentT







				@classmethod
				def 				__a   ( cls     :List[str]  ,     lowerCamelCase__     :torch.Tensor  ,     lowerCamelCase__     :torch.Tensor  ,     lowerCamelCase__     :torch.Tensor     ):
					UpperCamelCase__ :Dict						=      cls.squareplus(lowerCamelCase__     ).clamp_min(torch.finfo(scale.dtype     ).eps     )
					UpperCamelCase__ :Dict						=      2.0 + cls.squareplus(lowerCamelCase__     )
					return df.squeeze(-1     ), loc.squeeze(-1     ), scale.squeeze(-1     )

class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				_snake_case				:   Dict[str, int]        =			{"loc": 1, "scale": 1}
				_snake_case				:   type        =			Normal







				@classmethod
				def 				__a   ( cls     :Union[str, Any]  ,     lowerCamelCase__     :torch.Tensor  ,     lowerCamelCase__     :torch.Tensor     ):
					UpperCamelCase__ :Tuple						=      cls.squareplus(lowerCamelCase__     ).clamp_min(torch.finfo(scale.dtype     ).eps     )
					return loc.squeeze(-1     ), scale.squeeze(-1     )

class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				_snake_case				:   Dict[str, int]        =			{"total_count": 1, "logits": 1}
				_snake_case				:   type        =			NegativeBinomial







				@classmethod
				def 				__a   ( cls     :Tuple  ,     lowerCamelCase__     :torch.Tensor  ,     lowerCamelCase__     :torch.Tensor     ):
					UpperCamelCase__ :Union[str, Any]						=      cls.squareplus(lowerCamelCase__     )
					return total_count.squeeze(-1     ), logits.squeeze(-1     )







				def 				__a   ( self     :Optional[Any]  ,     lowerCamelCase__     :int     ):
					UpperCamelCase__				,		UpperCamelCase__ :Optional[int]						=      distr_args
					if self.dim == 1:
						return self.distribution_class(total_count=lowerCamelCase__  ,     logits=lowerCamelCase__     )
					else:
						return Independent(self.distribution_class(total_count=lowerCamelCase__  ,     logits=lowerCamelCase__     )  ,     1     )







				def 				__a   ( self     :Union[str, Any]  ,     lowerCamelCase__     :List[Any]  ,     lowerCamelCase__     :Optional[torch.Tensor] = None  ,     lowerCamelCase__     :Optional[torch.Tensor] = None     ):
					UpperCamelCase__				,		UpperCamelCase__ :int						=      distr_args

					if scale is not None:
						# See scaling property of Gamma.
						logits += scale.log()

					return self._base_distribution((total_count, logits)     )


from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__												=						logging.get_logger(__name__)

a__												=						{
    '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/config.json''',
    '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/config.json''',
    '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/config.json''',
    '''funnel-transformer/medium-base''': '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/config.json''',
    '''funnel-transformer/intermediate''': (
        '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/config.json'''
    ),
    '''funnel-transformer/intermediate-base''': (
        '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/config.json'''
    ),
    '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/config.json''',
    '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/config.json''',
    '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/config.json''',
    '''funnel-transformer/xlarge-base''': '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/config.json''',
}

class 							UpperCAmelCase_						(     __lowercase  ):
							"""simple docstring"""







							UpperCAmelCase__			:		Optional[int]             =       "funnel"
							UpperCAmelCase__			:		Tuple             =       {
							    "hidden_size": "d_model",
							    "num_attention_heads": "n_head",
							}


							def __init__(     self			,				_a=3_0_5_2_2			,				_a=[4, 4, 4]			,				_a=None			,				_a=2			,				_a=7_6_8			,				_a=1_2			,				_a=6_4			,				_a=3_0_7_2			,				_a="gelu_new"			,				_a=0.1			,				_a=0.1			,				_a=0.0			,				_a=0.1			,				_a=None			,				_a=1e-9			,				_a="mean"			,				_a="relative_shift"			,				_a=True			,				_a=True			,				_a=True			,				**_a			,				)       ->       List[Any]:
												_a      :       Optional[int]   =						vocab_size
												_a      :       Dict   =						block_sizes
												_a      :       Optional[int]   =						[1] * len(_a			) if block_repeats is None else block_repeats
												assert len(_a			) == len(
												    self.block_repeats			), "`block_sizes` and `block_repeats` should have the same length."
												_a      :       int   =						num_decoder_layers
												_a      :       List[str]   =						d_model
												_a      :       Optional[Any]   =						n_head
												_a      :       Tuple   =						d_head
												_a      :       Dict   =						d_inner
												_a      :       List[str]   =						hidden_act
												_a      :       int   =						hidden_dropout
												_a      :       Union[str, Any]   =						attention_dropout
												_a      :       Tuple   =						activation_dropout
												_a      :       Optional[Any]   =						initializer_range
												_a      :       Dict   =						initializer_std
												_a      :       Union[str, Any]   =						layer_norm_eps
												assert pooling_type in [
												    "mean",
												    "max",
												], F"""Got {pooling_type} for `pooling_type` but only 'mean' and 'max' are supported."""
												_a      :       Any   =						pooling_type
												assert attention_type in [
												    "relative_shift",
												    "factorized",
												], F"""Got {attention_type} for `attention_type` but only 'relative_shift' and 'factorized' are supported."""
												_a      :       Optional[Any]   =						attention_type
												_a      :       int   =						separate_cls
												_a      :       Tuple   =						truncate_seq
												_a      :       List[Any]   =						pool_q_only

												super().__init__(**_a			)


							@property
							def 							__lowercase							(     self			)       ->       Tuple:
												return sum(self.block_sizes			)


							@num_hidden_layers.setter
							def 							__lowercase							(     self			,				_a			)       ->       List[str]:
												raise NotImplementedError(
												    '''This model does not support the setting of `num_hidden_layers`. Please set `block_sizes`.'''			)


							@property
							def 							__lowercase							(     self			)       ->       Optional[int]:
												return len(self.block_sizes			)




							@num_blocks.setter
							def 							__lowercase							(     self			,				_a			)       ->       Dict:
												raise NotImplementedError('''This model does not support the setting of `num_blocks`. Please set `block_sizes`.'''			)



