


from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		List[str] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[Any]=0.0 ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		str = "geglu" ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = True ,   _UpperCAmelCase       :		str = "layer_norm" ,   _UpperCAmelCase       :		bool = False ,   )		->  Dict:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						only_cross_attention

							lowercase__	     =						(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
							lowercase__	     =						(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

							if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
									raise ValueError(
									    f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
									    f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''						)

							# Define 3 blocks. Each block has its own normalization layer.
							# 1. Self-Attn
							if self.use_ada_layer_norm:
									lowercase__	     =						AdaLayerNorm(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif self.use_ada_layer_norm_zero:
									lowercase__	     =						AdaLayerNormZero(_UpperCAmelCase ,   _UpperCAmelCase						)
							else:
									lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
							lowercase__	     =						Attention(
							    query_dim=_UpperCAmelCase ,   heads=_UpperCAmelCase ,   dim_head=_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   bias=_UpperCAmelCase ,   cross_attention_dim=cross_attention_dim if only_cross_attention else None ,   upcast_attention=_UpperCAmelCase ,   )

							# 2. Cross-Attn
							if cross_attention_dim is not None or double_self_attention:
									# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
									# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
									# the second cross attention block.
									lowercase__	     =						(
									    AdaLayerNorm(_UpperCAmelCase ,   _UpperCAmelCase						)
									    if self.use_ada_layer_norm
									    else nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
									)
									lowercase__	     =						Attention(
									    query_dim=_UpperCAmelCase ,   cross_attention_dim=cross_attention_dim if not double_self_attention else None ,   heads=_UpperCAmelCase ,   dim_head=_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   bias=_UpperCAmelCase ,   upcast_attention=_UpperCAmelCase ,   )  # is self-attn if encoder_hidden_states is none
							else:
									lowercase__	     =						None
									lowercase__	     =						None

							# 3. Feed-forward
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)
							lowercase__	     =						FeedForward(_UpperCAmelCase ,   dropout=_UpperCAmelCase ,   activation_fn=_UpperCAmelCase ,   final_dropout=_UpperCAmelCase						)

							# let chunk size default to None
							lowercase__	     =						None
							lowercase__	     =						0

					def     lowerCamelCase__	(self       :		List[Any] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		int						)		->  int:




							"""simple docstring"""


							lowercase__	     =						chunk_size
							lowercase__	     =						dim


					def     lowerCamelCase__	(self       :		Optional[int] ,   _UpperCAmelCase       :		torch.FloatTensor ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.FloatTensor] = None ,   _UpperCAmelCase       :		Optional[torch.LongTensor] = None ,   _UpperCAmelCase       :		Dict[str, Any] = None ,   _UpperCAmelCase       :		Optional[torch.LongTensor] = None ,   )		->  List[str]:




							"""simple docstring"""


							if self.use_ada_layer_norm:
									lowercase__	     =						self.norma(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif self.use_ada_layer_norm_zero:
									lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__	     =						self.norma(
									    _UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   hidden_dtype=hidden_states.dtype						)
							else:
									lowercase__	     =						self.norma(_UpperCAmelCase						)

							lowercase__	     =						cross_attention_kwargs if cross_attention_kwargs is not None else {}

							lowercase__	     =						self.attna(
							    _UpperCAmelCase ,   encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None ,   attention_mask=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							if self.use_ada_layer_norm_zero:
									lowercase__	     =						gate_msa.unsqueeze(1						) * attn_output
							lowercase__	     =						attn_output + hidden_states

							# 2. Cross-Attention
							if self.attna is not None:
									lowercase__	     =						(
									    self.norma(_UpperCAmelCase ,   _UpperCAmelCase						) if self.use_ada_layer_norm else self.norma(_UpperCAmelCase						)
									)

									lowercase__	     =						self.attna(
									    _UpperCAmelCase ,   encoder_hidden_states=_UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
									lowercase__	     =						attn_output + hidden_states

							# 3. Feed-forward
							lowercase__	     =						self.norma(_UpperCAmelCase						)

							if self.use_ada_layer_norm_zero:
									lowercase__	     =						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

							if self._chunk_size is not None:
									# "feed_forward_chunk_size" can be used to save memory
									if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
											raise ValueError(
											    f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''						)

									lowercase__	     =						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
									lowercase__	     =						torch.cat(
									    [self.ff(_UpperCAmelCase						) for hid_slice in norm_hidden_states.chunk(_UpperCAmelCase ,   dim=self._chunk_dim						)] ,   dim=self._chunk_dim ,   )
							else:
									lowercase__	     =						self.ff(_UpperCAmelCase						)

							if self.use_ada_layer_norm_zero:
									lowercase__	     =						gate_mlp.unsqueeze(1						) * ff_output

							lowercase__	     =						ff_output + hidden_states

							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] = None ,   _UpperCAmelCase       :		int = 4 ,   _UpperCAmelCase       :		float = 0.0 ,   _UpperCAmelCase       :		str = "geglu" ,   _UpperCAmelCase       :		bool = False ,   )		->  Union[str, Any]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						int(dim * mult						)
							lowercase__	     =						dim_out if dim_out is not None else dim

							if activation_fn == "gelu":
									lowercase__	     =						GELU(_UpperCAmelCase ,   _UpperCAmelCase						)
							if activation_fn == "gelu-approximate":
									lowercase__	     =						GELU(_UpperCAmelCase ,   _UpperCAmelCase ,   approximate="""tanh"""						)
							elif activation_fn == "geglu":
									lowercase__	     =						GEGLU(_UpperCAmelCase ,   _UpperCAmelCase						)
							elif activation_fn == "geglu-approximate":
									lowercase__	     =						ApproximateGELU(_UpperCAmelCase ,   _UpperCAmelCase						)

							lowercase__	     =						nn.ModuleList([]						)
							# project in
							self.net.append(_UpperCAmelCase						)
							# project dropout
							self.net.append(nn.Dropout(_UpperCAmelCase						)						)
							# project out
							self.net.append(nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)						)
							# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
							if final_dropout:
									self.net.append(nn.Dropout(_UpperCAmelCase						)						)


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Optional[Any]						)		->  Tuple:




							"""simple docstring"""


							for module in self.net:
									lowercase__	     =						module(_UpperCAmelCase						)
							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		str = "none"						)		->  int:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)
							lowercase__	     =						approximate

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		List[str]						)		->  List[Any]:




							"""simple docstring"""


							if gate.device.type != "mps":
									return F.gelu(_UpperCAmelCase ,   approximate=self.approximate						)
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa						) ,   approximate=self.approximate						).to(dtype=gate.dtype						)


					def     lowerCamelCase__	(self       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str]						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.proj(_UpperCAmelCase						)
							lowercase__	     =						self.gelu(_UpperCAmelCase						)
							return hidden_states






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  Union[str, Any]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   dim_out * 2						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Any						)		->  Optional[int]:




							"""simple docstring"""


							if gate.device.type != "mps":
									return F.gelu(_UpperCAmelCase						)
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa						)						).to(dtype=gate.dtype						)


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Optional[Any]						)		->  List[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.proj(_UpperCAmelCase						).chunk(2 ,   dim=-1						)
							return hidden_states * self.gelu(_UpperCAmelCase						)






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  str:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   _UpperCAmelCase						)


					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		int						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.proj(_UpperCAmelCase						)
							return x * torch.sigmoid(1.702 * x						)






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Union[str, Any]						)		->  str:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						nn.Embedding(_UpperCAmelCase ,   _UpperCAmelCase						)
							lowercase__	     =						nn.SiLU()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   embedding_dim * 2						)
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase						)


					def     lowerCamelCase__	(self       :		Tuple ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Dict						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.linear(self.silu(self.emb(_UpperCAmelCase						)						)						)
							lowercase__     ,      lowercase__	     =						torch.chunk(_UpperCAmelCase ,   2						)
							lowercase__	     =						self.norm(_UpperCAmelCase						) * (1 + scale) + shift
							return x






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Dict						)		->  List[str]:




							"""simple docstring"""


							super().__init__()

							lowercase__	     =						CombinedTimestepLabelEmbeddings(_UpperCAmelCase ,   _UpperCAmelCase						)

							lowercase__	     =						nn.SiLU()
							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   6 * embedding_dim ,   bias=_UpperCAmelCase						)
							lowercase__	     =						nn.LayerNorm(_UpperCAmelCase ,   elementwise_affine=_UpperCAmelCase ,   eps=1E-6						)


					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int=None						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.linear(self.silu(self.emb(_UpperCAmelCase ,   _UpperCAmelCase ,   hidden_dtype=_UpperCAmelCase						)						)						)
							lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__	     =						emb.chunk(6 ,   dim=1						)
							lowercase__	     =						self.norm(_UpperCAmelCase						) * (1 + scale_msa[:, None]) + shift_msa[:, None]
							return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class 				A       (			nn.Module				):
					'''simple docstring'''






					def __init__(self       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[str] = None ,   _UpperCAmelCase       :		float = 1E-5						)		->  Optional[int]:




							"""simple docstring"""


							super().__init__()
							lowercase__	     =						num_groups
							lowercase__	     =						eps

							if act_fn is None:
									lowercase__	     =						None
							else:
									lowercase__	     =						get_activation(_UpperCAmelCase						)

							lowercase__	     =						nn.Linear(_UpperCAmelCase ,   out_dim * 2						)


					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		List[Any]						)		->  int:




							"""simple docstring"""


							if self.act:
									lowercase__	     =						self.act(_UpperCAmelCase						)
							lowercase__	     =						self.linear(_UpperCAmelCase						)
							lowercase__	     =						emb[:, :, None, None]
							lowercase__     ,      lowercase__	     =						emb.chunk(2 ,   dim=1						)

							lowercase__	     =						F.group_norm(_UpperCAmelCase ,   self.num_groups ,   eps=self.eps						)
							lowercase__	     =						x * (1 + scale) + shift
							return x






'''simple docstring'''

import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						a_		(     snake_case    ,   unittest.TestCase    ):
							UpperCAmelCase			:		int        =				FunnelTokenizer
							UpperCAmelCase			:		Tuple        =				FunnelTokenizerFast
							UpperCAmelCase			:		List[str]        =				True
							UpperCAmelCase			:		Tuple        =				True


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					Optional[Any]:
										super().setUp()

										snake_case:    Optional[Any]      							=[
										    '<unk>',
										    '<cls>',
										    '<sep>',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										snake_case:    Any      							=os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES['vocab_file']			)
										with open(self.vocab_file      ,						'w'      ,						encoding='utf-8'			) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]			)			)


							def    UpperCamelCase     (				self   :						Tuple      ,						**a_   :						str			)			->					Any:
										return FunnelTokenizer.from_pretrained(self.tmpdirname      ,						**a_			)


							def    UpperCamelCase     (				self   :						List[Any]      ,						**a_   :						int			)			->					str:
										return FunnelTokenizerFast.from_pretrained(self.tmpdirname      ,						**a_			)


							def    UpperCamelCase     (				self   :						Any      ,						a_   :						Union[str, Any]			)			->					Union[str, Any]:
										snake_case:    List[str]      							='UNwant\u00E9d,running'
										snake_case:    Dict      							='unwanted, running'
										return input_text, output_text


							def    UpperCamelCase     (				self   :						Optional[int]			)			->					List[Any]:
										snake_case:    Union[str, Any]      							=self.tokenizer_class(self.vocab_file			)

										snake_case:    Dict      							=tokenizer.tokenize('UNwant\u00E9d,running'			)
										self.assertListEqual(a_      ,						['un', '##want', '##ed', ',', 'runn', '##ing']			)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)      ,						[7, 4, 5, 1_0, 8, 9]			)




							def    UpperCamelCase     (				self   :						List[str]			)			->					Dict:
										snake_case:    Optional[Any]      							=self.get_tokenizers(do_lower_case=a_			)
										for tokenizer in tokenizers:
													snake_case:    str      							=tokenizer('UNwant\u00E9d,running'			)
													snake_case:    Union[str, Any]      							=len(inputs['input_ids']			) - 1
													self.assertListEqual(inputs['token_type_ids']      ,						[2] + [0] * sentence_len			)

													snake_case:    List[str]      							=tokenizer('UNwant\u00E9d,running'      ,						'UNwant\u00E9d,running'			)
													self.assertListEqual(inputs['token_type_ids']      ,						[2] + [0] * sentence_len + [1] * sentence_len			)




