



from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin
@dataclass
class  _snake_case	(    lowercase__):
				UpperCamelCase__ :    Optional[int]   						=4_2
class  _snake_case	(    lowercase__			,		lowercase__):




				@register_to_config
				def __init__(  self    :  Tuple,    __lowercase    :  int = 16,    __lowercase    :  Tuple = 88,    __lowercase    :  str = None,    __lowercase    :  Optional[int] = None,    __lowercase    :  Optional[Any] = 1,    __lowercase    :  Optional[Any] = 0.0,    __lowercase    :  int = 32,    __lowercase    :  Optional[Any] = None,    __lowercase    :  Dict = False,    __lowercase    :  Union[str, Any] = None,    __lowercase    :  List[Any] = None,    __lowercase    :  Any = None,    __lowercase    :  int = "geglu",    __lowercase    :  Tuple = None,    __lowercase    :  Optional[int] = False,    __lowercase    :  int = False,    __lowercase    :  int = False,    __lowercase    :  Union[str, Any] = "layer_norm",    __lowercase    :  Tuple = True,    ):
							super().__init__()
							lowercase__         = use_linear_projection
							lowercase__         = num_attention_heads
							lowercase__         = attention_head_dim
							lowercase__         = num_attention_heads * attention_head_dim

							# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
							# Define whether input is continuous or discrete depending on configuration
							lowercase__         = (in_channels is not None) and (patch_size is None)
							lowercase__         = num_vector_embeds is not None
							lowercase__         = in_channels is not None and patch_size is not None

							if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
										lowercase__         = (
										    F'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
										    " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
										    " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
										    " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
										    " would be very nice if you could open a Pull request for the `transformer/config.json` file"
										)
										deprecate("norm_type!=num_embeds_ada_norm",    "1.0.0",    UpperCAmelCase_,    standard_warn=UpperCAmelCase_  )
										lowercase__         = "ada_norm"

							if self.is_input_continuous and self.is_input_vectorized:
										raise ValueError(
										    F'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
										    " sure that either `in_channels` or `num_vector_embeds` is None."  )
							elif self.is_input_vectorized and self.is_input_patches:
										raise ValueError(
										    F'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
										    " sure that either `num_vector_embeds` or `num_patches` is None."  )
							elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
										raise ValueError(
										    F'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
										    F''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''  )

							# 2. Define input layers
							if self.is_input_continuous:
										lowercase__         = in_channels

										lowercase__         = torch.nn.GroupNorm(num_groups=UpperCAmelCase_,    num_channels=UpperCAmelCase_,    eps=1e-6,    affine=UpperCAmelCase_  )
										if use_linear_projection:
													lowercase__         = nn.Linear(UpperCAmelCase_,    UpperCAmelCase_  )
										else:
													lowercase__         = nn.Convad(UpperCAmelCase_,    UpperCAmelCase_,    kernel_size=1,    stride=1,    padding=0  )
							elif self.is_input_vectorized:
										assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
										assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

										lowercase__         = sample_size
										lowercase__         = sample_size
										lowercase__         = num_vector_embeds
										lowercase__         = self.height * self.width

										lowercase__         = ImagePositionalEmbeddings(
										    num_embed=UpperCAmelCase_,    embed_dim=UpperCAmelCase_,    height=self.height,    width=self.width  )
							elif self.is_input_patches:
										assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

										lowercase__         = sample_size
										lowercase__         = sample_size

										lowercase__         = patch_size
										lowercase__         = PatchEmbed(
										    height=UpperCAmelCase_,    width=UpperCAmelCase_,    patch_size=UpperCAmelCase_,    in_channels=UpperCAmelCase_,    embed_dim=UpperCAmelCase_,    )

							# 3. Define transformers blocks
							lowercase__         = nn.ModuleList(
							    [
							        BasicTransformerBlock(
							            UpperCAmelCase_,    UpperCAmelCase_,    UpperCAmelCase_,    dropout=UpperCAmelCase_,    cross_attention_dim=UpperCAmelCase_,    activation_fn=UpperCAmelCase_,    num_embeds_ada_norm=UpperCAmelCase_,    attention_bias=UpperCAmelCase_,    only_cross_attention=UpperCAmelCase_,    upcast_attention=UpperCAmelCase_,    norm_type=UpperCAmelCase_,    norm_elementwise_affine=UpperCAmelCase_,    )
							        for d in range(UpperCAmelCase_  )
							    ]  )

							# 4. Define output layers
							lowercase__         = in_channels if out_channels is None else out_channels
							if self.is_input_continuous:
										# TODO: should use out_channels for continuous projections
										if use_linear_projection:
													lowercase__         = nn.Linear(UpperCAmelCase_,    UpperCAmelCase_  )
										else:
													lowercase__         = nn.Convad(UpperCAmelCase_,    UpperCAmelCase_,    kernel_size=1,    stride=1,    padding=0  )
							elif self.is_input_vectorized:
										lowercase__         = nn.LayerNorm(UpperCAmelCase_  )
										lowercase__         = nn.Linear(UpperCAmelCase_,    self.num_vector_embeds - 1  )
							elif self.is_input_patches:
										lowercase__         = nn.LayerNorm(UpperCAmelCase_,    elementwise_affine=UpperCAmelCase_,    eps=1e-6  )
										lowercase__         = nn.Linear(UpperCAmelCase_,    2 * inner_dim  )
										lowercase__         = nn.Linear(UpperCAmelCase_,    patch_size * patch_size * self.out_channels  )




				def 							A__					(  self    :  Union[str, Any],    __lowercase    :  List[Any],    __lowercase    :  List[Any] = None,    __lowercase    :  Dict = None,    __lowercase    :  int = None,    __lowercase    :  Tuple = None,    __lowercase    :  Tuple = None,    __lowercase    :  Optional[int] = None,    __lowercase    :  Tuple = True,    ):
							# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
							#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
							#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
							# expects mask of shape:
							#   [batch, key_tokens]
							# adds singleton query_tokens dimension:
							#   [batch,                    1, key_tokens]
							# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
							#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
							#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
							if attention_mask is not None and attention_mask.ndim == 2:
										# assume that mask is expressed as:
										#   (1 = keep,      0 = discard)
										# convert mask into a bias that can be added to attention scores:
										#       (keep = +0,     discard = -10000.0)
										lowercase__         = (1 - attention_mask.to(hidden_states.dtype  )) * -1_0000.0
										lowercase__         = attention_mask.unsqueeze(1  )

							# convert encoder_attention_mask to a bias the same way we do for attention_mask
							if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
										lowercase__         = (1 - encoder_attention_mask.to(hidden_states.dtype  )) * -1_0000.0
										lowercase__         = encoder_attention_mask.unsqueeze(1  )

							# 1. Input
							if self.is_input_continuous:
										lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__         = hidden_states.shape
										lowercase__         = hidden_states

										lowercase__         = self.norm(UpperCAmelCase_  )
										if not self.use_linear_projection:
													lowercase__         = self.proj_in(UpperCAmelCase_  )
													lowercase__         = hidden_states.shape[1]
													lowercase__         = hidden_states.permute(0,    2,    3,    1  ).reshape(UpperCAmelCase_,    height * width,    UpperCAmelCase_  )
										else:
													lowercase__         = hidden_states.shape[1]
													lowercase__         = hidden_states.permute(0,    2,    3,    1  ).reshape(UpperCAmelCase_,    height * width,    UpperCAmelCase_  )
													lowercase__         = self.proj_in(UpperCAmelCase_  )
							elif self.is_input_vectorized:
										lowercase__         = self.latent_image_embedding(UpperCAmelCase_  )
							elif self.is_input_patches:
										lowercase__         = self.pos_embed(UpperCAmelCase_  )

							# 2. Blocks
							for block in self.transformer_blocks:
										lowercase__         = block(
										    UpperCAmelCase_,    attention_mask=UpperCAmelCase_,    encoder_hidden_states=UpperCAmelCase_,    encoder_attention_mask=UpperCAmelCase_,    timestep=UpperCAmelCase_,    cross_attention_kwargs=UpperCAmelCase_,    class_labels=UpperCAmelCase_,    )

							# 3. Output
							if self.is_input_continuous:
										if not self.use_linear_projection:
													lowercase__         = hidden_states.reshape(UpperCAmelCase_,    UpperCAmelCase_,    UpperCAmelCase_,    UpperCAmelCase_  ).permute(0,    3,    1,    2  ).contiguous()
													lowercase__         = self.proj_out(UpperCAmelCase_  )
										else:
													lowercase__         = self.proj_out(UpperCAmelCase_  )
													lowercase__         = hidden_states.reshape(UpperCAmelCase_,    UpperCAmelCase_,    UpperCAmelCase_,    UpperCAmelCase_  ).permute(0,    3,    1,    2  ).contiguous()

										lowercase__         = hidden_states + residual
							elif self.is_input_vectorized:
										lowercase__         = self.norm_out(UpperCAmelCase_  )
										lowercase__         = self.out(UpperCAmelCase_  )
										# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
										lowercase__         = logits.permute(0,    2,    1  )

										# log(p(x_0))
										lowercase__         = F.log_softmax(logits.double(),    dim=1  ).float()
							elif self.is_input_patches:
										# TODO: cleanup!
										lowercase__         = self.transformer_blocks[0].norma.emb(
										    UpperCAmelCase_,    UpperCAmelCase_,    hidden_dtype=hidden_states.dtype  )
										lowercase__     ,				lowercase__         = self.proj_out_a(F.silu(UpperCAmelCase_  )  ).chunk(2,    dim=1  )
										lowercase__         = self.norm_out(UpperCAmelCase_  ) * (1 + scale[:, None]) + shift[:, None]
										lowercase__         = self.proj_out_a(UpperCAmelCase_  )

										# unpatchify
										lowercase__         = lowercase__         = int(hidden_states.shape[1] ** 0.5  )
										lowercase__         = hidden_states.reshape(
										    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)  )
										lowercase__         = torch.einsum("nhwpqc->nchpwq",    UpperCAmelCase_  )
										lowercase__         = hidden_states.reshape(
										    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)  )

							if not return_dict:
										return (output,)

							return TransformeraDModelOutput(sample=UpperCAmelCase_  )











from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCAmelCase__  :	str     =logging.get_logger(__name__)

UpperCAmelCase__  :	Optional[int]     ={
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}

class 					__A   (  a			):
				__A            = """gpt2"""
				__A            = ["""past_key_values"""]
				__A            = {
				    """hidden_size""": """n_embd""",
				    """max_position_embeddings""": """n_positions""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}


				def __init__(						self      ,       UpperCAmelCase_=50257      ,       UpperCAmelCase_=1024      ,       UpperCAmelCase_=768      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_="gelu_new"      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=1E-5      ,       UpperCAmelCase_=0.0_2      ,       UpperCAmelCase_="cls_index"      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=False      ,       UpperCAmelCase_=False      ,       **UpperCAmelCase_      ,       ):
								lowerCamelCase		       =vocab_size
								lowerCamelCase		       =n_positions
								lowerCamelCase		       =n_embd
								lowerCamelCase		       =n_layer
								lowerCamelCase		       =n_head
								lowerCamelCase		       =n_inner
								lowerCamelCase		       =activation_function
								lowerCamelCase		       =resid_pdrop
								lowerCamelCase		       =embd_pdrop
								lowerCamelCase		       =attn_pdrop
								lowerCamelCase		       =layer_norm_epsilon
								lowerCamelCase		       =initializer_range
								lowerCamelCase		       =summary_type
								lowerCamelCase		       =summary_use_proj
								lowerCamelCase		       =summary_activation
								lowerCamelCase		       =summary_first_dropout
								lowerCamelCase		       =summary_proj_to_labels
								lowerCamelCase		       =scale_attn_weights
								lowerCamelCase		       =use_cache
								lowerCamelCase		       =scale_attn_by_inverse_layer_idx
								lowerCamelCase		       =reorder_and_upcast_attn

								lowerCamelCase		       =bos_token_id
								lowerCamelCase		       =eos_token_id

								super().__init__(bos_token_id=UpperCAmelCase_      ,       eos_token_id=UpperCAmelCase_      ,       **UpperCAmelCase_							)




class 					__A   (  a			):


				def __init__(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = "default"      ,       UpperCAmelCase_ = None      ,       UpperCAmelCase_ = False      ,       ):
								super().__init__(UpperCAmelCase_      ,       task=UpperCAmelCase_      ,       patching_specs=UpperCAmelCase_      ,       use_past=UpperCAmelCase_							)
								if not getattr(self._config      ,       """pad_token_id"""      ,       UpperCAmelCase_							):
												# TODO: how to do that better?
												lowerCamelCase		       =0


				@property
				def _snake_case				(						self							):
								lowerCamelCase		       =OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
								if self.use_past:
												self.fill_with_past_key_values_(UpperCAmelCase_      ,       direction="""inputs"""							)
												lowerCamelCase		       ={0: """batch""", 1: """past_sequence + sequence"""}
								else:
												lowerCamelCase		       ={0: """batch""", 1: """sequence"""}

								return common_inputs


				@property
				def _snake_case				(						self							):
								return self._config.n_layer


				@property
				def _snake_case				(						self							):
								return self._config.n_head


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = False      ,       UpperCAmelCase_ = None      ,       ):
								lowerCamelCase		       =super(UpperCAmelCase_      ,       self							).generate_dummy_inputs(
								    UpperCAmelCase_      ,       batch_size=UpperCAmelCase_      ,       seq_length=UpperCAmelCase_      ,       is_pair=UpperCAmelCase_      ,       framework=UpperCAmelCase_							)

								# We need to order the input in the way they appears in the forward()
								lowerCamelCase		       =OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

								# Need to add the past_keys
								if self.use_past:
												if not is_torch_available():
																raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
												else:
																import torch

																lowerCamelCase       ,     lowerCamelCase		       =common_inputs["""input_ids"""].shape
																# Not using the same length for past_key_values
																lowerCamelCase		       =seqlen + 2
																lowerCamelCase		       =(
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																lowerCamelCase		       =[
																    (torch.zeros(UpperCAmelCase_							), torch.zeros(UpperCAmelCase_							)) for _ in range(self.num_layers							)
																]

								lowerCamelCase		       =common_inputs["""attention_mask"""]
								if self.use_past:
												lowerCamelCase		       =ordered_inputs["""attention_mask"""].dtype
												lowerCamelCase		       =torch.cat(
												    [ordered_inputs["""attention_mask"""], torch.ones(UpperCAmelCase_      ,       UpperCAmelCase_      ,       dtype=UpperCAmelCase_							)]      ,       dim=1							)

								return ordered_inputs


				@property
				def _snake_case				(						self							):
								return 13





