from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin


class 			UpperCAmelCase__    (		snake_case		,	snake_case		,	snake_case					):



  """simple docstring"""






  lowerCAmelCase__		:	Dict    		= [R'h\.\d+\.attn\.bias', R'h\.\d+\.attn\.masked_bias']






  @register_to_config
  def __init__(				self:			Optional[int]				,				__lowerCAmelCase:			int				,				__lowerCAmelCase:			int				,				__lowerCAmelCase:			Optional[int] = None				,				__lowerCAmelCase:			int = 50_257				,				__lowerCAmelCase:			int = 1_024				,				__lowerCAmelCase:			int = 768				,				__lowerCAmelCase:			int = 12				,				__lowerCAmelCase:			int = 12				,				__lowerCAmelCase:			Optional[int] = None				,				__lowerCAmelCase:			str = "gelu_new"				,				__lowerCAmelCase:			float = 0.1				,				__lowerCAmelCase:			float = 0.1				,				__lowerCAmelCase:			float = 0.1				,				__lowerCAmelCase:			float = 1E-5				,				__lowerCAmelCase:			float = 0.02				,				__lowerCAmelCase:			bool = True				,				__lowerCAmelCase:			bool = True				,				__lowerCAmelCase:			bool = False				,				__lowerCAmelCase:			bool = False				,				) -> List[Any]:




    '''simple docstring'''




    super().__init__()

    __UpperCAmelCase						    =       prefix_length

    if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
      raise ValueError(
          F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
          F''' `n_embd`: {n_embd} are not equal.'''   )

    __UpperCAmelCase						    =       prefix_inner_dim
    __UpperCAmelCase						    =       prefix_hidden_dim

    __UpperCAmelCase						    =       (
        nn.Linear(self.prefix_inner_dim				,				self.prefix_hidden_dim   )
        if self.prefix_hidden_dim is not None
        else nn.Identity()
    )
    __UpperCAmelCase						    =       (
        nn.Linear(self.prefix_hidden_dim				,				__lowerCAmelCase   ) if self.prefix_hidden_dim is not None else nn.Identity()
    )

    __UpperCAmelCase						    =       GPTaConfig(
        vocab_size=__lowerCAmelCase				,				n_positions=__lowerCAmelCase				,				n_embd=__lowerCAmelCase				,				n_layer=__lowerCAmelCase				,				n_head=__lowerCAmelCase				,				n_inner=__lowerCAmelCase				,				activation_function=__lowerCAmelCase				,				resid_pdrop=__lowerCAmelCase				,				embd_pdrop=__lowerCAmelCase				,				attn_pdrop=__lowerCAmelCase				,				layer_norm_epsilon=__lowerCAmelCase				,				initializer_range=__lowerCAmelCase				,				scale_attn_weights=__lowerCAmelCase				,				use_cache=__lowerCAmelCase				,				scale_attn_by_inverse_layer_idx=__lowerCAmelCase				,				reorder_and_upcast_attn=__lowerCAmelCase				,				)
    __UpperCAmelCase						    =       GPTaLMHeadModel(__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Any				,				__lowerCAmelCase:			torch.Tensor				,				__lowerCAmelCase:			torch.Tensor				,				__lowerCAmelCase:			Optional[torch.Tensor] = None				,				__lowerCAmelCase:			Optional[torch.Tensor] = None				,				) -> Optional[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.transformer.transformer.wte(__lowerCAmelCase   )
    __UpperCAmelCase						    =       self.encode_prefix(__lowerCAmelCase   )
    __UpperCAmelCase						    =       self.decode_prefix(__lowerCAmelCase   )
    __UpperCAmelCase						    =       torch.cat((prefix_embeds, embedding_text)				,				dim=1   )

    if labels is not None:
      __UpperCAmelCase						    =       self.get_dummy_token(input_ids.shape[0]				,				input_ids.device   )
      __UpperCAmelCase						    =       torch.cat((dummy_token, input_ids)				,				dim=1   )
    __UpperCAmelCase						    =       self.transformer(inputs_embeds=__lowerCAmelCase				,				labels=__lowerCAmelCase				,				attention_mask=__lowerCAmelCase   )
    if self.prefix_hidden_dim is not None:
      return out, hidden
    else:
      return out






  def 	_UpperCAmelCase				(				self:			Dict				,				__lowerCAmelCase:			int				,				__lowerCAmelCase:			torch.device   ) -> torch.Tensor:




    '''simple docstring'''




    return torch.zeros(__lowerCAmelCase				,				self.prefix_length				,				dtype=torch.intaa				,				device=__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Dict				,				__lowerCAmelCase:			int   ) -> str:




    '''simple docstring'''




    return self.encode_prefix(__lowerCAmelCase   )






  @torch.no_grad()
  def 	_UpperCAmelCase				(				self:			Dict				,				__lowerCAmelCase:			Union[str, Any]				,				__lowerCAmelCase:			List[Any]				,				__lowerCAmelCase:			Tuple   ) -> List[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       torch.split(__lowerCAmelCase				,				1				,				dim=0   )
    __UpperCAmelCase						    =       []
    __UpperCAmelCase						    =       []
    for feature in features:
      __UpperCAmelCase						    =       self.decode_prefix(feature.to(__lowerCAmelCase   )   )  # back to the clip feature
      # Only support beam search for now
      __UpperCAmelCase    ,		__UpperCAmelCase						    =       self.generate_beam(
          input_embeds=__lowerCAmelCase				,				device=__lowerCAmelCase				,				eos_token_id=__lowerCAmelCase   )
      generated_tokens.append(output_tokens[0]   )
      generated_seq_lengths.append(seq_lengths[0]   )
    __UpperCAmelCase						    =       torch.stack(__lowerCAmelCase   )
    __UpperCAmelCase						    =       torch.stack(__lowerCAmelCase   )
    return generated_tokens, generated_seq_lengths






  @torch.no_grad()
  def 	_UpperCAmelCase				(				self:			Optional[int]				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Any=None				,				__lowerCAmelCase:			int=None				,				__lowerCAmelCase:			int = 5				,				__lowerCAmelCase:			int = 67				,				__lowerCAmelCase:			float = 1.0				,				__lowerCAmelCase:			Optional[int] = None				,				) -> List[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       eos_token_id
    __UpperCAmelCase						    =       None
    __UpperCAmelCase						    =       None
    __UpperCAmelCase						    =       torch.ones(__lowerCAmelCase				,				device=__lowerCAmelCase				,				dtype=torch.int   )
    __UpperCAmelCase						    =       torch.zeros(__lowerCAmelCase				,				device=__lowerCAmelCase				,				dtype=torch.bool   )

    if input_embeds is not None:
      __UpperCAmelCase						    =       input_embeds
    else:
      __UpperCAmelCase						    =       self.transformer.transformer.wte(__lowerCAmelCase   )

    for i in range(__lowerCAmelCase   ):
      __UpperCAmelCase						    =       self.transformer(inputs_embeds=__lowerCAmelCase   )
      __UpperCAmelCase						    =       outputs.logits
      __UpperCAmelCase						    =       logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
      __UpperCAmelCase						    =       logits.softmax(-1   ).log()

      if scores is None:
        __UpperCAmelCase    ,		__UpperCAmelCase						    =       logits.topk(__lowerCAmelCase				,				-1   )
        __UpperCAmelCase						    =       generated.expand(__lowerCAmelCase				,				*generated.shape[1:]   )
        __UpperCAmelCase    ,		__UpperCAmelCase						    =       next_tokens.permute(1				,				0   ), scores.squeeze(0   )
        if tokens is None:
          __UpperCAmelCase						    =       next_tokens
        else:
          __UpperCAmelCase						    =       tokens.expand(__lowerCAmelCase				,				*tokens.shape[1:]   )
          __UpperCAmelCase						    =       torch.cat((tokens, next_tokens)				,				dim=1   )
      else:
        __UpperCAmelCase						    =       -float(np.inf   )
        __UpperCAmelCase						    =       0
        __UpperCAmelCase						    =       scores[:, None] + logits
        seq_lengths[~is_stopped] += 1
        __UpperCAmelCase						    =       scores_sum / seq_lengths[:, None]
        __UpperCAmelCase    ,		__UpperCAmelCase						    =       scores_sum_average.view(-1   ).topk(__lowerCAmelCase				,				-1   )
        __UpperCAmelCase						    =       next_tokens // scores_sum.shape[1]
        __UpperCAmelCase						    =       seq_lengths[next_tokens_source]
        __UpperCAmelCase						    =       next_tokens % scores_sum.shape[1]
        __UpperCAmelCase						    =       next_tokens.unsqueeze(1   )
        __UpperCAmelCase						    =       tokens[next_tokens_source]
        __UpperCAmelCase						    =       torch.cat((tokens, next_tokens)				,				dim=1   )
        __UpperCAmelCase						    =       generated[next_tokens_source]
        __UpperCAmelCase						    =       scores_sum_average * seq_lengths
        __UpperCAmelCase						    =       is_stopped[next_tokens_source]

      __UpperCAmelCase						    =       self.transformer.transformer.wte(next_tokens.squeeze()   ).view(generated.shape[0]				,				1				,				-1   )
      __UpperCAmelCase						    =       torch.cat((generated, next_token_embed)				,				dim=1   )
      __UpperCAmelCase						    =       is_stopped + next_tokens.eq(__lowerCAmelCase   ).squeeze()
      if is_stopped.all():
        break

    __UpperCAmelCase						    =       scores / seq_lengths
    __UpperCAmelCase						    =       scores.argsort(descending=__lowerCAmelCase   )
    # tokens tensors are already padded to max_seq_length
    __UpperCAmelCase						    =       [tokens[i] for i in order]
    __UpperCAmelCase						    =       torch.stack(__lowerCAmelCase				,				dim=0   )
    __UpperCAmelCase						    =       torch.tensor([seq_lengths[i] for i in order]				,				dtype=seq_lengths.dtype   )
    return output_texts, seq_lengths




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a_							=      {
    """configuration_altclip""": [
        """ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AltCLIPConfig""",
        """AltCLIPTextConfig""",
        """AltCLIPVisionConfig""",
    ],
    """processing_altclip""": ["""AltCLIPProcessor"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_							=      [
           """ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """AltCLIPPreTrainedModel""",
           """AltCLIPModel""",
           """AltCLIPTextModel""",
           """AltCLIPVisionModel""",
       ]


if TYPE_CHECKING:
       from .configuration_altclip import (
           ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           AltCLIPConfig,
           AltCLIPTextConfig,
           AltCLIPVisionConfig,
       )
       from .processing_altclip import AltCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_altclip import (
                  ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  AltCLIPModel,
                  AltCLIPPreTrainedModel,
                  AltCLIPTextModel,
                  AltCLIPVisionModel,
              )


else:
       import sys

       a_							=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



