





"""simple docstring"""
from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class       a							(   a__   , a__   , a__  ):
 snake_case__             =       [r'''h\.\d+\.attn\.bias''', r'''h\.\d+\.attn\.masked_bias''']



 @register_to_config
 def __init__(     self				,       _snake_case				,       _snake_case				,       _snake_case = None				,       _snake_case = 5_02_57				,       _snake_case = 10_24				,       _snake_case = 7_68				,       _snake_case = 12				,       _snake_case = 12				,       _snake_case = None				,       _snake_case = "gelu_new"				,       _snake_case = 0.1				,       _snake_case = 0.1				,       _snake_case = 0.1				,       _snake_case = 1E-5				,       _snake_case = 0.02				,       _snake_case = True				,       _snake_case = True				,       _snake_case = False				,       _snake_case = False				,       ):







  """simple docstring"""






  super().__init__()

  lowerCAmelCase    				= prefix_length

  if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
   raise ValueError(
       F'`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'
       F' `n_embd`: {n_embd} are not equal.'     )

  lowerCAmelCase    				= prefix_inner_dim
  lowerCAmelCase    				= prefix_hidden_dim

  lowerCAmelCase    				= (
      nn.Linear(self.prefix_inner_dim				,       self.prefix_hidden_dim     )
      if self.prefix_hidden_dim is not None
      else nn.Identity()
  )
  lowerCAmelCase    				= (
      nn.Linear(self.prefix_hidden_dim				,       _snake_case     ) if self.prefix_hidden_dim is not None else nn.Identity()
  )

  lowerCAmelCase    				= GPTaConfig(
      vocab_size=_snake_case				,       n_positions=_snake_case				,       n_embd=_snake_case				,       n_layer=_snake_case				,       n_head=_snake_case				,       n_inner=_snake_case				,       activation_function=_snake_case				,       resid_pdrop=_snake_case				,       embd_pdrop=_snake_case				,       attn_pdrop=_snake_case				,       layer_norm_epsilon=_snake_case				,       initializer_range=_snake_case				,       scale_attn_weights=_snake_case				,       use_cache=_snake_case				,       scale_attn_by_inverse_layer_idx=_snake_case				,       reorder_and_upcast_attn=_snake_case				,       )
  lowerCAmelCase    				= GPTaLMHeadModel(_snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case = None				,       _snake_case = None				,       ):







  """simple docstring"""






  lowerCAmelCase    				= self.transformer.transformer.wte(_snake_case     )
  lowerCAmelCase    				= self.encode_prefix(_snake_case     )
  lowerCAmelCase    				= self.decode_prefix(_snake_case     )
  lowerCAmelCase    				= torch.cat((prefix_embeds, embedding_text)				,       dim=1     )

  if labels is not None:
   lowerCAmelCase    				= self.get_dummy_token(input_ids.shape[0]				,       input_ids.device     )
   lowerCAmelCase    				= torch.cat((dummy_token, input_ids)				,       dim=1     )
  lowerCAmelCase    				= self.transformer(inputs_embeds=_snake_case				,       labels=_snake_case				,       attention_mask=_snake_case     )
  if self.prefix_hidden_dim is not None:
   return out, hidden
  else:
   return out



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  return torch.zeros(_snake_case				,       self.prefix_length				,       dtype=torch.intaa				,       device=_snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  return self.encode_prefix(_snake_case     )



 @torch.no_grad()
 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= torch.split(_snake_case				,       1				,       dim=0     )
  lowerCAmelCase    				= []
  lowerCAmelCase    				= []
  for feature in features:
   lowerCAmelCase    				= self.decode_prefix(feature.to(_snake_case     )     )  # back to the clip feature
   # Only support beam search for now
   lowerCAmelCase				,lowerCAmelCase    				= self.generate_beam(
       input_embeds=_snake_case				,       device=_snake_case				,       eos_token_id=_snake_case     )
   generated_tokens.append(output_tokens[0]     )
   generated_seq_lengths.append(seq_lengths[0]     )
  lowerCAmelCase    				= torch.stack(_snake_case     )
  lowerCAmelCase    				= torch.stack(_snake_case     )
  return generated_tokens, generated_seq_lengths



 @torch.no_grad()
 def    UpperCamelCase__  (     self				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case = 5				,       _snake_case = 67				,       _snake_case = 1.0				,       _snake_case = None				,       ):







  """simple docstring"""






  lowerCAmelCase    				= eos_token_id
  lowerCAmelCase    				= None
  lowerCAmelCase    				= None
  lowerCAmelCase    				= torch.ones(_snake_case				,       device=_snake_case				,       dtype=torch.int     )
  lowerCAmelCase    				= torch.zeros(_snake_case				,       device=_snake_case				,       dtype=torch.bool     )

  if input_embeds is not None:
   lowerCAmelCase    				= input_embeds
  else:
   lowerCAmelCase    				= self.transformer.transformer.wte(_snake_case     )

  for i in range(_snake_case     ):
   lowerCAmelCase    				= self.transformer(inputs_embeds=_snake_case     )
   lowerCAmelCase    				= outputs.logits
   lowerCAmelCase    				= logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
   lowerCAmelCase    				= logits.softmax(-1     ).log()

   if scores is None:
    lowerCAmelCase				,lowerCAmelCase    				= logits.topk(_snake_case				,       -1     )
    lowerCAmelCase    				= generated.expand(_snake_case				,       *generated.shape[1:]     )
    lowerCAmelCase				,lowerCAmelCase    				= next_tokens.permute(1				,       0     ), scores.squeeze(0     )
    if tokens is None:
     lowerCAmelCase    				= next_tokens
    else:
     lowerCAmelCase    				= tokens.expand(_snake_case				,       *tokens.shape[1:]     )
     lowerCAmelCase    				= torch.cat((tokens, next_tokens)				,       dim=1     )
   else:
    lowerCAmelCase    				= -float(np.inf     )
    lowerCAmelCase    				= 0
    lowerCAmelCase    				= scores[:, None] + logits
    seq_lengths[~is_stopped] += 1
    lowerCAmelCase    				= scores_sum / seq_lengths[:, None]
    lowerCAmelCase				,lowerCAmelCase    				= scores_sum_average.view(-1     ).topk(_snake_case				,       -1     )
    lowerCAmelCase    				= next_tokens // scores_sum.shape[1]
    lowerCAmelCase    				= seq_lengths[next_tokens_source]
    lowerCAmelCase    				= next_tokens % scores_sum.shape[1]
    lowerCAmelCase    				= next_tokens.unsqueeze(1     )
    lowerCAmelCase    				= tokens[next_tokens_source]
    lowerCAmelCase    				= torch.cat((tokens, next_tokens)				,       dim=1     )
    lowerCAmelCase    				= generated[next_tokens_source]
    lowerCAmelCase    				= scores_sum_average * seq_lengths
    lowerCAmelCase    				= is_stopped[next_tokens_source]

   lowerCAmelCase    				= self.transformer.transformer.wte(next_tokens.squeeze()     ).view(generated.shape[0]				,       1				,       -1     )
   lowerCAmelCase    				= torch.cat((generated, next_token_embed)				,       dim=1     )
   lowerCAmelCase    				= is_stopped + next_tokens.eq(_snake_case     ).squeeze()
   if is_stopped.all():
    break

  lowerCAmelCase    				= scores / seq_lengths
  lowerCAmelCase    				= scores.argsort(descending=_snake_case     )
  # tokens tensors are already padded to max_seq_length
  lowerCAmelCase    				= [tokens[i] for i in order]
  lowerCAmelCase    				= torch.stack(_snake_case				,       dim=0     )
  lowerCAmelCase    				= torch.tensor([seq_lengths[i] for i in order]				,       dtype=seq_lengths.dtype     )
  return output_texts, seq_lengths








import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


_lowerCAmelCase  :      int			 ="""scheduler_config.json"""

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   1
					__magic_name__        =   2
					__magic_name__        =   3
					__magic_name__        =   4
					__magic_name__        =   5
					__magic_name__        =   6
					__magic_name__        =   7
					__magic_name__        =   8
					__magic_name__        =   9
					__magic_name__        =   1_0
					__magic_name__        =   1_1
					__magic_name__        =   1_2
					__magic_name__        =   1_3
					__magic_name__        =   1_4

@dataclass
class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   42

class __UpperCamelCase     :






					'''simple docstring'''



					__magic_name__        =   SCHEDULER_CONFIG_NAME
					__magic_name__        =   []
					__magic_name__        =   True





					@classmethod
					def    _UpperCAmelCase    (				cls				,      lowerCamelCase__ = None				,      lowerCamelCase__ = None				,      lowerCamelCase__=False				,      **lowerCamelCase__				,      ):
						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= cls.load_config(
						    pretrained_model_name_or_path=lowerCamelCase__				,      subfolder=lowerCamelCase__				,      return_unused_kwargs=lowerCamelCase__				,      return_commit_hash=lowerCamelCase__				,      **lowerCamelCase__				,      )
						return cls.from_config(lowerCamelCase__				,      return_unused_kwargs=lowerCamelCase__				,      **lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = False				,      **lowerCamelCase__						):
						self.save_config(save_directory=lowerCamelCase__				,      push_to_hub=lowerCamelCase__				,      **lowerCamelCase__						)





					@property
					def    _UpperCAmelCase    (				self						):
						return self._get_compatibles()





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						UpperCAmelCase__:	List[str]									= list(set([cls.__name__] + cls._compatibles						)						)
						UpperCAmelCase__:	Union[str, Any]									= importlib.import_module(__name__.split("."						)[0]						)
						UpperCAmelCase__:	int									= [
						    getattr(lowerCamelCase__				,      lowerCamelCase__						) for c in compatible_classes_str if hasattr(lowerCamelCase__				,      lowerCamelCase__						)
						]
						return compatible_classes