






from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class     _lowerCAmelCase(	UpperCAmelCase_							,     UpperCAmelCase_							,     UpperCAmelCase_ ):





       """simple docstring"""







       a    :		Optional[int]									=[r'''h\.\d+\.attn\.bias''', r'''h\.\d+\.attn\.masked_bias''']





       @register_to_config
       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase = None  ,      _lowerCamelCase = 5_0_2_5_7  ,      _lowerCamelCase = 1_0_2_4  ,      _lowerCamelCase = 7_6_8  ,      _lowerCamelCase = 1_2  ,      _lowerCamelCase = 1_2  ,      _lowerCamelCase = None  ,      _lowerCamelCase = "gelu_new"  ,      _lowerCamelCase = 0.1  ,      _lowerCamelCase = 0.1  ,      _lowerCamelCase = 0.1  ,      _lowerCamelCase = 1e-5  ,      _lowerCamelCase = 0.0_2  ,      _lowerCamelCase = True  ,      _lowerCamelCase = True  ,      _lowerCamelCase = False  ,      _lowerCamelCase = False  ,      ):
         super().__init__()

         UpperCamelCase_: str   				=    prefix_length

         if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
           raise ValueError(
               f'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
               f''' `n_embd`: {n_embd} are not equal.'''							)

         UpperCamelCase_: Any   				=    prefix_inner_dim
         UpperCamelCase_: Dict   				=    prefix_hidden_dim

         UpperCamelCase_: int   				=    (
             nn.Linear(self.prefix_inner_dim  ,      self.prefix_hidden_dim							)
             if self.prefix_hidden_dim is not None
             else nn.Identity()
         )
         UpperCamelCase_: Tuple   				=    (
             nn.Linear(self.prefix_hidden_dim  ,      _lowerCamelCase							) if self.prefix_hidden_dim is not None else nn.Identity()
         )

         UpperCamelCase_: Union[str, Any]   				=    GPTaConfig(
             vocab_size=_lowerCamelCase  ,      n_positions=_lowerCamelCase  ,      n_embd=_lowerCamelCase  ,      n_layer=_lowerCamelCase  ,      n_head=_lowerCamelCase  ,      n_inner=_lowerCamelCase  ,      activation_function=_lowerCamelCase  ,      resid_pdrop=_lowerCamelCase  ,      embd_pdrop=_lowerCamelCase  ,      attn_pdrop=_lowerCamelCase  ,      layer_norm_epsilon=_lowerCamelCase  ,      initializer_range=_lowerCamelCase  ,      scale_attn_weights=_lowerCamelCase  ,      use_cache=_lowerCamelCase  ,      scale_attn_by_inverse_layer_idx=_lowerCamelCase  ,      reorder_and_upcast_attn=_lowerCamelCase  ,      )
         UpperCamelCase_: Optional[Any]   				=    GPTaLMHeadModel(_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase = None  ,      _lowerCamelCase = None  ,      ):
         UpperCamelCase_: Any   				=    self.transformer.transformer.wte(_lowerCamelCase							)
         UpperCamelCase_: Any   				=    self.encode_prefix(_lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    self.decode_prefix(_lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    torch.cat((prefix_embeds, embedding_text)  ,      dim=1							)

         if labels is not None:
           UpperCamelCase_: Any   				=    self.get_dummy_token(input_ids.shape[0]  ,      input_ids.device							)
           UpperCamelCase_: str   				=    torch.cat((dummy_token, input_ids)  ,      dim=1							)
         UpperCamelCase_: int   				=    self.transformer(inputs_embeds=_lowerCamelCase  ,      labels=_lowerCamelCase  ,      attention_mask=_lowerCamelCase							)
         if self.prefix_hidden_dim is not None:
           return out, hidden
         else:
           return out





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase							):
         return torch.zeros(_lowerCamelCase  ,      self.prefix_length  ,      dtype=torch.intaa  ,      device=_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.encode_prefix(_lowerCamelCase							)





       @torch.no_grad()
       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
         UpperCamelCase_: List[Any]   				=    torch.split(_lowerCamelCase  ,      1  ,      dim=0							)
         UpperCamelCase_: Optional[Any]   				=    []
         UpperCamelCase_: Optional[int]   				=    []
         for feature in features:
           UpperCamelCase_: Tuple   				=    self.decode_prefix(feature.to(_lowerCamelCase							)							)  # back to the clip feature
           # Only support beam search for now
           UpperCamelCase_: Optional[Any]   				=    self.generate_beam(
               input_embeds=_lowerCamelCase  ,      device=_lowerCamelCase  ,      eos_token_id=_lowerCamelCase							)
           generated_tokens.append(output_tokens[0]							)
           generated_seq_lengths.append(seq_lengths[0]							)
         UpperCamelCase_: Any   				=    torch.stack(_lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    torch.stack(_lowerCamelCase							)
         return generated_tokens, generated_seq_lengths





       @torch.no_grad()
       def       _a   ( self  ,      _lowerCamelCase=None  ,      _lowerCamelCase=None  ,      _lowerCamelCase=None  ,      _lowerCamelCase = 5  ,      _lowerCamelCase = 6_7  ,      _lowerCamelCase = 1.0  ,      _lowerCamelCase = None  ,      ):
         UpperCamelCase_: Dict   				=    eos_token_id
         UpperCamelCase_: Tuple   				=    None
         UpperCamelCase_: Optional[Any]   				=    None
         UpperCamelCase_: List[Any]   				=    torch.ones(_lowerCamelCase  ,      device=_lowerCamelCase  ,      dtype=torch.int							)
         UpperCamelCase_: int   				=    torch.zeros(_lowerCamelCase  ,      device=_lowerCamelCase  ,      dtype=torch.bool							)

         if input_embeds is not None:
           UpperCamelCase_: List[str]   				=    input_embeds
         else:
           UpperCamelCase_: str   				=    self.transformer.transformer.wte(_lowerCamelCase							)

         for i in range(_lowerCamelCase							):
           UpperCamelCase_: Dict   				=    self.transformer(inputs_embeds=_lowerCamelCase							)
           UpperCamelCase_: Any   				=    outputs.logits
           UpperCamelCase_: Optional[Any]   				=    logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
           UpperCamelCase_: str   				=    logits.softmax(-1							).log()

           if scores is None:
             UpperCamelCase_: List[Any]   				=    logits.topk(_lowerCamelCase  ,      -1							)
             UpperCamelCase_: Union[str, Any]   				=    generated.expand(_lowerCamelCase  ,      *generated.shape[1:]							)
             UpperCamelCase_: Union[str, Any]   				=    next_tokens.permute(1  ,      0							), scores.squeeze(0							)
             if tokens is None:
               UpperCamelCase_: Optional[Any]   				=    next_tokens
             else:
               UpperCamelCase_: Tuple   				=    tokens.expand(_lowerCamelCase  ,      *tokens.shape[1:]							)
               UpperCamelCase_: Optional[int]   				=    torch.cat((tokens, next_tokens)  ,      dim=1							)
           else:
             UpperCamelCase_: Tuple   				=    -float(np.inf							)
             UpperCamelCase_: Tuple   				=    0
             UpperCamelCase_: Optional[Any]   				=    scores[:, None] + logits
             seq_lengths[~is_stopped] += 1
             UpperCamelCase_: Optional[int]   				=    scores_sum / seq_lengths[:, None]
             UpperCamelCase_: str   				=    scores_sum_average.view(-1							).topk(_lowerCamelCase  ,      -1							)
             UpperCamelCase_: List[str]   				=    next_tokens // scores_sum.shape[1]
             UpperCamelCase_: Union[str, Any]   				=    seq_lengths[next_tokens_source]
             UpperCamelCase_: int   				=    next_tokens % scores_sum.shape[1]
             UpperCamelCase_: Optional[Any]   				=    next_tokens.unsqueeze(1							)
             UpperCamelCase_: Dict   				=    tokens[next_tokens_source]
             UpperCamelCase_: List[Any]   				=    torch.cat((tokens, next_tokens)  ,      dim=1							)
             UpperCamelCase_: List[Any]   				=    generated[next_tokens_source]
             UpperCamelCase_: Union[str, Any]   				=    scores_sum_average * seq_lengths
             UpperCamelCase_: List[Any]   				=    is_stopped[next_tokens_source]

           UpperCamelCase_: Tuple   				=    self.transformer.transformer.wte(next_tokens.squeeze()							).view(generated.shape[0]  ,      1  ,      -1							)
           UpperCamelCase_: int   				=    torch.cat((generated, next_token_embed)  ,      dim=1							)
           UpperCamelCase_: Dict   				=    is_stopped + next_tokens.eq(_lowerCamelCase							).squeeze()
           if is_stopped.all():
             break

         UpperCamelCase_: Optional[Any]   				=    scores / seq_lengths
         UpperCamelCase_: str   				=    scores.argsort(descending=_lowerCamelCase							)
         # tokens tensors are already padded to max_seq_length
         UpperCamelCase_: Dict   				=    [tokens[i] for i in order]
         UpperCamelCase_: Optional[int]   				=    torch.stack(_lowerCamelCase  ,      dim=0							)
         UpperCamelCase_: Optional[Any]   				=    torch.tensor([seq_lengths[i] for i in order]  ,      dtype=seq_lengths.dtype							)
         return output_texts, seq_lengths









from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:		Tuple		     =			logging.get_logger(__name__)

A_		:		Dict		     =			{
    'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/config.json',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       a    :		Tuple									='''xglm'''
       a    :		List[Any]									=['''past_key_values''']

       a    :		Union[str, Any]									={
           '''num_attention_heads''': '''attention_heads''',
           '''hidden_size''': '''d_model''',
           '''num_hidden_layers''': '''num_layers''',
       }





       def __init__( self  ,      _lowerCamelCase=2_5_6_0_0_8  ,      _lowerCamelCase=2_0_4_8  ,      _lowerCamelCase=1_0_2_4  ,      _lowerCamelCase=4_0_9_6  ,      _lowerCamelCase=2_4  ,      _lowerCamelCase=1_6  ,      _lowerCamelCase="gelu"  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.0  ,      _lowerCamelCase=0.0  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=2  ,      _lowerCamelCase=1  ,      _lowerCamelCase=0  ,      _lowerCamelCase=2  ,      **_lowerCamelCase  ,      ):
         UpperCamelCase_: Optional[Any]   				=    vocab_size
         UpperCamelCase_: Optional[int]   				=    max_position_embeddings
         UpperCamelCase_: List[str]   				=    d_model
         UpperCamelCase_: List[Any]   				=    ffn_dim
         UpperCamelCase_: List[Any]   				=    num_layers
         UpperCamelCase_: List[Any]   				=    attention_heads
         UpperCamelCase_: Tuple   				=    activation_function
         UpperCamelCase_: Tuple   				=    dropout
         UpperCamelCase_: Tuple   				=    attention_dropout
         UpperCamelCase_: Optional[Any]   				=    activation_dropout
         UpperCamelCase_: List[str]   				=    layerdrop
         UpperCamelCase_: Any   				=    init_std
         UpperCamelCase_: Any   				=    scale_embedding  # scale factor will be sqrt(d_model) if True
         UpperCamelCase_: Union[str, Any]   				=    use_cache

         super().__init__(
             pad_token_id=_lowerCamelCase  ,      bos_token_id=_lowerCamelCase  ,      eos_token_id=_lowerCamelCase  ,      decoder_start_token_id=_lowerCamelCase  ,      **_lowerCamelCase  ,      )