





'''simple docstring'''






from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class 					UpperCAmelCase	(     snake_case_				,						snake_case_				,						snake_case_  ):
      _lowercase:     Dict			=  [r'''h\.\d+\.attn\.bias''', r'''h\.\d+\.attn\.masked_bias''']




      @register_to_config
      def __init__(     self     :					Optional[int]     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					Optional[int] = None     ,     __snake_case     :					int = 5_02_57     ,     __snake_case     :					int = 10_24     ,     __snake_case     :					int = 7_68     ,     __snake_case     :					int = 12     ,     __snake_case     :					int = 12     ,     __snake_case     :					Optional[int] = None     ,     __snake_case     :					str = "gelu_new"     ,     __snake_case     :					float = 0.1     ,     __snake_case     :					float = 0.1     ,     __snake_case     :					float = 0.1     ,     __snake_case     :					float = 1E-5     ,     __snake_case     :					float = 0.02     ,     __snake_case     :					bool = True     ,     __snake_case     :					bool = True     ,     __snake_case     :					bool = False     ,     __snake_case     :					bool = False     ,     )							-> List[str]:
            super().__init__()

            _lowerCAmelCase					=				prefix_length

            if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
                  raise ValueError(
                      f"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"
                      f" `n_embd`: {n_embd} are not equal."    )

            _lowerCAmelCase					=				prefix_inner_dim
            _lowerCAmelCase					=				prefix_hidden_dim

            _lowerCAmelCase					=				(
                nn.Linear(self.prefix_inner_dim     ,     self.prefix_hidden_dim    )
                if self.prefix_hidden_dim is not None
                else nn.Identity()
            )
            _lowerCAmelCase					=				(
                nn.Linear(self.prefix_hidden_dim     ,     __snake_case    ) if self.prefix_hidden_dim is not None else nn.Identity()
            )

            _lowerCAmelCase					=				GPTaConfig(
                vocab_size=__snake_case     ,     n_positions=__snake_case     ,     n_embd=__snake_case     ,     n_layer=__snake_case     ,     n_head=__snake_case     ,     n_inner=__snake_case     ,     activation_function=__snake_case     ,     resid_pdrop=__snake_case     ,     embd_pdrop=__snake_case     ,     attn_pdrop=__snake_case     ,     layer_norm_epsilon=__snake_case     ,     initializer_range=__snake_case     ,     scale_attn_weights=__snake_case     ,     use_cache=__snake_case     ,     scale_attn_by_inverse_layer_idx=__snake_case     ,     reorder_and_upcast_attn=__snake_case     ,     )
            _lowerCAmelCase					=				GPTaLMHeadModel(__snake_case    )




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					torch.Tensor     ,     __snake_case     :					torch.Tensor     ,     __snake_case     :					Optional[torch.Tensor] = None     ,     __snake_case     :					Optional[torch.Tensor] = None     ,     )							-> Tuple:
            _lowerCAmelCase					=				self.transformer.transformer.wte(__snake_case    )
            _lowerCAmelCase					=				self.encode_prefix(__snake_case    )
            _lowerCAmelCase					=				self.decode_prefix(__snake_case    )
            _lowerCAmelCase					=				torch.cat((prefix_embeds, embedding_text)     ,     dim=1    )

            if labels is not None:
                  _lowerCAmelCase					=				self.get_dummy_token(input_ids.shape[0]     ,     input_ids.device    )
                  _lowerCAmelCase					=				torch.cat((dummy_token, input_ids)     ,     dim=1    )
            _lowerCAmelCase					=				self.transformer(inputs_embeds=__snake_case     ,     labels=__snake_case     ,     attention_mask=__snake_case    )
            if self.prefix_hidden_dim is not None:
                  return out, hidden
            else:
                  return out




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					int     ,     __snake_case     :					torch.device    )							-> torch.Tensor:
            return torch.zeros(__snake_case     ,     self.prefix_length     ,     dtype=torch.intaa     ,     device=__snake_case    )




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					Optional[int]    )							-> Optional[int]:
            return self.encode_prefix(__snake_case    )




      @torch.no_grad()
      def 							lowercase__       (     self     :					Any     ,     __snake_case     :					str     ,     __snake_case     :					List[str]     ,     __snake_case     :					Any    )							-> Tuple:
            _lowerCAmelCase					=				torch.split(__snake_case     ,     1     ,     dim=0    )
            _lowerCAmelCase					=				[]
            _lowerCAmelCase					=				[]
            for feature in features:
                  _lowerCAmelCase					=				self.decode_prefix(feature.to(__snake_case    )    )  # back to the clip feature
                  # Only support beam search for now
                  _lowerCAmelCase	,				_lowerCAmelCase					=				self.generate_beam(
                      input_embeds=__snake_case     ,     device=__snake_case     ,     eos_token_id=__snake_case    )
                  generated_tokens.append(output_tokens[0]    )
                  generated_seq_lengths.append(seq_lengths[0]    )
            _lowerCAmelCase					=				torch.stack(__snake_case    )
            _lowerCAmelCase					=				torch.stack(__snake_case    )
            return generated_tokens, generated_seq_lengths




      @torch.no_grad()
      def 							lowercase__       (     self     :					str     ,     __snake_case     :					List[str]=None     ,     __snake_case     :					Dict=None     ,     __snake_case     :					List[str]=None     ,     __snake_case     :					int = 5     ,     __snake_case     :					int = 67     ,     __snake_case     :					float = 1.0     ,     __snake_case     :					Optional[int] = None     ,     )							-> int:
            _lowerCAmelCase					=				eos_token_id
            _lowerCAmelCase					=				None
            _lowerCAmelCase					=				None
            _lowerCAmelCase					=				torch.ones(__snake_case     ,     device=__snake_case     ,     dtype=torch.int    )
            _lowerCAmelCase					=				torch.zeros(__snake_case     ,     device=__snake_case     ,     dtype=torch.bool    )

            if input_embeds is not None:
                  _lowerCAmelCase					=				input_embeds
            else:
                  _lowerCAmelCase					=				self.transformer.transformer.wte(__snake_case    )

            for i in range(__snake_case    ):
                  _lowerCAmelCase					=				self.transformer(inputs_embeds=__snake_case    )
                  _lowerCAmelCase					=				outputs.logits
                  _lowerCAmelCase					=				logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
                  _lowerCAmelCase					=				logits.softmax(-1    ).log()

                  if scores is None:
                        _lowerCAmelCase	,				_lowerCAmelCase					=				logits.topk(__snake_case     ,     -1    )
                        _lowerCAmelCase					=				generated.expand(__snake_case     ,     *generated.shape[1:]    )
                        _lowerCAmelCase	,				_lowerCAmelCase					=				next_tokens.permute(1     ,     0    ), scores.squeeze(0    )
                        if tokens is None:
                              _lowerCAmelCase					=				next_tokens
                        else:
                              _lowerCAmelCase					=				tokens.expand(__snake_case     ,     *tokens.shape[1:]    )
                              _lowerCAmelCase					=				torch.cat((tokens, next_tokens)     ,     dim=1    )
                  else:
                        _lowerCAmelCase					=				-float(np.inf    )
                        _lowerCAmelCase					=				0
                        _lowerCAmelCase					=				scores[:, None] + logits
                        seq_lengths[~is_stopped] += 1
                        _lowerCAmelCase					=				scores_sum / seq_lengths[:, None]
                        _lowerCAmelCase	,				_lowerCAmelCase					=				scores_sum_average.view(-1    ).topk(__snake_case     ,     -1    )
                        _lowerCAmelCase					=				next_tokens // scores_sum.shape[1]
                        _lowerCAmelCase					=				seq_lengths[next_tokens_source]
                        _lowerCAmelCase					=				next_tokens % scores_sum.shape[1]
                        _lowerCAmelCase					=				next_tokens.unsqueeze(1    )
                        _lowerCAmelCase					=				tokens[next_tokens_source]
                        _lowerCAmelCase					=				torch.cat((tokens, next_tokens)     ,     dim=1    )
                        _lowerCAmelCase					=				generated[next_tokens_source]
                        _lowerCAmelCase					=				scores_sum_average * seq_lengths
                        _lowerCAmelCase					=				is_stopped[next_tokens_source]

                  _lowerCAmelCase					=				self.transformer.transformer.wte(next_tokens.squeeze()    ).view(generated.shape[0]     ,     1     ,     -1    )
                  _lowerCAmelCase					=				torch.cat((generated, next_token_embed)     ,     dim=1    )
                  _lowerCAmelCase					=				is_stopped + next_tokens.eq(__snake_case    ).squeeze()
                  if is_stopped.all():
                        break

            _lowerCAmelCase					=				scores / seq_lengths
            _lowerCAmelCase					=				scores.argsort(descending=__snake_case    )
            # tokens tensors are already padded to max_seq_length
            _lowerCAmelCase					=				[tokens[i] for i in order]
            _lowerCAmelCase					=				torch.stack(__snake_case     ,     dim=0    )
            _lowerCAmelCase					=				torch.tensor([seq_lengths[i] for i in order]     ,     dtype=seq_lengths.dtype    )
            return output_texts, seq_lengths













'''simple docstring'''






# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__							:  int    ={
    '''configuration_xmod''': [
        '''XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''XmodConfig''',
        '''XmodOnnxConfig''',
    ],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__							:  List[str]    =[
          '''XMOD_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''XmodForCausalLM''',
          '''XmodForMaskedLM''',
          '''XmodForMultipleChoice''',
          '''XmodForQuestionAnswering''',
          '''XmodForSequenceClassification''',
          '''XmodForTokenClassification''',
          '''XmodModel''',
          '''XmodPreTrainedModel''',
      ]

if TYPE_CHECKING:
      from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_xmod import (
                XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
                XmodForCausalLM,
                XmodForMaskedLM,
                XmodForMultipleChoice,
                XmodForQuestionAnswering,
                XmodForSequenceClassification,
                XmodForTokenClassification,
                XmodModel,
                XmodPreTrainedModel,
            )

else:
      import sys

      A__							:  str    =_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






