



'''simple docstring'''





from importlib import import_module

from .logging import get_logger


lowercase_									=    get_logger(__name__)






class __A     :



 '''simple docstring'''







 def __init__(self					,  A					,  A=None			)       ->	List[str]:


      """simple docstring"""



      _a           =    attrs or []
      if module is not None:
           for key in module.__dict__:
                if key in attrs or not key.startswith('''__'''			):
                     setattr(self					,  A					,  getattr(A					,  A			)			)
      _a           =    module._original_module if isinstance(A					,  _PatchedModuleObj			) else module






class __A     :



 '''simple docstring'''







 __lowerCamelCase       :			Tuple		=			[]



 def __init__(self					,  A					,  A					,  A					,  A=None			)       ->	Optional[int]:


      """simple docstring"""



      _a           =    obj
      _a           =    target
      _a           =    new
      _a           =    target.split('''.'''			)[0]
      _a           =    {}
      _a           =    attrs or []



 def __enter__(self			)       ->	Any:


      """simple docstring"""



      *_a    ,       _a           =    self.target.split('''.'''			)

      # Patch modules:
      # it's used to patch attributes of submodules like "os.path.join";
      # in this case we need to patch "os" and "os.path"

      for i in range(len(A			)			):
           try:
                _a           =    import_module('''.'''.join(submodules[: i + 1]			)			)
           except ModuleNotFoundError:
                continue
           # We iterate over all the globals in self.obj in case we find "os" or "os.path"
           for attr in self.obj.__dir__():
                _a           =    getattr(self.obj					,  A			)
                # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
                # This allows to patch renamed modules like "from os import path as ospath".
                if obj_attr is submodule or (
                    (isinstance(A					,  _PatchedModuleObj			) and obj_attr._original_module is submodule)
                ):
                     _a           =    obj_attr
                     # patch at top level
                     setattr(self.obj					,  A					,  _PatchedModuleObj(A					,  attrs=self.attrs			)			)
                     _a           =    getattr(self.obj					,  A			)
                     # construct lower levels patches
                     for key in submodules[i + 1 :]:
                          setattr(A					,  A					,  _PatchedModuleObj(getattr(A					,  A					,  A			)					,  attrs=self.attrs			)			)
                          _a           =    getattr(A					,  A			)
                     # finally set the target attribute
                     setattr(A					,  A					,  self.new			)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

      if submodules:  # if it's an attribute of a submodule like "os.path.join"
           try:
                _a           =    getattr(import_module('''.'''.join(A			)			)					,  A			)
           except (AttributeError, ModuleNotFoundError):
                return
           # We iterate over all the globals in self.obj in case we find "os.path.join"
           for attr in self.obj.__dir__():
                # We don't check for the name of the global, but rather if its value *is* "os.path.join".
                # This allows to patch renamed attributes like "from os.path import join as pjoin".
                if getattr(self.obj					,  A			) is attr_value:
                     _a           =    getattr(self.obj					,  A			)
                     setattr(self.obj					,  A					,  self.new			)
      elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
           _a           =    globals()['''__builtins__'''][target_attr]
           setattr(self.obj					,  A					,  self.new			)
      else:
           raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''			)



 def __exit__(self					,  *A			)       ->	List[str]:


      """simple docstring"""



      for attr in list(self.original			):
           setattr(self.obj					,  A					,  self.original.pop(A			)			)



 def    a__    (self			)       ->	str:


      """simple docstring"""



      self.__enter__()
      self._active_patches.append(self			)



 def    a__    (self			)       ->	List[Any]:


      """simple docstring"""



      try:
           self._active_patches.remove(self			)
      except ValueError:
           # If the patch hasn't been started this will fail
           return None

      return self.__exit__()







"""simple docstring"""



from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class    __a						( __snake_case					,			__snake_case					,			__snake_case							):
				lowerCamelCase     :       Optional[int]			       =[R'h\.\d+\.attn\.bias', R'h\.\d+\.attn\.masked_bias']







				@register_to_config
				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase = None      ,	UpperCAmelCase = 5_0257      ,	UpperCAmelCase = 1024      ,	UpperCAmelCase = 768      ,	UpperCAmelCase = 12      ,	UpperCAmelCase = 12      ,	UpperCAmelCase = None      ,	UpperCAmelCase = "gelu_new"      ,	UpperCAmelCase = 0.1      ,	UpperCAmelCase = 0.1      ,	UpperCAmelCase = 0.1      ,	UpperCAmelCase = 1E-5      ,	UpperCAmelCase = 0.0_2      ,	UpperCAmelCase = True      ,	UpperCAmelCase = True      ,	UpperCAmelCase = False      ,	UpperCAmelCase = False      ,	):

					'''simple docstring'''






					super().__init__()

					lowerCAmelCase_      							=			prefix_length

					if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
						raise ValueError(
						    F"""`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"""
						    F""" `n_embd`: {n_embd} are not equal."""   )

					lowerCAmelCase_      							=			prefix_inner_dim
					lowerCAmelCase_      							=			prefix_hidden_dim

					lowerCAmelCase_      							=			(
					    nn.Linear(self.prefix_inner_dim      ,	self.prefix_hidden_dim   )
					    if self.prefix_hidden_dim is not None
					    else nn.Identity()
					)
					lowerCAmelCase_      							=			(
					    nn.Linear(self.prefix_hidden_dim      ,	UpperCAmelCase   ) if self.prefix_hidden_dim is not None else nn.Identity()
					)

					lowerCAmelCase_      							=			GPTaConfig(
					    vocab_size=UpperCAmelCase      ,	n_positions=UpperCAmelCase      ,	n_embd=UpperCAmelCase      ,	n_layer=UpperCAmelCase      ,	n_head=UpperCAmelCase      ,	n_inner=UpperCAmelCase      ,	activation_function=UpperCAmelCase      ,	resid_pdrop=UpperCAmelCase      ,	embd_pdrop=UpperCAmelCase      ,	attn_pdrop=UpperCAmelCase      ,	layer_norm_epsilon=UpperCAmelCase      ,	initializer_range=UpperCAmelCase      ,	scale_attn_weights=UpperCAmelCase      ,	use_cache=UpperCAmelCase      ,	scale_attn_by_inverse_layer_idx=UpperCAmelCase      ,	reorder_and_upcast_attn=UpperCAmelCase      ,	)
					lowerCAmelCase_      							=			GPTaLMHeadModel(UpperCAmelCase   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase = None      ,	UpperCAmelCase = None      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.transformer.transformer.wte(UpperCAmelCase   )
					lowerCAmelCase_      							=			self.encode_prefix(UpperCAmelCase   )
					lowerCAmelCase_      							=			self.decode_prefix(UpperCAmelCase   )
					lowerCAmelCase_      							=			torch.cat((prefix_embeds, embedding_text)      ,	dim=1   )

					if labels is not None:
						lowerCAmelCase_      							=			self.get_dummy_token(input_ids.shape[0]      ,	input_ids.device   )
						lowerCAmelCase_      							=			torch.cat((dummy_token, input_ids)      ,	dim=1   )
					lowerCAmelCase_      							=			self.transformer(inputs_embeds=UpperCAmelCase      ,	labels=UpperCAmelCase      ,	attention_mask=UpperCAmelCase   )
					if self.prefix_hidden_dim is not None:
						return out, hidden
					else:
						return out







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					return torch.zeros(UpperCAmelCase      ,	self.prefix_length      ,	dtype=torch.intaa      ,	device=UpperCAmelCase   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					return self.encode_prefix(UpperCAmelCase   )







				@torch.no_grad()
				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			torch.split(UpperCAmelCase      ,	1      ,	dim=0   )
					lowerCAmelCase_      							=			[]
					lowerCAmelCase_      							=			[]
					for feature in features:
						lowerCAmelCase_      							=			self.decode_prefix(feature.to(UpperCAmelCase   )   )  # back to the clip feature
						# Only support beam search for now
						lowerCAmelCase_							,	lowerCAmelCase_      							=			self.generate_beam(
						    input_embeds=UpperCAmelCase      ,	device=UpperCAmelCase      ,	eos_token_id=UpperCAmelCase   )
						generated_tokens.append(output_tokens[0]   )
						generated_seq_lengths.append(seq_lengths[0]   )
					lowerCAmelCase_      							=			torch.stack(UpperCAmelCase   )
					lowerCAmelCase_      							=			torch.stack(UpperCAmelCase   )
					return generated_tokens, generated_seq_lengths







				@torch.no_grad()
				def      lowerCamelCase_     ( self      ,	UpperCAmelCase=None      ,	UpperCAmelCase=None      ,	UpperCAmelCase=None      ,	UpperCAmelCase = 5      ,	UpperCAmelCase = 67      ,	UpperCAmelCase = 1.0      ,	UpperCAmelCase = None      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			eos_token_id
					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			None
					lowerCAmelCase_      							=			torch.ones(UpperCAmelCase      ,	device=UpperCAmelCase      ,	dtype=torch.int   )
					lowerCAmelCase_      							=			torch.zeros(UpperCAmelCase      ,	device=UpperCAmelCase      ,	dtype=torch.bool   )

					if input_embeds is not None:
						lowerCAmelCase_      							=			input_embeds
					else:
						lowerCAmelCase_      							=			self.transformer.transformer.wte(UpperCAmelCase   )

					for i in range(UpperCAmelCase   ):
						lowerCAmelCase_      							=			self.transformer(inputs_embeds=UpperCAmelCase   )
						lowerCAmelCase_      							=			outputs.logits
						lowerCAmelCase_      							=			logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
						lowerCAmelCase_      							=			logits.softmax(-1   ).log()

						if scores is None:
							lowerCAmelCase_							,	lowerCAmelCase_      							=			logits.topk(UpperCAmelCase      ,	-1   )
							lowerCAmelCase_      							=			generated.expand(UpperCAmelCase      ,	*generated.shape[1:]   )
							lowerCAmelCase_							,	lowerCAmelCase_      							=			next_tokens.permute(1      ,	0   ), scores.squeeze(0   )
							if tokens is None:
								lowerCAmelCase_      							=			next_tokens
							else:
								lowerCAmelCase_      							=			tokens.expand(UpperCAmelCase      ,	*tokens.shape[1:]   )
								lowerCAmelCase_      							=			torch.cat((tokens, next_tokens)      ,	dim=1   )
						else:
							lowerCAmelCase_      							=			-float(np.inf   )
							lowerCAmelCase_      							=			0
							lowerCAmelCase_      							=			scores[:, None] + logits
							seq_lengths[~is_stopped] += 1
							lowerCAmelCase_      							=			scores_sum / seq_lengths[:, None]
							lowerCAmelCase_							,	lowerCAmelCase_      							=			scores_sum_average.view(-1   ).topk(UpperCAmelCase      ,	-1   )
							lowerCAmelCase_      							=			next_tokens // scores_sum.shape[1]
							lowerCAmelCase_      							=			seq_lengths[next_tokens_source]
							lowerCAmelCase_      							=			next_tokens % scores_sum.shape[1]
							lowerCAmelCase_      							=			next_tokens.unsqueeze(1   )
							lowerCAmelCase_      							=			tokens[next_tokens_source]
							lowerCAmelCase_      							=			torch.cat((tokens, next_tokens)      ,	dim=1   )
							lowerCAmelCase_      							=			generated[next_tokens_source]
							lowerCAmelCase_      							=			scores_sum_average * seq_lengths
							lowerCAmelCase_      							=			is_stopped[next_tokens_source]

						lowerCAmelCase_      							=			self.transformer.transformer.wte(next_tokens.squeeze()   ).view(generated.shape[0]      ,	1      ,	-1   )
						lowerCAmelCase_      							=			torch.cat((generated, next_token_embed)      ,	dim=1   )
						lowerCAmelCase_      							=			is_stopped + next_tokens.eq(UpperCAmelCase   ).squeeze()
						if is_stopped.all():
							break

					lowerCAmelCase_      							=			scores / seq_lengths
					lowerCAmelCase_      							=			scores.argsort(descending=UpperCAmelCase   )
					# tokens tensors are already padded to max_seq_length
					lowerCAmelCase_      							=			[tokens[i] for i in order]
					lowerCAmelCase_      							=			torch.stack(UpperCAmelCase      ,	dim=0   )
					lowerCAmelCase_      							=			torch.tensor([seq_lengths[i] for i in order]      ,	dtype=seq_lengths.dtype   )
					return output_texts, seq_lengths