



"""simple docstring"""






from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class      A__  (    __SCREAMING_SNAKE_CASE    ,						__SCREAMING_SNAKE_CASE    ,						__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      [R'h\.\d+\.attn\.bias', R'h\.\d+\.attn\.masked_bias']

	@register_to_config
	def __init__(	self:						str			,     _SCREAMING_SNAKE_CASE:						int			,     _SCREAMING_SNAKE_CASE:						int			,     _SCREAMING_SNAKE_CASE:						Optional[int] = None			,     _SCREAMING_SNAKE_CASE:						int = 5_0257			,     _SCREAMING_SNAKE_CASE:						int = 1024			,     _SCREAMING_SNAKE_CASE:						int = 768			,     _SCREAMING_SNAKE_CASE:						int = 12			,     _SCREAMING_SNAKE_CASE:						int = 12			,     _SCREAMING_SNAKE_CASE:						Optional[int] = None			,     _SCREAMING_SNAKE_CASE:						str = "gelu_new"			,     _SCREAMING_SNAKE_CASE:						float = 0.1			,     _SCREAMING_SNAKE_CASE:						float = 0.1			,     _SCREAMING_SNAKE_CASE:						float = 0.1			,     _SCREAMING_SNAKE_CASE:						float = 1e-5			,     _SCREAMING_SNAKE_CASE:						float = 0.02			,     _SCREAMING_SNAKE_CASE:						bool = True			,     _SCREAMING_SNAKE_CASE:						bool = True			,     _SCREAMING_SNAKE_CASE:						bool = False			,     _SCREAMING_SNAKE_CASE:						bool = False			,     )							->			Dict:

				"""simple docstring"""

				super().__init__()

				__lowerCAmelCase :    int				=			prefix_length

				if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
							raise ValueError(
							    F"""`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"""
							    F""" `n_embd`: {n_embd} are not equal.""")

				__lowerCAmelCase :    Dict				=			prefix_inner_dim
				__lowerCAmelCase :    Dict				=			prefix_hidden_dim

				__lowerCAmelCase :    List[str]				=			(
				    nn.Linear(self.prefix_inner_dim			,     self.prefix_hidden_dim)
				    if self.prefix_hidden_dim is not None
				    else nn.Identity()
				)
				__lowerCAmelCase :    Any				=			(
				    nn.Linear(self.prefix_hidden_dim			,     _SCREAMING_SNAKE_CASE) if self.prefix_hidden_dim is not None else nn.Identity()
				)

				__lowerCAmelCase :    Tuple				=			GPTaConfig(
				    vocab_size=_SCREAMING_SNAKE_CASE			,     n_positions=_SCREAMING_SNAKE_CASE			,     n_embd=_SCREAMING_SNAKE_CASE			,     n_layer=_SCREAMING_SNAKE_CASE			,     n_head=_SCREAMING_SNAKE_CASE			,     n_inner=_SCREAMING_SNAKE_CASE			,     activation_function=_SCREAMING_SNAKE_CASE			,     resid_pdrop=_SCREAMING_SNAKE_CASE			,     embd_pdrop=_SCREAMING_SNAKE_CASE			,     attn_pdrop=_SCREAMING_SNAKE_CASE			,     layer_norm_epsilon=_SCREAMING_SNAKE_CASE			,     initializer_range=_SCREAMING_SNAKE_CASE			,     scale_attn_weights=_SCREAMING_SNAKE_CASE			,     use_cache=_SCREAMING_SNAKE_CASE			,     scale_attn_by_inverse_layer_idx=_SCREAMING_SNAKE_CASE			,     reorder_and_upcast_attn=_SCREAMING_SNAKE_CASE			,     )
				__lowerCAmelCase :    Tuple				=			GPTaLMHeadModel(_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						int			,     _SCREAMING_SNAKE_CASE:						torch.Tensor			,     _SCREAMING_SNAKE_CASE:						torch.Tensor			,     _SCREAMING_SNAKE_CASE:						Optional[torch.Tensor] = None			,     _SCREAMING_SNAKE_CASE:						Optional[torch.Tensor] = None			,     )							->			Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase :    Dict				=			self.transformer.transformer.wte(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    Optional[int]				=			self.encode_prefix(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    Any				=			self.decode_prefix(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    Tuple				=			torch.cat((prefix_embeds, embedding_text)			,     dim=1)

				if labels is not None:
							__lowerCAmelCase :    Union[str, Any]				=			self.get_dummy_token(input_ids.shape[0]			,     input_ids.device)
							__lowerCAmelCase :    Union[str, Any]				=			torch.cat((dummy_token, input_ids)			,     dim=1)
				__lowerCAmelCase :    List[str]				=			self.transformer(inputs_embeds=_SCREAMING_SNAKE_CASE			,     labels=_SCREAMING_SNAKE_CASE			,     attention_mask=_SCREAMING_SNAKE_CASE)
				if self.prefix_hidden_dim is not None:
							return out, hidden
				else:
							return out

	def 				_SCREAMING_SNAKE_CASE  (	self:						Optional[Any]			,     _SCREAMING_SNAKE_CASE:						int			,     _SCREAMING_SNAKE_CASE:						torch.device)							->			torch.Tensor:

				"""simple docstring"""

				return torch.zeros(_SCREAMING_SNAKE_CASE			,     self.prefix_length			,     dtype=torch.intaa			,     device=_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Optional[int]			,     _SCREAMING_SNAKE_CASE:						List[str])							->			Optional[int]:

				"""simple docstring"""

				return self.encode_prefix(_SCREAMING_SNAKE_CASE)

	@torch.no_grad()
	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any]			,     _SCREAMING_SNAKE_CASE:						Dict			,     _SCREAMING_SNAKE_CASE:						Dict			,     _SCREAMING_SNAKE_CASE:						Union[str, Any])							->			Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase :    int				=			torch.split(_SCREAMING_SNAKE_CASE			,     1			,     dim=0)
				__lowerCAmelCase :    List[Any]				=			[]
				__lowerCAmelCase :    Optional[Any]				=			[]
				for feature in features:
							__lowerCAmelCase :    int				=			self.decode_prefix(feature.to(_SCREAMING_SNAKE_CASE))  # back to the clip feature
							# Only support beam search for now
							__lowerCAmelCase		,      __lowerCAmelCase :    str				=			self.generate_beam(
							    input_embeds=_SCREAMING_SNAKE_CASE			,     device=_SCREAMING_SNAKE_CASE			,     eos_token_id=_SCREAMING_SNAKE_CASE)
							generated_tokens.append(output_tokens[0])
							generated_seq_lengths.append(seq_lengths[0])
				__lowerCAmelCase :    Tuple				=			torch.stack(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    Tuple				=			torch.stack(_SCREAMING_SNAKE_CASE)
				return generated_tokens, generated_seq_lengths







	@torch.no_grad()
	def 				_SCREAMING_SNAKE_CASE  (	self:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						Dict=None			,     _SCREAMING_SNAKE_CASE:						Optional[Any]=None			,     _SCREAMING_SNAKE_CASE:						List[Any]=None			,     _SCREAMING_SNAKE_CASE:						int = 5			,     _SCREAMING_SNAKE_CASE:						int = 67			,     _SCREAMING_SNAKE_CASE:						float = 1.0			,     _SCREAMING_SNAKE_CASE:						Optional[int] = None			,     )							->			Tuple:

				"""simple docstring"""

				__lowerCAmelCase :    Tuple				=			eos_token_id
				__lowerCAmelCase :    List[str]				=			None
				__lowerCAmelCase :    List[str]				=			None
				__lowerCAmelCase :    str				=			torch.ones(_SCREAMING_SNAKE_CASE			,     device=_SCREAMING_SNAKE_CASE			,     dtype=torch.int)
				__lowerCAmelCase :    int				=			torch.zeros(_SCREAMING_SNAKE_CASE			,     device=_SCREAMING_SNAKE_CASE			,     dtype=torch.bool)

				if input_embeds is not None:
							__lowerCAmelCase :    List[str]				=			input_embeds
				else:
							__lowerCAmelCase :    List[Any]				=			self.transformer.transformer.wte(_SCREAMING_SNAKE_CASE)

				for i in range(_SCREAMING_SNAKE_CASE):
							__lowerCAmelCase :    Optional[int]				=			self.transformer(inputs_embeds=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase :    Any				=			outputs.logits
							__lowerCAmelCase :    Tuple				=			logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
							__lowerCAmelCase :    Dict				=			logits.softmax(-1).log()

							if scores is None:
										__lowerCAmelCase		,      __lowerCAmelCase :    int				=			logits.topk(_SCREAMING_SNAKE_CASE			,     -1)
										__lowerCAmelCase :    Tuple				=			generated.expand(_SCREAMING_SNAKE_CASE			,     *generated.shape[1:])
										__lowerCAmelCase		,      __lowerCAmelCase :    str				=			next_tokens.permute(1			,     0), scores.squeeze(0)
										if tokens is None:
													__lowerCAmelCase :    Any				=			next_tokens
										else:
													__lowerCAmelCase :    List[Any]				=			tokens.expand(_SCREAMING_SNAKE_CASE			,     *tokens.shape[1:])
													__lowerCAmelCase :    Any				=			torch.cat((tokens, next_tokens)			,     dim=1)
							else:
										__lowerCAmelCase :    Dict				=			-float(np.inf)
										__lowerCAmelCase :    Dict				=			0
										__lowerCAmelCase :    Union[str, Any]				=			scores[:, None] + logits
										seq_lengths[~is_stopped] += 1
										__lowerCAmelCase :    Tuple				=			scores_sum / seq_lengths[:, None]
										__lowerCAmelCase		,      __lowerCAmelCase :    int				=			scores_sum_average.view(-1).topk(_SCREAMING_SNAKE_CASE			,     -1)
										__lowerCAmelCase :    Union[str, Any]				=			next_tokens // scores_sum.shape[1]
										__lowerCAmelCase :    List[Any]				=			seq_lengths[next_tokens_source]
										__lowerCAmelCase :    Dict				=			next_tokens % scores_sum.shape[1]
										__lowerCAmelCase :    Any				=			next_tokens.unsqueeze(1)
										__lowerCAmelCase :    str				=			tokens[next_tokens_source]
										__lowerCAmelCase :    Any				=			torch.cat((tokens, next_tokens)			,     dim=1)
										__lowerCAmelCase :    Tuple				=			generated[next_tokens_source]
										__lowerCAmelCase :    List[Any]				=			scores_sum_average * seq_lengths
										__lowerCAmelCase :    Any				=			is_stopped[next_tokens_source]

							__lowerCAmelCase :    str				=			self.transformer.transformer.wte(next_tokens.squeeze()).view(generated.shape[0]			,     1			,     -1)
							__lowerCAmelCase :    Optional[int]				=			torch.cat((generated, next_token_embed)			,     dim=1)
							__lowerCAmelCase :    Optional[int]				=			is_stopped + next_tokens.eq(_SCREAMING_SNAKE_CASE).squeeze()
							if is_stopped.all():
										break

				__lowerCAmelCase :    Optional[int]				=			scores / seq_lengths
				__lowerCAmelCase :    Optional[int]				=			scores.argsort(descending=_SCREAMING_SNAKE_CASE)
				# tokens tensors are already padded to max_seq_length
				__lowerCAmelCase :    List[Any]				=			[tokens[i] for i in order]
				__lowerCAmelCase :    Tuple				=			torch.stack(_SCREAMING_SNAKE_CASE			,     dim=0)
				__lowerCAmelCase :    Optional[int]				=			torch.tensor([seq_lengths[i] for i in order]			,     dtype=seq_lengths.dtype)
				return output_texts, seq_lengths




"""simple docstring"""






def      _lowercase							(    __snake_case      )		->  float:
			return 10 - x * x


def      _lowercase							(    __snake_case   ,__snake_case      )		->  float:
			# Bolzano theory in order to find if there is a root between a and b
			if equation(__snake_case      ) * equation(__snake_case      ) >= 0:
						raise ValueError("Wrong space!"      )

			__lowerCAmelCase :    List[Any]				=			a
			while (b - a) >= 0.01:
						# Find middle point
						__lowerCAmelCase :    str				=			(a + b) / 2
						# Check if middle point is root
						if equation(__snake_case      ) == 0.0:
									break
						# Decide the side to repeat the steps
						if equation(__snake_case      ) * equation(__snake_case      ) < 0:
									__lowerCAmelCase :    List[Any]				=			c
						else:
									__lowerCAmelCase :    List[str]				=			c
			return c


if __name__ == "__main__":
	import doctest

	doctest.testmod()

	print(bisection(-2, 5))
	print(bisection(0, 6))