




"""simple docstring"""







from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class       SCREAMING_SNAKE_CASE__					(					_a						,				_a						,				_a					):
						_a    						=						[r'h\.\d+\.attn\.bias', r'h\.\d+\.attn\.masked_bias']


						@register_to_config
						def __init__(							self				:			Optional[int]						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			Optional[int] = None						,						lowerCAmelCase				:			int = 5_0257						,						lowerCAmelCase				:			int = 1024						,						lowerCAmelCase				:			int = 768						,						lowerCAmelCase				:			int = 12						,						lowerCAmelCase				:			int = 12						,						lowerCAmelCase				:			Optional[int] = None						,						lowerCAmelCase				:			str = "gelu_new"						,						lowerCAmelCase				:			float = 0.1						,						lowerCAmelCase				:			float = 0.1						,						lowerCAmelCase				:			float = 0.1						,						lowerCAmelCase				:			float = 1e-5						,						lowerCAmelCase				:			float = 0.02						,						lowerCAmelCase				:			bool = True						,						lowerCAmelCase				:			bool = True						,						lowerCAmelCase				:			bool = False						,						lowerCAmelCase				:			bool = False						,						):
										super().__init__()

										lowerCAmelCase  =				prefix_length

										if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
														raise ValueError(
														    f'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
														    f''' `n_embd`: {n_embd} are not equal.'''					)

										lowerCAmelCase  =				prefix_inner_dim
										lowerCAmelCase  =				prefix_hidden_dim

										lowerCAmelCase  =				(
										    nn.Linear(self.prefix_inner_dim						,						self.prefix_hidden_dim					)
										    if self.prefix_hidden_dim is not None
										    else nn.Identity()
										)
										lowerCAmelCase  =				(
										    nn.Linear(self.prefix_hidden_dim						,						UpperCamelCase_					) if self.prefix_hidden_dim is not None else nn.Identity()
										)

										lowerCAmelCase  =				GPTaConfig(
										    vocab_size=UpperCamelCase_						,						n_positions=UpperCamelCase_						,						n_embd=UpperCamelCase_						,						n_layer=UpperCamelCase_						,						n_head=UpperCamelCase_						,						n_inner=UpperCamelCase_						,						activation_function=UpperCamelCase_						,						resid_pdrop=UpperCamelCase_						,						embd_pdrop=UpperCamelCase_						,						attn_pdrop=UpperCamelCase_						,						layer_norm_epsilon=UpperCamelCase_						,						initializer_range=UpperCamelCase_						,						scale_attn_weights=UpperCamelCase_						,						use_cache=UpperCamelCase_						,						scale_attn_by_inverse_layer_idx=UpperCamelCase_						,						reorder_and_upcast_attn=UpperCamelCase_						,						)
										lowerCAmelCase  =				GPTaLMHeadModel(UpperCamelCase_					)


						def   __lowercase (							self				:			str						,						lowerCAmelCase				:			torch.Tensor						,						lowerCAmelCase				:			torch.Tensor						,						lowerCAmelCase				:			Optional[torch.Tensor] = None						,						lowerCAmelCase				:			Optional[torch.Tensor] = None						,						):
										lowerCAmelCase  =				self.transformer.transformer.wte(UpperCamelCase_					)
										lowerCAmelCase  =				self.encode_prefix(UpperCamelCase_					)
										lowerCAmelCase  =				self.decode_prefix(UpperCamelCase_					)
										lowerCAmelCase  =				torch.cat((prefix_embeds, embedding_text)						,						dim=1					)

										if labels is not None:
														lowerCAmelCase  =				self.get_dummy_token(input_ids.shape[0]						,						input_ids.device					)
														lowerCAmelCase  =				torch.cat((dummy_token, input_ids)						,						dim=1					)
										lowerCAmelCase  =				self.transformer(inputs_embeds=UpperCamelCase_						,						labels=UpperCamelCase_						,						attention_mask=UpperCamelCase_					)
										if self.prefix_hidden_dim is not None:
														return out, hidden
										else:
														return out


						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			torch.device					):
										return torch.zeros(UpperCamelCase_						,						self.prefix_length						,						dtype=torch.intaa						,						device=UpperCamelCase_					)


						def   __lowercase (							self				:			int						,						lowerCAmelCase				:			Optional[Any]					):
										return self.encode_prefix(UpperCamelCase_					)


						@torch.no_grad()
						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			Optional[Any]						,						lowerCAmelCase				:			Optional[int]						,						lowerCAmelCase				:			Union[str, Any]					):
										lowerCAmelCase  =				torch.split(UpperCamelCase_						,						1						,						dim=0					)
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										for feature in features:
														lowerCAmelCase  =				self.decode_prefix(feature.to(UpperCamelCase_					)					)  # back to the clip feature
														# Only support beam search for now
														lowerCAmelCase  =				self.generate_beam(
														    input_embeds=UpperCamelCase_						,						device=UpperCamelCase_						,						eos_token_id=UpperCamelCase_					)
														generated_tokens.append(output_tokens[0]					)
														generated_seq_lengths.append(seq_lengths[0]					)
										lowerCAmelCase  =				torch.stack(UpperCamelCase_					)
										lowerCAmelCase  =				torch.stack(UpperCamelCase_					)
										return generated_tokens, generated_seq_lengths




						@torch.no_grad()
						def   __lowercase (							self				:			Any						,						lowerCAmelCase				:			int=None						,						lowerCAmelCase				:			int=None						,						lowerCAmelCase				:			Optional[int]=None						,						lowerCAmelCase				:			int = 5						,						lowerCAmelCase				:			int = 67						,						lowerCAmelCase				:			float = 1.0						,						lowerCAmelCase				:			Optional[int] = None						,						):
										lowerCAmelCase  =				eos_token_id
										lowerCAmelCase  =				None
										lowerCAmelCase  =				None
										lowerCAmelCase  =				torch.ones(UpperCamelCase_						,						device=UpperCamelCase_						,						dtype=torch.int					)
										lowerCAmelCase  =				torch.zeros(UpperCamelCase_						,						device=UpperCamelCase_						,						dtype=torch.bool					)

										if input_embeds is not None:
														lowerCAmelCase  =				input_embeds
										else:
														lowerCAmelCase  =				self.transformer.transformer.wte(UpperCamelCase_					)

										for i in range(UpperCamelCase_					):
														lowerCAmelCase  =				self.transformer(inputs_embeds=UpperCamelCase_					)
														lowerCAmelCase  =				outputs.logits
														lowerCAmelCase  =				logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
														lowerCAmelCase  =				logits.softmax(-1					).log()

														if scores is None:
																		lowerCAmelCase  =				logits.topk(UpperCamelCase_						,						-1					)
																		lowerCAmelCase  =				generated.expand(UpperCamelCase_						,						*generated.shape[1:]					)
																		lowerCAmelCase  =				next_tokens.permute(1						,						0					), scores.squeeze(0					)
																		if tokens is None:
																						lowerCAmelCase  =				next_tokens
																		else:
																						lowerCAmelCase  =				tokens.expand(UpperCamelCase_						,						*tokens.shape[1:]					)
																						lowerCAmelCase  =				torch.cat((tokens, next_tokens)						,						dim=1					)
														else:
																		lowerCAmelCase  =				-float(np.inf					)
																		lowerCAmelCase  =				0
																		lowerCAmelCase  =				scores[:, None] + logits
																		seq_lengths[~is_stopped] += 1
																		lowerCAmelCase  =				scores_sum / seq_lengths[:, None]
																		lowerCAmelCase  =				scores_sum_average.view(-1					).topk(UpperCamelCase_						,						-1					)
																		lowerCAmelCase  =				next_tokens // scores_sum.shape[1]
																		lowerCAmelCase  =				seq_lengths[next_tokens_source]
																		lowerCAmelCase  =				next_tokens % scores_sum.shape[1]
																		lowerCAmelCase  =				next_tokens.unsqueeze(1					)
																		lowerCAmelCase  =				tokens[next_tokens_source]
																		lowerCAmelCase  =				torch.cat((tokens, next_tokens)						,						dim=1					)
																		lowerCAmelCase  =				generated[next_tokens_source]
																		lowerCAmelCase  =				scores_sum_average * seq_lengths
																		lowerCAmelCase  =				is_stopped[next_tokens_source]

														lowerCAmelCase  =				self.transformer.transformer.wte(next_tokens.squeeze()					).view(generated.shape[0]						,						1						,						-1					)
														lowerCAmelCase  =				torch.cat((generated, next_token_embed)						,						dim=1					)
														lowerCAmelCase  =				is_stopped + next_tokens.eq(UpperCamelCase_					).squeeze()
														if is_stopped.all():
																		break

										lowerCAmelCase  =				scores / seq_lengths
										lowerCAmelCase  =				scores.argsort(descending=UpperCamelCase_					)
										# tokens tensors are already padded to max_seq_length
										lowerCAmelCase  =				[tokens[i] for i in order]
										lowerCAmelCase  =				torch.stack(UpperCamelCase_						,						dim=0					)
										lowerCAmelCase  =				torch.tensor([seq_lengths[i] for i in order]						,						dtype=seq_lengths.dtype					)
										return output_texts, seq_lengths











"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a      =							{
    'configuration_time_series_transformer': [
        'TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'TimeSeriesTransformerConfig',
    ],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a      =							[
		    'TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TimeSeriesTransformerForPrediction',
		    'TimeSeriesTransformerModel',
		    'TimeSeriesTransformerPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_time_series_transformer import (
		    TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    TimeSeriesTransformerConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_time_series_transformer import (
				    TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TimeSeriesTransformerForPrediction,
				    TimeSeriesTransformerModel,
				    TimeSeriesTransformerPreTrainedModel,
				)

else:
		import sys

		a      =							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





