



'''simple docstring'''






import torch
from torch import nn

class        SCREAMING_SNAKE_CASE(							nn.Module    ):







 """simple docstring"""






 def __init__(		self    :  Dict	,    __snake_case    :  Union[str, Any]	,    __snake_case    :  Tuple	,    __snake_case    :  Dict	,    __snake_case    :  str	,    __snake_case    :  int=1	,    __snake_case    :  str=False	)    ->	str:
        super().__init__()

        UpperCAmelCase				: Optional[Any]											=	n_token
        UpperCAmelCase				: Tuple											=	d_embed
        UpperCAmelCase				: List[Any]											=	d_proj

        UpperCAmelCase				: Optional[int]											=	cutoffs + [n_token]
        UpperCAmelCase				: Any											=	[0] + self.cutoffs
        UpperCAmelCase				: Tuple											=	div_val

        UpperCAmelCase				: List[Any]											=	self.cutoffs[0]
        UpperCAmelCase				: Optional[Any]											=	len(self.cutoffs	) - 1
        UpperCAmelCase				: List[str]											=	self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
               UpperCAmelCase				: Optional[int]											=	nn.Parameter(torch.zeros(self.n_clusters	,    self.d_embed	)	)
               UpperCAmelCase				: List[str]											=	nn.Parameter(torch.zeros(self.n_clusters	)	)

        UpperCAmelCase				: List[str]											=	nn.ModuleList()
        UpperCAmelCase				: Tuple											=	nn.ParameterList()

        if div_val == 1:
               for i in range(len(self.cutoffs	)	):
                      if d_proj != d_embed:
                             self.out_projs.append(nn.Parameter(torch.FloatTensor(__snake_case	,    __snake_case	)	)	)
                      else:
                             self.out_projs.append(__snake_case	)

               self.out_layers.append(nn.Linear(__snake_case	,    __snake_case	)	)
        else:
               for i in range(len(self.cutoffs	)	):
                      UpperCAmelCase				: int											=	self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      UpperCAmelCase				: Tuple											=	d_embed // (div_val**i)

                      self.out_projs.append(nn.Parameter(torch.FloatTensor(__snake_case	,    __snake_case	)	)	)

                      self.out_layers.append(nn.Linear(__snake_case	,    r_idx - l_idx	)	)

        UpperCAmelCase				: Dict											=	keep_order







 def   A			(		self    :  List[str]	,    __snake_case    :  Optional[Any]	,    __snake_case    :  Tuple	,    __snake_case    :  List[Any]	,    __snake_case    :  Union[str, Any]	)    ->	Optional[int]:
        if proj is None:
               UpperCAmelCase				: Optional[int]											=	nn.functional.linear(__snake_case	,    __snake_case	,    bias=__snake_case	)
        else:
               # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
               UpperCAmelCase				: Dict											=	nn.functional.linear(__snake_case	,    proj.t().contiguous()	)
               UpperCAmelCase				: Optional[Any]											=	nn.functional.linear(__snake_case	,    __snake_case	,    bias=__snake_case	)
               # else:
               #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
               #     if bias is not None:
               #         logit = logit + bias

        return logit







 def   A			(		self    :  Optional[Any]	,    __snake_case    :  Dict	,    __snake_case    :  Union[str, Any]=None	,    __snake_case    :  Optional[Any]=False	)    ->	Optional[int]:

        if labels is not None:
               # Shift so that tokens < n predict n
               UpperCAmelCase				: Tuple											=	hidden[..., :-1, :].contiguous()
               UpperCAmelCase				: List[str]											=	labels[..., 1:].contiguous()
               UpperCAmelCase				: Optional[Any]											=	hidden.view(-1	,    hidden.size(-1	)	)
               UpperCAmelCase				: Optional[int]											=	labels.view(-1	)
               if hidden.size(0	) != labels.size(0	):
                      raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''	)
        else:
               UpperCAmelCase				: int											=	hidden.view(-1	,    hidden.size(-1	)	)

        if self.n_clusters == 0:
               UpperCAmelCase				: Tuple											=	self._compute_logit(__snake_case	,    self.out_layers[0].weight	,    self.out_layers[0].bias	,    self.out_projs[0]	)
               if labels is not None:
                      UpperCAmelCase				: List[str]											=	labels != -100
                      UpperCAmelCase				: str											=	torch.zeros_like(__snake_case	,    dtype=hidden.dtype	,    device=hidden.device	)
                      UpperCAmelCase				: List[str]											=	(
                          -nn.functional.log_softmax(__snake_case	,    dim=-1	)[mask].gather(1	,    labels[mask].unsqueeze(1	)	).squeeze(1	)
                      )
               else:
                      UpperCAmelCase				: Union[str, Any]											=	nn.functional.log_softmax(__snake_case	,    dim=-1	)
        else:
               # construct weights and biases
               UpperCAmelCase				: int											=	[], []
               for i in range(len(self.cutoffs	)	):
                      if self.div_val == 1:
                             UpperCAmelCase				: Any											=	self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCAmelCase				: Dict											=	self.out_layers[0].weight[l_idx:r_idx]
                             UpperCAmelCase				: Optional[Any]											=	self.out_layers[0].bias[l_idx:r_idx]
                      else:
                             UpperCAmelCase				: int											=	self.out_layers[i].weight
                             UpperCAmelCase				: str											=	self.out_layers[i].bias

                      if i == 0:
                             UpperCAmelCase				: Optional[int]											=	torch.cat([weight_i, self.cluster_weight]	,    dim=0	)
                             UpperCAmelCase				: Union[str, Any]											=	torch.cat([bias_i, self.cluster_bias]	,    dim=0	)

                      weights.append(__snake_case	)
                      biases.append(__snake_case	)

               UpperCAmelCase				: Any											=	weights[0], biases[0], self.out_projs[0]

               UpperCAmelCase				: Union[str, Any]											=	self._compute_logit(__snake_case	,    __snake_case	,    __snake_case	,    __snake_case	)
               UpperCAmelCase				: Tuple											=	nn.functional.log_softmax(__snake_case	,    dim=1	)

               if labels is None:
                      UpperCAmelCase				: List[Any]											=	hidden.new_empty((head_logit.size(0	), self.n_token)	)
               else:
                      UpperCAmelCase				: Union[str, Any]											=	torch.zeros_like(__snake_case	,    dtype=hidden.dtype	,    device=hidden.device	)

               UpperCAmelCase				: List[Any]											=	0
               UpperCAmelCase				: int											=	[0] + self.cutoffs
               for i in range(len(__snake_case	) - 1	):
                      UpperCAmelCase				: Any											=	cutoff_values[i], cutoff_values[i + 1]

                      if labels is not None:
                             UpperCAmelCase				: Optional[int]											=	(labels >= l_idx) & (labels < r_idx)
                             UpperCAmelCase				: Union[str, Any]											=	mask_i.nonzero().squeeze()

                             if indices_i.numel() == 0:
                                    continue

                             UpperCAmelCase				: Optional[Any]											=	labels.index_select(0	,    __snake_case	) - l_idx
                             UpperCAmelCase				: Optional[int]											=	head_logprob.index_select(0	,    __snake_case	)
                             UpperCAmelCase				: Optional[Any]											=	hidden.index_select(0	,    __snake_case	)
                      else:
                             UpperCAmelCase				: str											=	hidden

                      if i == 0:
                             if labels is not None:
                                    UpperCAmelCase				: int											=	head_logprob_i.gather(1	,    target_i[:, None]	).squeeze(1	)
                             else:
                                    UpperCAmelCase				: List[str]											=	head_logprob[:, : self.cutoffs[0]]
                      else:
                             UpperCAmelCase				: int											=	weights[i], biases[i], self.out_projs[i]

                             UpperCAmelCase				: Union[str, Any]											=	self._compute_logit(__snake_case	,    __snake_case	,    __snake_case	,    __snake_case	)
                             UpperCAmelCase				: List[Any]											=	nn.functional.log_softmax(__snake_case	,    dim=1	)
                             UpperCAmelCase				: Dict											=	self.cutoffs[0] + i - 1  # No probability for the head cluster
                             if labels is not None:
                                    UpperCAmelCase				: Tuple											=	head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                        1	,    target_i[:, None]	).squeeze(1	)
                             else:
                                    UpperCAmelCase				: str											=	head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                    UpperCAmelCase				: int											=	logprob_i

                      if labels is not None:
                             if (hasattr(self	,    '''keep_order'''	) and self.keep_order) or keep_order:
                                    out.index_copy_(0	,    __snake_case	,    -logprob_i	)
                             else:
                                    out[offset : offset + logprob_i.size(0	)].copy_(-logprob_i	)
                             offset += logprob_i.size(0	)

        return out







 def   A			(		self    :  str	,    __snake_case    :  int	)    ->	Optional[int]:
        if self.n_clusters == 0:
               UpperCAmelCase				: List[Any]											=	self._compute_logit(__snake_case	,    self.out_layers[0].weight	,    self.out_layers[0].bias	,    self.out_projs[0]	)
               return nn.functional.log_softmax(__snake_case	,    dim=-1	)
        else:
               # construct weights and biases
               UpperCAmelCase				: Any											=	[], []
               for i in range(len(self.cutoffs	)	):
                      if self.div_val == 1:
                             UpperCAmelCase				: Union[str, Any]											=	self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCAmelCase				: List[Any]											=	self.out_layers[0].weight[l_idx:r_idx]
                             UpperCAmelCase				: Optional[Any]											=	self.out_layers[0].bias[l_idx:r_idx]
                      else:
                             UpperCAmelCase				: Optional[Any]											=	self.out_layers[i].weight
                             UpperCAmelCase				: Optional[int]											=	self.out_layers[i].bias

                      if i == 0:
                             UpperCAmelCase				: List[Any]											=	torch.cat([weight_i, self.cluster_weight]	,    dim=0	)
                             UpperCAmelCase				: List[str]											=	torch.cat([bias_i, self.cluster_bias]	,    dim=0	)

                      weights.append(__snake_case	)
                      biases.append(__snake_case	)

               UpperCAmelCase				: Any											=	weights[0], biases[0], self.out_projs[0]
               UpperCAmelCase				: Union[str, Any]											=	self._compute_logit(__snake_case	,    __snake_case	,    __snake_case	,    __snake_case	)

               UpperCAmelCase				: str											=	hidden.new_empty((head_logit.size(0	), self.n_token)	)
               UpperCAmelCase				: Dict											=	nn.functional.log_softmax(__snake_case	,    dim=1	)

               UpperCAmelCase				: str											=	[0] + self.cutoffs
               for i in range(len(__snake_case	) - 1	):
                      UpperCAmelCase				: str											=	cutoff_values[i], cutoff_values[i + 1]

                      if i == 0:
                             UpperCAmelCase				: Any											=	head_logprob[:, : self.cutoffs[0]]
                      else:
                             UpperCAmelCase				: Tuple											=	weights[i], biases[i], self.out_projs[i]

                             UpperCAmelCase				: Union[str, Any]											=	self._compute_logit(__snake_case	,    __snake_case	,    __snake_case	,    __snake_case	)
                             UpperCAmelCase				: List[Any]											=	nn.functional.log_softmax(__snake_case	,    dim=1	)

                             UpperCAmelCase				: str											=	head_logprob[:, -i] + tail_logprob_i
                             UpperCAmelCase				: Optional[int]											=	logprob_i

               return out










'''simple docstring'''






from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__:     Optional[Any]										=    logging.get_logger(__name__)

UpperCamelCase__:     Tuple										=    {
    "huggingface/time-series-transformer-tourism-monthly": (
        "https://huggingface.co/huggingface/time-series-transformer-tourism-monthly/resolve/main/config.json"
    ),
    # See all TimeSeriesTransformer models at https://huggingface.co/models?filter=time_series_transformer
}

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	lowerCamelCase__    				= """time_series_transformer"""
	lowerCamelCase__    				= {
	    """hidden_size""": """d_model""",
	    """num_attention_heads""": """encoder_attention_heads""",
	    """num_hidden_layers""": """encoder_layers""",
	}







	def __init__(		self    :  str	,    __snake_case    :  Optional[int] = None	,    __snake_case    :  Optional[int] = None	,    __snake_case    :  str = "student_t"	,    __snake_case    :  str = "nll"	,    __snake_case    :  int = 1	,    __snake_case    :  List[int] = [1, 2, 3, 4, 5, 6, 7]	,    __snake_case    :  Optional[Union[str, bool]] = "mean"	,    __snake_case    :  int = 0	,    __snake_case    :  int = 0	,    __snake_case    :  int = 0	,    __snake_case    :  int = 0	,    __snake_case    :  Optional[List[int]] = None	,    __snake_case    :  Optional[List[int]] = None	,    __snake_case    :  int = 32	,    __snake_case    :  int = 32	,    __snake_case    :  int = 2	,    __snake_case    :  int = 2	,    __snake_case    :  int = 2	,    __snake_case    :  int = 2	,    __snake_case    :  bool = True	,    __snake_case    :  str = "gelu"	,    __snake_case    :  int = 64	,    __snake_case    :  float = 0.1	,    __snake_case    :  float = 0.1	,    __snake_case    :  float = 0.1	,    __snake_case    :  float = 0.1	,    __snake_case    :  float = 0.1	,    __snake_case    :  int = 100	,    __snake_case    :  float = 0.02	,    __snake_case    :  Optional[Any]=True	,    **__snake_case    :  List[Any]	,    )    ->	List[str]:
								# time series specific configuration
								UpperCAmelCase				: List[Any]											=	prediction_length
								UpperCAmelCase				: List[Any]											=	context_length or prediction_length
								UpperCAmelCase				: Tuple											=	distribution_output
								UpperCAmelCase				: Optional[Any]											=	loss
								UpperCAmelCase				: Tuple											=	input_size
								UpperCAmelCase				: Optional[int]											=	num_time_features
								UpperCAmelCase				: Dict											=	lags_sequence
								UpperCAmelCase				: Any											=	scaling
								UpperCAmelCase				: Tuple											=	num_dynamic_real_features
								UpperCAmelCase				: Any											=	num_static_real_features
								UpperCAmelCase				: Optional[int]											=	num_static_categorical_features
								if cardinality and num_static_categorical_features > 0:
															if len(__snake_case	) != num_static_categorical_features:
																						raise ValueError(
																						    '''The cardinality should be a list of the same length as `num_static_categorical_features`'''	)
															UpperCAmelCase				: Any											=	cardinality
								else:
															UpperCAmelCase				: Optional[Any]											=	[0]
								if embedding_dimension and num_static_categorical_features > 0:
															if len(__snake_case	) != num_static_categorical_features:
																						raise ValueError(
																						    '''The embedding dimension should be a list of the same length as `num_static_categorical_features`'''	)
															UpperCAmelCase				: Optional[int]											=	embedding_dimension
								else:
															UpperCAmelCase				: Optional[Any]											=	[min(50	,    (cat + 1) // 2	) for cat in self.cardinality]
								UpperCAmelCase				: List[Any]											=	num_parallel_samples

								# Transformer architecture configuration
								UpperCAmelCase				: int											=	input_size * len(__snake_case	) + self._number_of_features
								UpperCAmelCase				: int											=	d_model
								UpperCAmelCase				: str											=	encoder_attention_heads
								UpperCAmelCase				: str											=	decoder_attention_heads
								UpperCAmelCase				: List[str]											=	encoder_ffn_dim
								UpperCAmelCase				: Any											=	decoder_ffn_dim
								UpperCAmelCase				: Any											=	encoder_layers
								UpperCAmelCase				: str											=	decoder_layers

								UpperCAmelCase				: Optional[int]											=	dropout
								UpperCAmelCase				: Union[str, Any]											=	attention_dropout
								UpperCAmelCase				: List[Any]											=	activation_dropout
								UpperCAmelCase				: Optional[int]											=	encoder_layerdrop
								UpperCAmelCase				: Dict											=	decoder_layerdrop

								UpperCAmelCase				: Tuple											=	activation_function
								UpperCAmelCase				: Dict											=	init_std

								UpperCAmelCase				: Optional[int]											=	use_cache

								super().__init__(is_encoder_decoder=__snake_case	,    **__snake_case	)







	@property
	def   A			(		self    :  Union[str, Any]	)    ->	int:
								return (
								    sum(self.embedding_dimension	)
								    + self.num_dynamic_real_features
								    + self.num_time_features
								    + self.num_static_real_features
								    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
								)





