

from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
						from fairscale.optim import OSS


A__						=							logging.get_logger(__name__)

A__						=							{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
}

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):







	def __init__(   self: Dict  , __UpperCamelCase: Any=None  , __UpperCamelCase: List[Any]=None  , *__UpperCamelCase: int  , **__UpperCamelCase: Any      ):

						'''simple docstring'''





						super().__init__(*__UpperCamelCase  , **__UpperCamelCase      )

						if config is None:
											assert isinstance(self.model  , __UpperCamelCase      ), (
											    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
											    F' {self.model.__class__}'
											)
											__magic_name__							=    self.model.config
						else:
											__magic_name__							=    config

						__magic_name__							=    data_args
						__magic_name__							=    self.config.tgt_vocab_size if isinstance(self.config  , __UpperCamelCase      ) else self.config.vocab_size

						if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
											assert self.config.pad_token_id is not None, (
											    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
											    " calculation or doing label smoothing."
											)

						if self.config.pad_token_id is None and self.config.eos_token_id is not None:
											logger.warning(
											    F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
											    ' padding..'      )

						if self.args.label_smoothing == 0:
											__magic_name__							=    torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id      )
						else:
											# dynamically import label_smoothed_nll_loss
											from utils import label_smoothed_nll_loss

											__magic_name__							=    label_smoothed_nll_loss







	def 	_SCREAMING_SNAKE_CASE							(   self: str  , __UpperCamelCase: int      ):

						'''simple docstring'''





						if self.optimizer is None:
											__magic_name__							=    ['bias', 'LayerNorm.weight']
											__magic_name__							=    [
											    {
											        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay      )],
											        'weight_decay': self.args.weight_decay,
											    },
											    {
											        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay      )],
											        'weight_decay': 0.0,
											    },
											]
											__magic_name__							=    Adafactor if self.args.adafactor else AdamW
											if self.args.adafactor:
																__magic_name__							=    Adafactor
																__magic_name__							=    {'scale_parameter': False, 'relative_step': False}
											else:
																__magic_name__							=    AdamW
																__magic_name__							=    {
																    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																    'eps': self.args.adam_epsilon,
																}
											__magic_name__							=    self.args.learning_rate
											if self.sharded_ddp:
																__magic_name__							=    OSS(
																    params=__UpperCamelCase  , optim=__UpperCamelCase  , **__UpperCamelCase  , )
											else:
																__magic_name__							=    optimizer_cls(__UpperCamelCase  , **__UpperCamelCase      )

						if self.lr_scheduler is None:
											__magic_name__							=    self._get_lr_scheduler(__UpperCamelCase      )
						else:  # ignoring --lr_scheduler
											logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'      )







	def 	_SCREAMING_SNAKE_CASE							(   self: str  , __UpperCamelCase: List[str]      ):

						'''simple docstring'''





						__magic_name__							=    arg_to_scheduler[self.args.lr_scheduler]
						if self.args.lr_scheduler == "constant":
											__magic_name__							=    schedule_func(self.optimizer      )
						elif self.args.lr_scheduler == "constant_w_warmup":
											__magic_name__							=    schedule_func(self.optimizer  , num_warmup_steps=self.args.warmup_steps      )
						else:
											__magic_name__							=    schedule_func(
											    self.optimizer  , num_warmup_steps=self.args.warmup_steps  , num_training_steps=__UpperCamelCase      )
						return scheduler







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple      ):

						'''simple docstring'''





						if isinstance(self.train_dataset  , torch.utils.data.IterableDataset      ):
											return None
						elif is_torch_tpu_available():
											return get_tpu_sampler(self.train_dataset      )
						else:
											if self.args.sortish_sampler:
																self.train_dataset.make_sortish_sampler(
																    self.args.per_device_train_batch_size  , distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)  , )

											return (
											    RandomSampler(self.train_dataset      )
											    if self.args.local_rank == -1
											    else DistributedSampler(self.train_dataset      )
											)







	def 	_SCREAMING_SNAKE_CASE							(   self: Dict  , __UpperCamelCase: int  , __UpperCamelCase: str  , __UpperCamelCase: str      ):

						'''simple docstring'''





						if self.args.label_smoothing == 0:
											if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																# force training to ignore pad token
																__magic_name__							=    model(**__UpperCamelCase  , use_cache=__UpperCamelCase      )[0]
																__magic_name__							=    self.loss_fn(logits.view(-1  , logits.shape[-1]      )  , labels.view(-1      )      )
											else:
																# compute usual loss via models
																__magic_name__,	__magic_name__							=    model(**__UpperCamelCase  , labels=__UpperCamelCase  , use_cache=__UpperCamelCase      )[:2]
						else:
											# compute label smoothed loss
											__magic_name__							=    model(**__UpperCamelCase  , use_cache=__UpperCamelCase      )[0]
											__magic_name__							=    torch.nn.functional.log_softmax(__UpperCamelCase  , dim=-1      )
											__magic_name__,	__magic_name__							=    self.loss_fn(__UpperCamelCase  , __UpperCamelCase  , self.args.label_smoothing  , ignore_index=self.config.pad_token_id      )
						return loss, logits







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: Dict  , __UpperCamelCase: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    inputs.pop('labels'      )
						__magic_name__,	__magic_name__							=    self._compute_loss(__UpperCamelCase  , __UpperCamelCase  , __UpperCamelCase      )
						return loss







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: nn.Module  , __UpperCamelCase: Dict[str, Union[torch.Tensor, Any]]  , __UpperCamelCase: bool  , __UpperCamelCase: Optional[List[str]] = None  , ):

						'''simple docstring'''





						__magic_name__							=    self._prepare_inputs(__UpperCamelCase      )

						__magic_name__							=    {
						    'max_length': self.data_args.val_max_target_length
						    if self.data_args is not None
						    else self.config.max_length,
						    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
						}

						if self.args.predict_with_generate and not self.args.prediction_loss_only:
											__magic_name__							=    self.model.generate(
											    inputs['input_ids']  , attention_mask=inputs['attention_mask']  , **__UpperCamelCase  , )
											# in case the batch is shorter than max length, the output should be padded
											if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																__magic_name__							=    self._pad_tensors_to_max_len(__UpperCamelCase  , gen_kwargs['max_length']      )

						__magic_name__							=    inputs.pop('labels'      )
						with torch.no_grad():
											# compute loss on predict data
											__magic_name__,	__magic_name__							=    self._compute_loss(__UpperCamelCase  , __UpperCamelCase  , __UpperCamelCase      )

						__magic_name__							=    loss.mean().detach()
						if self.args.prediction_loss_only:
											return (loss, None, None)

						__magic_name__							=    generated_tokens if self.args.predict_with_generate else logits

						if labels.shape[-1] < gen_kwargs["max_length"]:
											__magic_name__							=    self._pad_tensors_to_max_len(__UpperCamelCase  , gen_kwargs['max_length']      )

						return (loss, logits, labels)







	def 	_SCREAMING_SNAKE_CASE							(   self: Dict  , __UpperCamelCase: List[str]  , __UpperCamelCase: Dict      ):

						'''simple docstring'''





						__magic_name__							=    self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

						if pad_token_id is None:
											raise ValueError(
											    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
											    F' padded to `max_length`={max_length}'      )

						__magic_name__							=    pad_token_id * torch.ones(
						    (tensor.shape[0], max_length)  , dtype=tensor.dtype  , device=tensor.device      )
						__magic_name__							=    tensor
						return padded_tensor







from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__						=							logging.get_logger(__name__)

A__						=							{
    "huggingface/time-series-transformer-tourism-monthly": (
        "https://huggingface.co/huggingface/time-series-transformer-tourism-monthly/resolve/main/config.json"
    ),
    # See all TimeSeriesTransformer models at https://huggingface.co/models?filter=time_series_transformer
}

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   Optional[int]       				=  "time_series_transformer"
	_lowercase  :   Dict       				=  {
	    "hidden_size": "d_model",
	    "num_attention_heads": "encoder_attention_heads",
	    "num_hidden_layers": "encoder_layers",
	}







	def __init__(   self: str  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: str = "student_t"  , __UpperCamelCase: str = "nll"  , __UpperCamelCase: int = 1  , __UpperCamelCase: List[int] = [1, 2, 3, 4, 5, 6, 7]  , __UpperCamelCase: Optional[Union[str, bool]] = "mean"  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: Optional[List[int]] = None  , __UpperCamelCase: Optional[List[int]] = None  , __UpperCamelCase: int = 32  , __UpperCamelCase: int = 32  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: bool = True  , __UpperCamelCase: str = "gelu"  , __UpperCamelCase: int = 64  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: int = 1_00  , __UpperCamelCase: float = 0.02  , __UpperCamelCase: Any=True  , **__UpperCamelCase: Optional[Any]  , ):

						'''simple docstring'''





						__magic_name__							=    prediction_length
						__magic_name__							=    context_length or prediction_length
						__magic_name__							=    distribution_output
						__magic_name__							=    loss
						__magic_name__							=    input_size
						__magic_name__							=    num_time_features
						__magic_name__							=    lags_sequence
						__magic_name__							=    scaling
						__magic_name__							=    num_dynamic_real_features
						__magic_name__							=    num_static_real_features
						__magic_name__							=    num_static_categorical_features
						if cardinality and num_static_categorical_features > 0:
											if len(__UpperCamelCase      ) != num_static_categorical_features:
																raise ValueError(
																    'The cardinality should be a list of the same length as `num_static_categorical_features`'      )
											__magic_name__							=    cardinality
						else:
											__magic_name__							=    [0]
						if embedding_dimension and num_static_categorical_features > 0:
											if len(__UpperCamelCase      ) != num_static_categorical_features:
																raise ValueError(
																    'The embedding dimension should be a list of the same length as `num_static_categorical_features`'      )
											__magic_name__							=    embedding_dimension
						else:
											__magic_name__							=    [min(50  , (cat + 1) // 2      ) for cat in self.cardinality]
						__magic_name__							=    num_parallel_samples

						# Transformer architecture configuration
						__magic_name__							=    input_size * len(__UpperCamelCase      ) + self._number_of_features
						__magic_name__							=    d_model
						__magic_name__							=    encoder_attention_heads
						__magic_name__							=    decoder_attention_heads
						__magic_name__							=    encoder_ffn_dim
						__magic_name__							=    decoder_ffn_dim
						__magic_name__							=    encoder_layers
						__magic_name__							=    decoder_layers

						__magic_name__							=    dropout
						__magic_name__							=    attention_dropout
						__magic_name__							=    activation_dropout
						__magic_name__							=    encoder_layerdrop
						__magic_name__							=    decoder_layerdrop

						__magic_name__							=    activation_function
						__magic_name__							=    init_std

						__magic_name__							=    use_cache

						super().__init__(is_encoder_decoder=__UpperCamelCase  , **__UpperCamelCase      )







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: str      ):

						'''simple docstring'''





						return (
						    sum(self.embedding_dimension      )
						    + self.num_dynamic_real_features
						    + self.num_time_features
						    + self.num_static_real_features
						    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
						)




