






from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
					from fairscale.optim import OSS


__snake_case         =     logging.get_logger(__name__)

__snake_case         =     {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}


class     __SCREAMING_SNAKE_CASE					(   _UpperCAmelCase):






						def __init__(	self     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												super().__init__(*lowercase__     ,						**lowercase__   )

												if config is None:
																		assert isinstance(self.model     ,						lowercase__   ), (
																		    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
																		    F" {self.model.__class__}"
																		)
																		lowerCAmelCase__    						=						self.model.config
												else:
																		lowerCAmelCase__    						=						config

												lowerCAmelCase__    						=						data_args
												lowerCAmelCase__    						=						self.config.tgt_vocab_size if isinstance(self.config     ,						lowercase__   ) else self.config.vocab_size

												if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
																		assert self.config.pad_token_id is not None, (
																		    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
																		    " calculation or doing label smoothing."
																		)

												if self.config.pad_token_id is None and self.config.eos_token_id is not None:
																		logger.warning(
																		    F"The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for"
																		    ' padding..'   )

												if self.args.label_smoothing == 0:
																		lowerCAmelCase__    						=						torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id   )
												else:
																		# dynamically import label_smoothed_nll_loss
																		from utils import label_smoothed_nll_loss

																		lowerCAmelCase__    						=						label_smoothed_nll_loss






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												if self.optimizer is None:
																		lowerCAmelCase__    						=						["""bias""", """LayerNorm.weight"""]
																		lowerCAmelCase__    						=						[
																		    {
																		        """params""": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay   )],
																		        """weight_decay""": self.args.weight_decay,
																		    },
																		    {
																		        """params""": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay   )],
																		        """weight_decay""": 0.0,
																		    },
																		]
																		lowerCAmelCase__    						=						Adafactor if self.args.adafactor else AdamW
																		if self.args.adafactor:
																								lowerCAmelCase__    						=						Adafactor
																								lowerCAmelCase__    						=						{"""scale_parameter""": False, """relative_step""": False}
																		else:
																								lowerCAmelCase__    						=						AdamW
																								lowerCAmelCase__    						=						{
																								    """betas""": (self.args.adam_betaa, self.args.adam_betaa),
																								    """eps""": self.args.adam_epsilon,
																								}
																		lowerCAmelCase__    						=						self.args.learning_rate
																		if self.sharded_ddp:
																								lowerCAmelCase__    						=						OSS(
																								    params=lowercase__     ,						optim=lowercase__     ,						**lowercase__     ,						)
																		else:
																								lowerCAmelCase__    						=						optimizer_cls(lowercase__     ,						**lowercase__   )

												if self.lr_scheduler is None:
																		lowerCAmelCase__    						=						self._get_lr_scheduler(lowercase__   )
												else:  # ignoring --lr_scheduler
																		logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						arg_to_scheduler[self.args.lr_scheduler]
												if self.args.lr_scheduler == "constant":
																		lowerCAmelCase__    						=						schedule_func(self.optimizer   )
												elif self.args.lr_scheduler == "constant_w_warmup":
																		lowerCAmelCase__    						=						schedule_func(self.optimizer     ,						num_warmup_steps=self.args.warmup_steps   )
												else:
																		lowerCAmelCase__    						=						schedule_func(
																		    self.optimizer     ,						num_warmup_steps=self.args.warmup_steps     ,						num_training_steps=lowercase__   )
												return scheduler






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												if isinstance(self.train_dataset     ,						torch.utils.data.IterableDataset   ):
																		return None
												elif is_torch_tpu_available():
																		return get_tpu_sampler(self.train_dataset   )
												else:
																		if self.args.sortish_sampler:
																								self.train_dataset.make_sortish_sampler(
																								    self.args.per_device_train_batch_size     ,						distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)     ,						)

																		return (
																		    RandomSampler(self.train_dataset   )
																		    if self.args.local_rank == -1
																		    else DistributedSampler(self.train_dataset   )
																		)






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






												"""simple docstring"""
												if self.args.label_smoothing == 0:
																		if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																								# force training to ignore pad token
																								lowerCAmelCase__    						=						model(**lowercase__     ,						use_cache=lowercase__   )[0]
																								lowerCAmelCase__    						=						self.loss_fn(logits.view(-1     ,						logits.shape[-1]   )     ,						labels.view(-1   )   )
																		else:
																								# compute usual loss via models
																								lowerCAmelCase__    						=						model(**lowercase__     ,						labels=lowercase__     ,						use_cache=lowercase__   )[:2]
												else:
																		# compute label smoothed loss
																		lowerCAmelCase__    						=						model(**lowercase__     ,						use_cache=lowercase__   )[0]
																		lowerCAmelCase__    						=						torch.nn.functional.log_softmax(lowercase__     ,						dim=-1   )
																		lowerCAmelCase__    						=						self.loss_fn(lowercase__     ,						lowercase__     ,						self.args.label_smoothing     ,						ignore_index=self.config.pad_token_id   )
												return loss, logits






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						inputs.pop('labels'   )
												lowerCAmelCase__    						=						self._compute_loss(lowercase__     ,						lowercase__     ,						lowercase__   )
												return loss






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase = None     ,						):






												"""simple docstring"""
												lowerCAmelCase__    						=						self._prepare_inputs(lowercase__   )

												lowerCAmelCase__    						=						{
												    """max_length""": self.data_args.val_max_target_length
												    if self.data_args is not None
												    else self.config.max_length,
												    """num_beams""": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
												}

												if self.args.predict_with_generate and not self.args.prediction_loss_only:
																		lowerCAmelCase__    						=						self.model.generate(
																		    inputs['input_ids']     ,						attention_mask=inputs['attention_mask']     ,						**lowercase__     ,						)
																		# in case the batch is shorter than max length, the output should be padded
																		if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																								lowerCAmelCase__    						=						self._pad_tensors_to_max_len(lowercase__     ,						gen_kwargs['max_length']   )

												lowerCAmelCase__    						=						inputs.pop('labels'   )
												with torch.no_grad():
																		# compute loss on predict data
																		lowerCAmelCase__    						=						self._compute_loss(lowercase__     ,						lowercase__     ,						lowercase__   )

												lowerCAmelCase__    						=						loss.mean().detach()
												if self.args.prediction_loss_only:
																		return (loss, None, None)

												lowerCAmelCase__    						=						generated_tokens if self.args.predict_with_generate else logits

												if labels.shape[-1] < gen_kwargs["max_length"]:
																		lowerCAmelCase__    						=						self._pad_tensors_to_max_len(lowercase__     ,						gen_kwargs['max_length']   )

												return (loss, logits, labels)







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






												"""simple docstring"""
												# If PAD token is not defined at least EOS token has to be defined
												lowerCAmelCase__    						=						self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

												if pad_token_id is None:
																		raise ValueError(
																		    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
																		    F" padded to `max_length`={max_length}"   )

												lowerCAmelCase__    						=						pad_token_id * torch.ones(
												    (tensor.shape[0], max_length)     ,						dtype=tensor.dtype     ,						device=tensor.device   )
												lowerCAmelCase__    						=						tensor
												return padded_tensor













def 	_UpperCamelCase   (    UpperCamelCase_       :					int		)  ->   int:

						"""simple docstring"""



						if not isinstance(UpperCamelCase_    ,       UpperCamelCase_		):
												raise ValueError('multiplicative_persistence() only accepts integral values'		)
						if num < 0:
												raise ValueError('multiplicative_persistence() does not accept negative values'		)

						lowerCAmelCase__    						=						0
						lowerCAmelCase__    						=						str(UpperCamelCase_		)

						while len(UpperCamelCase_		) != 1:
												lowerCAmelCase__    						=						[int(UpperCamelCase_		) for i in num_string]

												lowerCAmelCase__    						=						1
												for i in range(0    ,       len(UpperCamelCase_		)		):
																		total *= numbers[i]

												lowerCAmelCase__    						=						str(UpperCamelCase_		)

												steps += 1
						return steps







def 	_UpperCamelCase   (    UpperCamelCase_       :					int		)  ->   int:

						"""simple docstring"""



						if not isinstance(UpperCamelCase_    ,       UpperCamelCase_		):
												raise ValueError('additive_persistence() only accepts integral values'		)
						if num < 0:
												raise ValueError('additive_persistence() does not accept negative values'		)

						lowerCAmelCase__    						=						0
						lowerCAmelCase__    						=						str(UpperCamelCase_		)

						while len(UpperCamelCase_		) != 1:
												lowerCAmelCase__    						=						[int(UpperCamelCase_		) for i in num_string]

												lowerCAmelCase__    						=						0
												for i in range(0    ,       len(UpperCamelCase_		)		):
																		total += numbers[i]

												lowerCAmelCase__    						=						str(UpperCamelCase_		)

												steps += 1
						return steps


if __name__ == "__main__":
					import doctest

					doctest.testmod()





