





from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
     from fairscale.optim import OSS


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
}



class 							_UpperCamelCase					(	_SCREAMING_SNAKE_CASE     ):

       def __init__(		self: Any					,   _SCREAMING_SNAKE_CASE: Optional[int]=None					,   _SCREAMING_SNAKE_CASE: Tuple=None					,   *_SCREAMING_SNAKE_CASE: Optional[Any]					,   **_SCREAMING_SNAKE_CASE: Tuple		)       -> List[Any]:




              """simple docstring"""


              super().__init__(*A_					,   **A_		)

              if config is None:
                     assert isinstance(self.model					,   A_		), (
                         "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
                         f''' {self.model.__class__}'''
                     )
                     UpperCamelCase_          =  self.model.config
              else:
                     UpperCamelCase_          =  config

              UpperCamelCase_          =  data_args
              UpperCamelCase_          =  self.config.tgt_vocab_size if isinstance(self.config					,   A_		) else self.config.vocab_size

              if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
                     assert self.config.pad_token_id is not None, (
                         "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
                         " calculation or doing label smoothing."
                     )

              if self.config.pad_token_id is None and self.config.eos_token_id is not None:
                     logger.warning(
                         f'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
                         " padding.."		)

              if self.args.label_smoothing == 0:
                     UpperCamelCase_          =  torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id		)
              else:
                     # dynamically import label_smoothed_nll_loss
                     from utils import label_smoothed_nll_loss

                     UpperCamelCase_          =  label_smoothed_nll_loss

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: int		)       -> Union[str, Any]:




              """simple docstring"""


              if self.optimizer is None:
                     UpperCamelCase_          =  ["bias", "LayerNorm.weight"]
                     UpperCamelCase_          =  [
                         {
                             "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay		)],
                             "weight_decay": self.args.weight_decay,
                         },
                         {
                             "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay		)],
                             "weight_decay": 0.0,
                         },
                     ]
                     UpperCamelCase_          =  Adafactor if self.args.adafactor else AdamW
                     if self.args.adafactor:
                            UpperCamelCase_          =  Adafactor
                            UpperCamelCase_          =  {"scale_parameter": False, "relative_step": False}
                     else:
                            UpperCamelCase_          =  AdamW
                            UpperCamelCase_          =  {
                                "betas": (self.args.adam_betaa, self.args.adam_betaa),
                                "eps": self.args.adam_epsilon,
                            }
                     UpperCamelCase_          =  self.args.learning_rate
                     if self.sharded_ddp:
                            UpperCamelCase_          =  OSS(
                                params=A_					,   optim=A_					,   **A_					,   )
                     else:
                            UpperCamelCase_          =  optimizer_cls(A_					,   **A_		)

              if self.lr_scheduler is None:
                     UpperCamelCase_          =  self._get_lr_scheduler(A_		)
              else:  # ignoring --lr_scheduler
                     logger.warning("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."		)

       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: Union[str, Any]		)       -> Optional[Any]:




              """simple docstring"""


              UpperCamelCase_          =  arg_to_scheduler[self.args.lr_scheduler]
              if self.args.lr_scheduler == "constant":
                     UpperCamelCase_          =  schedule_func(self.optimizer		)
              elif self.args.lr_scheduler == "constant_w_warmup":
                     UpperCamelCase_          =  schedule_func(self.optimizer					,   num_warmup_steps=self.args.warmup_steps		)
              else:
                     UpperCamelCase_          =  schedule_func(
                         self.optimizer					,   num_warmup_steps=self.args.warmup_steps					,   num_training_steps=A_		)
              return scheduler

       def lowercase (		self: List[str]		)       -> Optional[torch.utils.data.Sampler]:




              """simple docstring"""


              if isinstance(self.train_dataset					,   torch.utils.data.IterableDataset		):
                     return None
              elif is_torch_tpu_available():
                     return get_tpu_sampler(self.train_dataset		)
              else:
                     if self.args.sortish_sampler:
                            self.train_dataset.make_sortish_sampler(
                                self.args.per_device_train_batch_size					,   distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)					,   )

                     return (
                         RandomSampler(self.train_dataset		)
                         if self.args.local_rank == -1
                         else DistributedSampler(self.train_dataset		)
                     )

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: str		)       -> Dict:




              """simple docstring"""


              if self.args.label_smoothing == 0:
                     if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
                            # force training to ignore pad token
                            UpperCamelCase_          =  model(**A_					,   use_cache=A_		)[0]
                            UpperCamelCase_          =  self.loss_fn(logits.view(-1					,   logits.shape[-1]		)					,   labels.view(-1		)		)
                     else:
                            # compute usual loss via models
                            UpperCamelCase_       , UpperCamelCase_          =  model(**A_					,   labels=A_					,   use_cache=A_		)[:2]
              else:
                     # compute label smoothed loss
                     UpperCamelCase_          =  model(**A_					,   use_cache=A_		)[0]
                     UpperCamelCase_          =  torch.nn.functional.log_softmax(A_					,   dim=-1		)
                     UpperCamelCase_       , UpperCamelCase_          =  self.loss_fn(A_					,   A_					,   self.args.label_smoothing					,   ignore_index=self.config.pad_token_id		)
              return loss, logits

       def lowercase (		self: Dict					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: str		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  inputs.pop("labels"		)
              UpperCamelCase_       , UpperCamelCase_          =  self._compute_loss(A_					,   A_					,   A_		)
              return loss

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: str = None					,   )       -> Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:




              """simple docstring"""


              UpperCamelCase_          =  self._prepare_inputs(A_		)

              UpperCamelCase_          =  {
                  "max_length": self.data_args.val_max_target_length
                  if self.data_args is not None
                  else self.config.max_length,
                  "num_beams": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
              }

              if self.args.predict_with_generate and not self.args.prediction_loss_only:
                     UpperCamelCase_          =  self.model.generate(
                         inputs["input_ids"]					,   attention_mask=inputs["attention_mask"]					,   **A_					,   )
                     # in case the batch is shorter than max length, the output should be padded
                     if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
                            UpperCamelCase_          =  self._pad_tensors_to_max_len(A_					,   gen_kwargs["max_length"]		)

              UpperCamelCase_          =  inputs.pop("labels"		)
              with torch.no_grad():
                     # compute loss on predict data
                     UpperCamelCase_       , UpperCamelCase_          =  self._compute_loss(A_					,   A_					,   A_		)

              UpperCamelCase_          =  loss.mean().detach()
              if self.args.prediction_loss_only:
                     return (loss, None, None)

              UpperCamelCase_          =  generated_tokens if self.args.predict_with_generate else logits

              if labels.shape[-1] < gen_kwargs["max_length"]:
                     UpperCamelCase_          =  self._pad_tensors_to_max_len(A_					,   gen_kwargs["max_length"]		)

              return (loss, logits, labels)



       def lowercase (		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Dict		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

              if pad_token_id is None:
                     raise ValueError(
                         "Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"
                         f''' padded to `max_length`={max_length}'''		)

              UpperCamelCase_          =  pad_token_id * torch.ones(
                  (tensor.shape[0], max_length)					,   dtype=tensor.dtype					,   device=tensor.device		)
              UpperCamelCase_          =  tensor
              return padded_tensor








from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/config.json'}



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					List[str]         =   '''openai-gpt'''
       _UpperCamelCase				:					List[str]         =   {
           '''max_position_embeddings''': '''n_positions''',
           '''hidden_size''': '''n_embd''',
           '''num_attention_heads''': '''n_head''',
           '''num_hidden_layers''': '''n_layer''',
       }

       def __init__(		self: int					,   _SCREAMING_SNAKE_CASE: int=40478					,   _SCREAMING_SNAKE_CASE: Optional[Any]=512					,   _SCREAMING_SNAKE_CASE: List[Any]=768					,   _SCREAMING_SNAKE_CASE: str=12					,   _SCREAMING_SNAKE_CASE: Any=12					,   _SCREAMING_SNAKE_CASE: Optional[int]="gelu"					,   _SCREAMING_SNAKE_CASE: int=0.1					,   _SCREAMING_SNAKE_CASE: Optional[int]=0.1					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=0.1					,   _SCREAMING_SNAKE_CASE: Any=1e-5					,   _SCREAMING_SNAKE_CASE: Tuple=0.02					,   _SCREAMING_SNAKE_CASE: str="cls_index"					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=None					,   _SCREAMING_SNAKE_CASE: List[Any]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=0.1					,   **_SCREAMING_SNAKE_CASE: List[Any]					,   )       -> int:




              """simple docstring"""


              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  n_positions
              UpperCamelCase_          =  n_embd
              UpperCamelCase_          =  n_layer
              UpperCamelCase_          =  n_head
              UpperCamelCase_          =  afn
              UpperCamelCase_          =  resid_pdrop
              UpperCamelCase_          =  embd_pdrop
              UpperCamelCase_          =  attn_pdrop
              UpperCamelCase_          =  layer_norm_epsilon
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  summary_type
              UpperCamelCase_          =  summary_use_proj
              UpperCamelCase_          =  summary_activation
              UpperCamelCase_          =  summary_first_dropout
              UpperCamelCase_          =  summary_proj_to_labels
              super().__init__(**_SCREAMING_SNAKE_CASE		)

