






from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
							from fairscale.optim import OSS


__UpperCamelCase       :		List[Any]							=    logging.get_logger(__name__)

__UpperCamelCase       :		Optional[Any]							=    {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}


class 	__SCREAMING_SNAKE_CASE(						_lowerCAmelCase						):
							def __init__(  self:   Optional[int]							,						UpperCamelCase:   str=None							,						UpperCamelCase:   Union[str, Any]=None							,						*UpperCamelCase:   List[str]							,						**UpperCamelCase:   Tuple    )		->		Any:
											super().__init__(*UpperCAmelCase_							,						**UpperCAmelCase_    )

											if config is None:
															assert isinstance(self.model							,						UpperCAmelCase_    ), (
															    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
															    F''' {self.model.__class__}'''
															)
															snake_case__						=  self.model.config
											else:
															snake_case__						=  config

											snake_case__						=  data_args
											snake_case__						=  self.config.tgt_vocab_size if isinstance(self.config							,						UpperCAmelCase_    ) else self.config.vocab_size

											if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
															assert self.config.pad_token_id is not None, (
															    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
															    " calculation or doing label smoothing."
															)

											if self.config.pad_token_id is None and self.config.eos_token_id is not None:
															logger.warning(
															    F'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
															    ' padding..'    )

											if self.args.label_smoothing == 0:
															snake_case__						=  torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id    )
											else:
															# dynamically import label_smoothed_nll_loss
															from utils import label_smoothed_nll_loss

															snake_case__						=  label_smoothed_nll_loss
							def lowerCAmelCase_			(  self:   Union[str, Any]							,						UpperCamelCase:   int    )		->		str:
											if self.optimizer is None:
															snake_case__						=  ['bias', 'LayerNorm.weight']
															snake_case__						=  [
															    {
															        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay    )],
															        'weight_decay': self.args.weight_decay,
															    },
															    {
															        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay    )],
															        'weight_decay': 0.0,
															    },
															]
															snake_case__						=  Adafactor if self.args.adafactor else AdamW
															if self.args.adafactor:
																			snake_case__						=  Adafactor
																			snake_case__						=  {'scale_parameter': False, 'relative_step': False}
															else:
																			snake_case__						=  AdamW
																			snake_case__						=  {
																			    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																			    'eps': self.args.adam_epsilon,
																			}
															snake_case__						=  self.args.learning_rate
															if self.sharded_ddp:
																			snake_case__						=  OSS(
																			    params=UpperCAmelCase_							,						optim=UpperCAmelCase_							,						**UpperCAmelCase_							,						)
															else:
																			snake_case__						=  optimizer_cls(UpperCAmelCase_							,						**UpperCAmelCase_    )

											if self.lr_scheduler is None:
															snake_case__						=  self._get_lr_scheduler(UpperCAmelCase_    )
											else:  # ignoring --lr_scheduler
															logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'    )
							def lowerCAmelCase_			(  self:   List[Any]							,						UpperCamelCase:   str    )		->		Any:
											snake_case__						=  arg_to_scheduler[self.args.lr_scheduler]
											if self.args.lr_scheduler == "constant":
															snake_case__						=  schedule_func(self.optimizer    )
											elif self.args.lr_scheduler == "constant_w_warmup":
															snake_case__						=  schedule_func(self.optimizer							,						num_warmup_steps=self.args.warmup_steps    )
											else:
															snake_case__						=  schedule_func(
															    self.optimizer							,						num_warmup_steps=self.args.warmup_steps							,						num_training_steps=UpperCAmelCase_    )
											return scheduler
							def lowerCAmelCase_			(  self:   str    )		->		Optional[torch.utils.data.Sampler]:
											if isinstance(self.train_dataset							,						torch.utils.data.IterableDataset    ):
															return None
											elif is_torch_tpu_available():
															return get_tpu_sampler(self.train_dataset    )
											else:
															if self.args.sortish_sampler:
																			self.train_dataset.make_sortish_sampler(
																			    self.args.per_device_train_batch_size							,						distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)							,						)

															return (
															    RandomSampler(self.train_dataset    )
															    if self.args.local_rank == -1
															    else DistributedSampler(self.train_dataset    )
															)
							def lowerCAmelCase_			(  self:   List[Any]							,						UpperCamelCase:   int							,						UpperCamelCase:   int							,						UpperCamelCase:   int    )		->		Optional[int]:
											if self.args.label_smoothing == 0:
															if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																			# force training to ignore pad token
																			snake_case__						=  model(**UpperCAmelCase_							,						use_cache=UpperCAmelCase_    )[0]
																			snake_case__						=  self.loss_fn(logits.view(-1							,						logits.shape[-1]    )							,						labels.view(-1    )    )
															else:
																			# compute usual loss via models
																			snake_case__				,    snake_case__						=  model(**UpperCAmelCase_							,						labels=UpperCAmelCase_							,						use_cache=UpperCAmelCase_    )[:2]
											else:
															# compute label smoothed loss
															snake_case__						=  model(**UpperCAmelCase_							,						use_cache=UpperCAmelCase_    )[0]
															snake_case__						=  torch.nn.functional.log_softmax(UpperCAmelCase_							,						dim=-1    )
															snake_case__				,    snake_case__						=  self.loss_fn(UpperCAmelCase_							,						UpperCAmelCase_							,						self.args.label_smoothing							,						ignore_index=self.config.pad_token_id    )
											return loss, logits
							def lowerCAmelCase_			(  self:   List[str]							,						UpperCamelCase:   Dict							,						UpperCamelCase:   List[Any]    )		->		Union[str, Any]:
											snake_case__						=  inputs.pop('labels'    )
											snake_case__				,    snake_case__						=  self._compute_loss(UpperCAmelCase_							,						UpperCAmelCase_							,						UpperCAmelCase_    )
											return loss
							def lowerCAmelCase_			(  self:   List[str]							,						UpperCamelCase:   nn.Module							,						UpperCamelCase:   Dict[str, Union[torch.Tensor, Any]]							,						UpperCamelCase:   bool							,						UpperCamelCase:   Optional[List[str]] = None							,						)		->		Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
											snake_case__						=  self._prepare_inputs(UpperCAmelCase_    )

											snake_case__						=  {
											    'max_length': self.data_args.val_max_target_length
											    if self.data_args is not None
											    else self.config.max_length,
											    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
											}

											if self.args.predict_with_generate and not self.args.prediction_loss_only:
															snake_case__						=  self.model.generate(
															    inputs['input_ids']							,						attention_mask=inputs['attention_mask']							,						**UpperCAmelCase_							,						)
															# in case the batch is shorter than max length, the output should be padded
															if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																			snake_case__						=  self._pad_tensors_to_max_len(UpperCAmelCase_							,						gen_kwargs['max_length']    )

											snake_case__						=  inputs.pop('labels'    )
											with torch.no_grad():
															# compute loss on predict data
															snake_case__				,    snake_case__						=  self._compute_loss(UpperCAmelCase_							,						UpperCAmelCase_							,						UpperCAmelCase_    )

											snake_case__						=  loss.mean().detach()
											if self.args.prediction_loss_only:
															return (loss, None, None)

											snake_case__						=  generated_tokens if self.args.predict_with_generate else logits

											if labels.shape[-1] < gen_kwargs["max_length"]:
															snake_case__						=  self._pad_tensors_to_max_len(UpperCAmelCase_							,						gen_kwargs['max_length']    )

											return (loss, logits, labels)


							def lowerCAmelCase_			(  self:   List[str]							,						UpperCamelCase:   Union[str, Any]							,						UpperCamelCase:   Dict    )		->		int:
											snake_case__						=  self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

											if pad_token_id is None:
															raise ValueError(
															    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
															    F''' padded to `max_length`={max_length}'''    )

											snake_case__						=  pad_token_id * torch.ones(
											    (tensor.shape[0], max_length)							,						dtype=tensor.dtype							,						device=tensor.device    )
											snake_case__						=  tensor
											return padded_tensor






"""simple docstring"""


from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
      from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


SCREAMING_SNAKE_CASE			  =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE			  =  {
    """microsoft/deberta-v2-xlarge""": """https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json""",
    """microsoft/deberta-v2-xxlarge""": """https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json""",
    """microsoft/deberta-v2-xlarge-mnli""": (
        """https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json"""
    ),
    """microsoft/deberta-v2-xxlarge-mnli""": (
        """https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json"""
    ),
}



class 	__a	(   _lowerCAmelCase						):
       UpperCamelCase_		:       Any				=      '''deberta-v2'''






       def __init__(	self			:				Dict							,   UpperCAmelCase_			:				Optional[int]=128_100							,   UpperCAmelCase_			:				List[str]=1_536							,   UpperCAmelCase_			:				List[str]=24							,   UpperCAmelCase_			:				int=24							,   UpperCAmelCase_			:				int=6_144							,   UpperCAmelCase_			:				Optional[Any]="gelu"							,   UpperCAmelCase_			:				Tuple=0.1							,   UpperCAmelCase_			:				int=0.1							,   UpperCAmelCase_			:				Optional[int]=512							,   UpperCAmelCase_			:				Optional[int]=0							,   UpperCAmelCase_			:				Dict=0.02							,   UpperCAmelCase_			:				Optional[Any]=1e-7							,   UpperCAmelCase_			:				str=False							,   UpperCAmelCase_			:				Tuple=-1							,   UpperCAmelCase_			:				Optional[int]=0							,   UpperCAmelCase_			:				List[str]=True							,   UpperCAmelCase_			:				Any=None							,   UpperCAmelCase_			:				Optional[Any]=0							,   UpperCAmelCase_			:				Optional[Any]="gelu"							,   **UpperCAmelCase_			:				Optional[int]							,   )->	int:







              """simple docstring"""
              super().__init__(**UpperCAmelCase_      )

              UpperCamelCase       			=		hidden_size
              UpperCamelCase       			=		num_hidden_layers
              UpperCamelCase       			=		num_attention_heads
              UpperCamelCase       			=		intermediate_size
              UpperCamelCase       			=		hidden_act
              UpperCamelCase       			=		hidden_dropout_prob
              UpperCamelCase       			=		attention_probs_dropout_prob
              UpperCamelCase       			=		max_position_embeddings
              UpperCamelCase       			=		type_vocab_size
              UpperCamelCase       			=		initializer_range
              UpperCamelCase       			=		relative_attention
              UpperCamelCase       			=		max_relative_positions
              UpperCamelCase       			=		pad_token_id
              UpperCamelCase       			=		position_biased_input

              # Backwards compatibility
              if type(UpperCAmelCase_      ) == str:
                     UpperCamelCase       			=		[x.strip() for x in pos_att_type.lower().split("|"      )]

              UpperCamelCase       			=		pos_att_type
              UpperCamelCase       			=		vocab_size
              UpperCamelCase       			=		layer_norm_eps

              UpperCamelCase       			=		kwargs.get("pooler_hidden_size"							,   UpperCAmelCase_      )
              UpperCamelCase       			=		pooler_dropout
              UpperCamelCase       			=		pooler_hidden_act





class 	__a	(   _lowerCAmelCase						):






       @property
       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Mapping[str, Mapping[int, str]]:







              """simple docstring"""
              if self.task == "multiple-choice":
                     UpperCamelCase       			=		{0: "batch", 1: "choice", 2: "sequence"}
              else:
                     UpperCamelCase       			=		{0: "batch", 1: "sequence"}
              if self._config.type_vocab_size > 0:
                     return OrderedDict(
                         [("input_ids", dynamic_axis), ("attention_mask", dynamic_axis), ("token_type_ids", dynamic_axis)]      )
              else:
                     return OrderedDict([("input_ids", dynamic_axis), ("attention_mask", dynamic_axis)]      )






       @property
       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]      )->	int:







              """simple docstring"""
              return 12






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]							,   UpperCAmelCase_			:				Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]							,   UpperCAmelCase_			:				int = -1							,   UpperCAmelCase_			:				int = -1							,   UpperCAmelCase_			:				int = -1							,   UpperCAmelCase_			:				bool = False							,   UpperCAmelCase_			:				Optional["TensorType"] = None							,   UpperCAmelCase_			:				int = 3							,   UpperCAmelCase_			:				int = 40							,   UpperCAmelCase_			:				int = 40							,   UpperCAmelCase_			:				"PreTrainedTokenizerBase" = None							,   )->	Mapping[str, Any]:







              """simple docstring"""
              UpperCamelCase       			=		super().generate_dummy_inputs(preprocessor=UpperCAmelCase_							,   framework=UpperCAmelCase_      )
              if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
                     del dummy_inputs["token_type_ids"]
              return dummy_inputs





