



from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
				from fairscale.optim import OSS


__snake_case     :Any			       =logging.get_logger(__name__)

__snake_case     :str			       ={
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    'constant': get_constant_schedule,
    'constant_w_warmup': get_constant_schedule_with_warmup,
}



class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  Any=None						, __UpperCamelCase      :  List[Any]=None						, *__UpperCamelCase      :  Union[str, Any]						, **__UpperCamelCase      :  Optional[Any]       )		->     Tuple:
											super().__init__(*__UpperCamelCase						, **__UpperCamelCase       )

											if config is None:
																		assert isinstance(self.model						, __UpperCamelCase       ), (
																		    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
																		    f''' {self.model.__class__}'''
																		)
																		A      						=    self.model.config
											else:
																		A      						=    config

											A      						=    data_args
											A      						=    self.config.tgt_vocab_size if isinstance(self.config						, __UpperCamelCase       ) else self.config.vocab_size

											if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
																		assert self.config.pad_token_id is not None, (
																		    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
																		    " calculation or doing label smoothing."
																		)

											if self.config.pad_token_id is None and self.config.eos_token_id is not None:
																		logger.warning(
																		    f'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
																		    ' padding..'       )

											if self.args.label_smoothing == 0:
																		A      						=    torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id       )
											else:
																		# dynamically import label_smoothed_nll_loss
																		from utils import label_smoothed_nll_loss

																		A      						=    label_smoothed_nll_loss


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  int       )		->     str:
											if self.optimizer is None:
																		A      						=    ['bias', 'LayerNorm.weight']
																		A      						=    [
																		    {
																		        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay       )],
																		        'weight_decay': self.args.weight_decay,
																		    },
																		    {
																		        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay       )],
																		        'weight_decay': 0.0,
																		    },
																		]
																		A      						=    Adafactor if self.args.adafactor else AdamW
																		if self.args.adafactor:
																									A      						=    Adafactor
																									A      						=    {'scale_parameter': False, 'relative_step': False}
																		else:
																									A      						=    AdamW
																									A      						=    {
																									    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																									    'eps': self.args.adam_epsilon,
																									}
																		A      						=    self.args.learning_rate
																		if self.sharded_ddp:
																									A      						=    OSS(
																									    params=__UpperCamelCase						, optim=__UpperCamelCase						, **__UpperCamelCase						, )
																		else:
																									A      						=    optimizer_cls(__UpperCamelCase						, **__UpperCamelCase       )

											if self.lr_scheduler is None:
																		A      						=    self._get_lr_scheduler(__UpperCamelCase       )
											else:  # ignoring --lr_scheduler
																		logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  List[Any]       )		->     Union[str, Any]:
											A      						=    arg_to_scheduler[self.args.lr_scheduler]
											if self.args.lr_scheduler == "constant":
																		A      						=    schedule_func(self.optimizer       )
											elif self.args.lr_scheduler == "constant_w_warmup":
																		A      						=    schedule_func(self.optimizer						, num_warmup_steps=self.args.warmup_steps       )
											else:
																		A      						=    schedule_func(
																		    self.optimizer						, num_warmup_steps=self.args.warmup_steps						, num_training_steps=__UpperCamelCase       )
											return scheduler


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Optional[torch.utils.data.Sampler]:
											if isinstance(self.train_dataset						, torch.utils.data.IterableDataset       ):
																		return None
											elif is_torch_tpu_available():
																		return get_tpu_sampler(self.train_dataset       )
											else:
																		if self.args.sortish_sampler:
																									self.train_dataset.make_sortish_sampler(
																									    self.args.per_device_train_batch_size						, distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)						, )

																		return (
																		    RandomSampler(self.train_dataset       )
																		    if self.args.local_rank == -1
																		    else DistributedSampler(self.train_dataset       )
																		)


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Optional[int]       )		->     int:
											if self.args.label_smoothing == 0:
																		if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																									# force training to ignore pad token
																									A      						=    model(**__UpperCamelCase						, use_cache=__UpperCamelCase       )[0]
																									A      						=    self.loss_fn(logits.view(-1						, logits.shape[-1]       )						, labels.view(-1       )       )
																		else:
																									# compute usual loss via models
																									A						,   A      						=    model(**__UpperCamelCase						, labels=__UpperCamelCase						, use_cache=__UpperCamelCase       )[:2]
											else:
																		# compute label smoothed loss
																		A      						=    model(**__UpperCamelCase						, use_cache=__UpperCamelCase       )[0]
																		A      						=    torch.nn.functional.log_softmax(__UpperCamelCase						, dim=-1       )
																		A						,   A      						=    self.loss_fn(__UpperCamelCase						, __UpperCamelCase						, self.args.label_smoothing						, ignore_index=self.config.pad_token_id       )
											return loss, logits


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  List[Any]       )		->     List[Any]:
											A      						=    inputs.pop('labels'       )
											A						,   A      						=    self._compute_loss(__UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase       )
											return loss


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  nn.Module						, __UpperCamelCase      :  Dict[str, Union[torch.Tensor, Any]]						, __UpperCamelCase      :  bool						, __UpperCamelCase      :  Optional[List[str]] = None						, )		->     Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
											A      						=    self._prepare_inputs(__UpperCamelCase       )

											A      						=    {
											    'max_length': self.data_args.val_max_target_length
											    if self.data_args is not None
											    else self.config.max_length,
											    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
											}

											if self.args.predict_with_generate and not self.args.prediction_loss_only:
																		A      						=    self.model.generate(
																		    inputs['input_ids']						, attention_mask=inputs['attention_mask']						, **__UpperCamelCase						, )
																		# in case the batch is shorter than max length, the output should be padded
																		if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																									A      						=    self._pad_tensors_to_max_len(__UpperCamelCase						, gen_kwargs['max_length']       )

											A      						=    inputs.pop('labels'       )
											with torch.no_grad():
																		# compute loss on predict data
																		A						,   A      						=    self._compute_loss(__UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase       )

											A      						=    loss.mean().detach()
											if self.args.prediction_loss_only:
																		return (loss, None, None)

											A      						=    generated_tokens if self.args.predict_with_generate else logits

											if labels.shape[-1] < gen_kwargs["max_length"]:
																		A      						=    self._pad_tensors_to_max_len(__UpperCamelCase						, gen_kwargs['max_length']       )

											return (loss, logits, labels)


				def __UpperCamelCase						(							self      :  Dict						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Dict       )		->     int:
											# If PAD token is not defined at least EOS token has to be defined
											A      						=    self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

											if pad_token_id is None:
																		raise ValueError(
																		    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
																		    f''' padded to `max_length`={max_length}'''       )

											A      						=    pad_token_id * torch.ones(
											    (tensor.shape[0], max_length)						, dtype=tensor.dtype						, device=tensor.device       )
											A      						=    tensor
											return padded_tensor







import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import CONFIG_MAPPING


A__      :     Tuple       =					logging.get_logger(__name__)
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			"""upernet"""




				def __init__(	self		:       Dict,    lowerCamelCase		:       Union[str, Any]=None,    lowerCamelCase		:       str=512,    lowerCamelCase		:       Optional[Any]=0.02,    lowerCamelCase		:       Optional[Any]=[1, 2, 3, 6],    lowerCamelCase		:       Optional[int]=True,    lowerCamelCase		:       Tuple=0.4,    lowerCamelCase		:       Optional[int]=384,    lowerCamelCase		:       Optional[int]=256,    lowerCamelCase		:       Dict=1,    lowerCamelCase		:       str=False,    lowerCamelCase		:       List[str]=255,    **lowerCamelCase		:       List[Any],    ):

								'''simple docstring'''
								super().__init__(**lowerCamelCase    )

								if backbone_config is None:
												logger.info('''`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'''    )
												lowercase__  					=     CONFIG_MAPPING['''resnet'''](out_features=['''stage1''', '''stage2''', '''stage3''', '''stage4''']    )
								elif isinstance(lowerCamelCase,    lowerCamelCase    ):
												lowercase__  					=     backbone_config.get('''model_type'''    )
												lowercase__  					=     CONFIG_MAPPING[backbone_model_type]
												lowercase__  					=     config_class.from_dict(lowerCamelCase    )

								lowercase__  					=     backbone_config
								lowercase__  					=     hidden_size
								lowercase__  					=     initializer_range
								lowercase__  					=     pool_scales
								lowercase__  					=     use_auxiliary_head
								lowercase__  					=     auxiliary_loss_weight
								lowercase__  					=     auxiliary_in_channels
								lowercase__  					=     auxiliary_channels
								lowercase__  					=     auxiliary_num_convs
								lowercase__  					=     auxiliary_concat_input
								lowercase__  					=     loss_ignore_index




				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								lowercase__  					=     copy.deepcopy(self.__dict__    )
								lowercase__  					=     self.backbone_config.to_dict()
								lowercase__  					=     self.__class__.model_type
								return output




