


from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
   from fairscale.optim import OSS


UpperCAmelCase    =					logging.get_logger(__name__)

UpperCAmelCase    =					{
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}







class lowerCAmelCase_		(				lowerCamelCase__ ):

 '''simple docstring'''

 def __init__(			self	,   _UpperCAmelCase=None	,   _UpperCAmelCase=None	,   *_UpperCAmelCase	,   **_UpperCAmelCase      ):
       super().__init__(*_UpperCAmelCase	,   **_UpperCAmelCase      )

       if config is None:
             assert isinstance(self.model	,   _UpperCAmelCase      ), (
                 "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
                 F''' {self.model.__class__}'''
             )
             snake_case_ =   self.model.config
       else:
             snake_case_ =   config

       snake_case_ =   data_args
       snake_case_ =   self.config.tgt_vocab_size if isinstance(self.config	,   _UpperCAmelCase      ) else self.config.vocab_size

       if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
             assert self.config.pad_token_id is not None, (
                 "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
                 " calculation or doing label smoothing."
             )

       if self.config.pad_token_id is None and self.config.eos_token_id is not None:
             logger.warning(
                 F'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
                 ''' padding..'''      )

       if self.args.label_smoothing == 0:
             snake_case_ =   torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id      )
       else:
             # dynamically import label_smoothed_nll_loss
             from utils import label_smoothed_nll_loss

             snake_case_ =   label_smoothed_nll_loss



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       if self.optimizer is None:
             snake_case_ =   ['''bias''', '''LayerNorm.weight''']
             snake_case_ =   [
                 {
                     '''params''': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay      )],
                     '''weight_decay''': self.args.weight_decay,
                 },
                 {
                     '''params''': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay      )],
                     '''weight_decay''': 0.0,
                 },
             ]
             snake_case_ =   Adafactor if self.args.adafactor else AdamW
             if self.args.adafactor:
                   snake_case_ =   Adafactor
                   snake_case_ =   {'''scale_parameter''': False, '''relative_step''': False}
             else:
                   snake_case_ =   AdamW
                   snake_case_ =   {
                       '''betas''': (self.args.adam_betaa, self.args.adam_betaa),
                       '''eps''': self.args.adam_epsilon,
                   }
             snake_case_ =   self.args.learning_rate
             if self.sharded_ddp:
                   snake_case_ =   OSS(
                       params=_UpperCAmelCase	,   optim=_UpperCAmelCase	,   **_UpperCAmelCase	,   )
             else:
                   snake_case_ =   optimizer_cls(_UpperCAmelCase	,   **_UpperCAmelCase      )

       if self.lr_scheduler is None:
             snake_case_ =   self._get_lr_scheduler(_UpperCAmelCase      )
       else:  # ignoring --lr_scheduler
             logger.warning('''scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'''      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       snake_case_ =   arg_to_scheduler[self.args.lr_scheduler]
       if self.args.lr_scheduler == "constant":
             snake_case_ =   schedule_func(self.optimizer      )
       elif self.args.lr_scheduler == "constant_w_warmup":
             snake_case_ =   schedule_func(self.optimizer	,   num_warmup_steps=self.args.warmup_steps      )
       else:
             snake_case_ =   schedule_func(
                 self.optimizer	,   num_warmup_steps=self.args.warmup_steps	,   num_training_steps=_UpperCAmelCase      )
       return scheduler



 def 		UpperCamelCase__  (			self      ):
       if isinstance(self.train_dataset	,   torch.utils.data.IterableDataset      ):
             return None
       elif is_torch_tpu_available():
             return get_tpu_sampler(self.train_dataset      )
       else:
             if self.args.sortish_sampler:
                   self.train_dataset.make_sortish_sampler(
                       self.args.per_device_train_batch_size	,   distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)	,   )

             return (
                 RandomSampler(self.train_dataset      )
                 if self.args.local_rank == -1
                 else DistributedSampler(self.train_dataset      )
             )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       if self.args.label_smoothing == 0:
             if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
                   # force training to ignore pad token
                   snake_case_ =   model(**_UpperCAmelCase	,   use_cache=_UpperCAmelCase      )[0]
                   snake_case_ =   self.loss_fn(logits.view(-1	,   logits.shape[-1]      )	,   labels.view(-1      )      )
             else:
                   # compute usual loss via models
                   snake_case_ ,					snake_case_ =   model(**_UpperCAmelCase	,   labels=_UpperCAmelCase	,   use_cache=_UpperCAmelCase      )[:2]
       else:
             # compute label smoothed loss
             snake_case_ =   model(**_UpperCAmelCase	,   use_cache=_UpperCAmelCase      )[0]
             snake_case_ =   torch.nn.functional.log_softmax(_UpperCAmelCase	,   dim=-1      )
             snake_case_ ,					snake_case_ =   self.loss_fn(_UpperCAmelCase	,   _UpperCAmelCase	,   self.args.label_smoothing	,   ignore_index=self.config.pad_token_id      )
       return loss, logits



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       snake_case_ =   inputs.pop('''labels'''      )
       snake_case_ ,					snake_case_ =   self._compute_loss(_UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase      )
       return loss



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase = None	,   ):
       snake_case_ =   self._prepare_inputs(_UpperCAmelCase      )

       snake_case_ =   {
           '''max_length''': self.data_args.val_max_target_length
           if self.data_args is not None
           else self.config.max_length,
           '''num_beams''': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
       }

       if self.args.predict_with_generate and not self.args.prediction_loss_only:
             snake_case_ =   self.model.generate(
                 inputs['''input_ids''']	,   attention_mask=inputs['''attention_mask''']	,   **_UpperCAmelCase	,   )
             # in case the batch is shorter than max length, the output should be padded
             if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
                   snake_case_ =   self._pad_tensors_to_max_len(_UpperCAmelCase	,   gen_kwargs['''max_length''']      )

       snake_case_ =   inputs.pop('''labels'''      )
       with torch.no_grad():
             # compute loss on predict data
             snake_case_ ,					snake_case_ =   self._compute_loss(_UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase      )

       snake_case_ =   loss.mean().detach()
       if self.args.prediction_loss_only:
             return (loss, None, None)

       snake_case_ =   generated_tokens if self.args.predict_with_generate else logits

       if labels.shape[-1] < gen_kwargs["max_length"]:
             snake_case_ =   self._pad_tensors_to_max_len(_UpperCAmelCase	,   gen_kwargs['''max_length''']      )

       return (loss, logits, labels)





 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       # If PAD token is not defined at least EOS token has to be defined
       snake_case_ =   self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

       if pad_token_id is None:
             raise ValueError(
                 '''Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'''
                 F''' padded to `max_length`={max_length}'''      )

       snake_case_ =   pad_token_id * torch.ones(
           (tensor.shape[0], max_length)	,   dtype=tensor.dtype	,   device=tensor.device      )
       snake_case_ =   tensor
       return padded_tensor



# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from packaging import version

from .. import __version__
from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
from .doc import (
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    copy_func,
    replace_return_docstrings,
)
from .generic import (
    ContextManagers,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    TensorType,
    add_model_info_to_auto_map,
    cached_property,
    can_return_loss,
    expand_dims,
    find_labels,
    flatten_dict,
    infer_framework,
    is_jax_tensor,
    is_numpy_array,
    is_tensor,
    is_tf_symbolic_tensor,
    is_tf_tensor,
    is_torch_device,
    is_torch_dtype,
    is_torch_tensor,
    reshape,
    squeeze,
    strtobool,
    tensor_size,
    to_numpy,
    to_py_obj,
    transpose,
    working_or_temp_dir,
)
from .hub import (
    CLOUDFRONT_DISTRIB_PREFIX,
    DISABLE_TELEMETRY,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    EntryNotFoundError,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    cached_file,
    default_cache_path,
    define_sagemaker_information,
    download_url,
    extract_commit_hash,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    has_file,
    http_user_agent,
    is_offline_mode,
    is_remote_url,
    move_cache,
    send_example_telemetry,
    try_to_load_from_cache,
)
from .import_utils import (
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    TORCH_FX_REQUIRED_VERSION,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    _LazyModule,
    ccl_version,
    direct_transformers_import,
    get_torch_version,
    is_accelerate_available,
    is_apex_available,
    is_bitsandbytes_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_cython_available,
    is_datasets_available,
    is_decord_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_jieba_available,
    is_jumanpp_available,
    is_kenlm_available,
    is_keras_nlp_available,
    is_librosa_available,
    is_natten_available,
    is_ninja_available,
    is_onnx_available,
    is_openai_available,
    is_optimum_available,
    is_pandas_available,
    is_peft_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytest_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sacremoses_available,
    is_safetensors_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_sudachi_available,
    is_tensorflow_probability_available,
    is_tensorflow_text_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_bfaa_cpu_available,
    is_torch_bfaa_gpu_available,
    is_torch_compile_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_neuroncore_available,
    is_torch_tensorrt_fx_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_torchdistx_available,
    is_torchdynamo_available,
    is_torchvision_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    requires_backends,
    torch_only_method,
)


UpperCAmelCase    =					"""pytorch_model.bin"""
UpperCAmelCase    =					"""pytorch_model.bin.index.json"""
UpperCAmelCase    =					"""adapter_config.json"""
UpperCAmelCase    =					"""adapter_model.bin"""
UpperCAmelCase    =					"""adapter_model.safetensors"""
UpperCAmelCase    =					"""tf_model.h5"""
UpperCAmelCase    =					"""tf_model.h5.index.json"""
UpperCAmelCase    =					"""model.ckpt"""
UpperCAmelCase    =					"""flax_model.msgpack"""
UpperCAmelCase    =					"""flax_model.msgpack.index.json"""
UpperCAmelCase    =					"""model.safetensors"""
UpperCAmelCase    =					"""model.safetensors.index.json"""
UpperCAmelCase    =					"""config.json"""
UpperCAmelCase    =					"""preprocessor_config.json"""
UpperCAmelCase    =					FEATURE_EXTRACTOR_NAME
UpperCAmelCase    =					"""generation_config.json"""
UpperCAmelCase    =					"""modelcard.json"""

UpperCAmelCase    =					"""▁"""
UpperCAmelCase    =					SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility

UpperCAmelCase    =					[
    [[0, 1, 0, 1], [1, 0, 0, 1]]
] * 2  # Needs to have 0s and 1s only since XLM uses it for langs too.
UpperCAmelCase    =					[[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]
UpperCAmelCase    =					[[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]]






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      )->						List[str]:




      """simple docstring"""

      if version.parse(SCREAMING_SNAKE_CASE      ) < version.parse(SCREAMING_SNAKE_CASE      ):
            if "dev" in min_version:
                  snake_case_ =   (
                      '''This example requires a source install from HuggingFace Transformers (see '''
                      '''`https://huggingface.co/docs/transformers/installation#install-from-source`),'''
                  )
            else:
                  snake_case_ =   f'''This example requires a minimum version of {min_version},'''
            error_message += f''' but the version found is {__version__}.\n'''
            raise ImportError(
                error_message
                + '''Check out https://github.com/huggingface/transformers/tree/main/examples#important-note for the examples corresponding to other '''
                '''versions of HuggingFace Transformers.'''      )