




import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						def __init__( self :     Optional[int]				,							*lowerCamelCase__ :     Optional[Any]				,							lowerCamelCase__ :     Dict=None				,							lowerCamelCase__ :     Optional[Any]=None				,							**lowerCamelCase__ :     Union[str, Any]				)    ->						Optional[Any]:
													"""simple docstring"""


													super().__init__(*lowerCamelCase__				,							**lowerCamelCase__				)
													__lowercase      =    eval_examples
													__lowercase      =    post_process_function






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     Optional[Dataset] = None				,							lowerCamelCase__ :     Optional[int]=None				,							lowerCamelCase__ :     Optional[List[str]] = None				,							lowerCamelCase__ :     str = "eval"				,							**lowerCamelCase__ :     Dict				,							)    ->						Dict[str, float]:
													"""simple docstring"""


													__lowercase      =    gen_kwargs.copy()
													__lowercase      =    (
													    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''				) is not None else self.args.generation_max_length
													)
													__lowercase      =    (
													    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''				) is not None else self.args.generation_num_beams
													)
													__lowercase      =    gen_kwargs

													__lowercase      =    self.eval_dataset if eval_dataset is None else eval_dataset
													__lowercase      =    self.get_eval_dataloader(lowerCamelCase__				)
													__lowercase      =    self.eval_examples if eval_examples is None else eval_examples

													# Temporarily disable metric computation, we will do it in the loop here.
													__lowercase      =    self.compute_metrics
													__lowercase      =    None
													__lowercase      =    time.time()
													__lowercase      =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
													try:
																				__lowercase      =    eval_loop(
																				    lowerCamelCase__				,							description='''Evaluation'''				,							prediction_loss_only=True if compute_metrics is None else None				,							ignore_keys=lowerCamelCase__				,							metric_key_prefix=lowerCamelCase__				,							)
													finally:
																				__lowercase      =    compute_metrics
													__lowercase      =    self.args.eval_batch_size * self.args.world_size
													if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																				start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
													output.metrics.update(
													    speed_metrics(
													        lowerCamelCase__				,							lowerCamelCase__				,							num_samples=output.num_samples				,							num_steps=math.ceil(output.num_samples / total_batch_size				)				,							)				)

													if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																				# Only the main node write the results by default
																				__lowercase      =    self.post_process_function(lowerCamelCase__				,							lowerCamelCase__				,							lowerCamelCase__				)
																				__lowercase      =    self.compute_metrics(lowerCamelCase__				)

																				# Prefix all keys with metric_key_prefix + '_'
																				for key in list(metrics.keys()				):
																											if not key.startswith(f'{metric_key_prefix}_'				):
																																		__lowercase      =    metrics.pop(lowerCamelCase__				)

																				metrics.update(output.metrics				)
													else:
																				__lowercase      =    output.metrics

													if self.args.should_log:
																				# Only the main node log the results by default
																				self.log(lowerCamelCase__				)

													if self.args.tpu_metrics_debug or self.args.debug:
																				# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																				xm.master_print(met.metrics_report()				)

													__lowercase      =    self.callback_handler.on_evaluate(self.args				,							self.state				,							self.control				,							lowerCamelCase__				)
													return metrics






						def     UpperCAmelCase_      ( self :     int				,							lowerCamelCase__ :     Optional[Any]				,							lowerCamelCase__ :     Tuple				,							lowerCamelCase__ :     Optional[int]=None				,							lowerCamelCase__ :     str = "test"				,							**lowerCamelCase__ :     Optional[Any]				)    ->						Optional[int]:
													"""simple docstring"""


													__lowercase      =    gen_kwargs.copy()

													__lowercase      =    self.get_test_dataloader(lowerCamelCase__				)

													# Temporarily disable metric computation, we will do it in the loop here.
													__lowercase      =    self.compute_metrics
													__lowercase      =    None
													__lowercase      =    time.time()
													__lowercase      =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
													try:
																				__lowercase      =    eval_loop(
																				    lowerCamelCase__				,							description='''Prediction'''				,							prediction_loss_only=True if compute_metrics is None else None				,							ignore_keys=lowerCamelCase__				,							metric_key_prefix=lowerCamelCase__				,							)
													finally:
																				__lowercase      =    compute_metrics

													__lowercase      =    self.args.eval_batch_size * self.args.world_size
													if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																				start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
													output.metrics.update(
													    speed_metrics(
													        lowerCamelCase__				,							lowerCamelCase__				,							num_samples=output.num_samples				,							num_steps=math.ceil(output.num_samples / total_batch_size				)				,							)				)
													if self.post_process_function is None or self.compute_metrics is None:
																				return output

													__lowercase      =    self.post_process_function(lowerCamelCase__				,							lowerCamelCase__				,							lowerCamelCase__				,							'''predict'''				)
													__lowercase      =    self.compute_metrics(lowerCamelCase__				)

													# Prefix all keys with metric_key_prefix + '_'
													for key in list(metrics.keys()				):
																				if not key.startswith(f'{metric_key_prefix}_'				):
																											__lowercase      =    metrics.pop(lowerCamelCase__				)
													metrics.update(output.metrics				)
													return PredictionOutput(predictions=predictions.predictions				,							label_ids=predictions.label_ids				,							metrics=lowerCamelCase__				)












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			UpperCAmelCase__				     = None


UpperCAmelCase__				     = logging.get_logger(__name__)


UpperCAmelCase__				     = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__				     = {
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

UpperCAmelCase__				     = {
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
UpperCAmelCase__				     = ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						UpperCamelCase_	:       int       	=		VOCAB_FILES_NAMES
						UpperCamelCase_	:       List[str]       	=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase_	:       int       	=		PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase_	:       Optional[int]       	=		['input_ids', 'attention_mask']
						UpperCamelCase_	:       Optional[int]       	=		MBartTokenizer

						UpperCamelCase_	:       List[int]       	=		[]
						UpperCamelCase_	:       List[int]       	=		[]






						def __init__( self :     Tuple				,							lowerCamelCase__ :     str=None				,							lowerCamelCase__ :     Dict=None				,							lowerCamelCase__ :     Optional[Any]="<s>"				,							lowerCamelCase__ :     int="</s>"				,							lowerCamelCase__ :     int="</s>"				,							lowerCamelCase__ :     List[Any]="<s>"				,							lowerCamelCase__ :     Union[str, Any]="<unk>"				,							lowerCamelCase__ :     Union[str, Any]="<pad>"				,							lowerCamelCase__ :     List[Any]="<mask>"				,							lowerCamelCase__ :     Optional[Any]=None				,							lowerCamelCase__ :     Any=None				,							lowerCamelCase__ :     Dict=None				,							**lowerCamelCase__ :     List[str]				,							)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    AddedToken(lowerCamelCase__				,							lstrip=lowerCamelCase__				,							rstrip=lowerCamelCase__				) if isinstance(lowerCamelCase__				,							lowerCamelCase__				) else mask_token

													super().__init__(
													    vocab_file=lowerCamelCase__				,							tokenizer_file=lowerCamelCase__				,							bos_token=lowerCamelCase__				,							eos_token=lowerCamelCase__				,							sep_token=lowerCamelCase__				,							cls_token=lowerCamelCase__				,							unk_token=lowerCamelCase__				,							pad_token=lowerCamelCase__				,							mask_token=lowerCamelCase__				,							src_lang=lowerCamelCase__				,							tgt_lang=lowerCamelCase__				,							additional_special_tokens=lowerCamelCase__				,							**lowerCamelCase__				,							)

													__lowercase      =    vocab_file
													__lowercase      =    False if not self.vocab_file else True

													__lowercase      =    FAIRSEQ_LANGUAGE_CODES.copy()

													if additional_special_tokens is not None:
																				# Only add those special tokens if they are not already there.
																				_additional_special_tokens.extend(
																				    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

													self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}				)
													__lowercase      =    {
													    lang_code: self.convert_tokens_to_ids(lowerCamelCase__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
													}

													__lowercase      =    src_lang if src_lang is not None else '''en_XX'''
													__lowercase      =    self.convert_tokens_to_ids(self._src_lang				)
													__lowercase      =    tgt_lang
													self.set_src_lang_special_tokens(self._src_lang				)






						@property
						def     UpperCAmelCase_      ( self :     Any				)    ->						str:
													"""simple docstring"""


													return self._src_lang






						@src_lang.setter
						def     UpperCAmelCase_      ( self :     Optional[Any]				,							lowerCamelCase__ :     str				)    ->						None:
													"""simple docstring"""


													__lowercase      =    new_src_lang
													self.set_src_lang_special_tokens(self._src_lang				)






						def     UpperCAmelCase_      ( self :     Any				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													if token_ids_a is None:
																				return self.prefix_tokens + token_ids_a + self.suffix_tokens
													# We don't expect to process pairs, but leave the pair logic for API consistency
													return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													__lowercase      =    [self.sep_token_id]
													__lowercase      =    [self.cls_token_id]

													if token_ids_a is None:
																				return len(cls + token_ids_a + sep				) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Optional[str]				,							lowerCamelCase__ :     Optional[str]				,							**lowerCamelCase__ :     List[str]				)    ->						Optional[int]:
													"""simple docstring"""


													if src_lang is None or tgt_lang is None:
																				raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''				)
													__lowercase      =    src_lang
													__lowercase      =    self(lowerCamelCase__				,							add_special_tokens=lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				)
													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    tgt_lang_id
													return inputs






						def     UpperCAmelCase_      ( self :     List[str]				,							lowerCamelCase__ :     List[str]				,							lowerCamelCase__ :     str = "en_XX"				,							lowerCamelCase__ :     Optional[List[str]] = None				,							lowerCamelCase__ :     str = "ro_RO"				,							**lowerCamelCase__ :     Union[str, Any]				,							)    ->						BatchEncoding:
													"""simple docstring"""


													__lowercase      =    src_lang
													__lowercase      =    tgt_lang
													return super().prepare_seqaseq_batch(lowerCamelCase__				,							lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     int				)    ->						Dict:
													"""simple docstring"""


													return self.set_src_lang_special_tokens(self.src_lang				)






						def     UpperCAmelCase_      ( self :     Union[str, Any]				)    ->						List[str]:
													"""simple docstring"""


													return self.set_tgt_lang_special_tokens(self.tgt_lang				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     Optional[Any]				)    ->						None:
													"""simple docstring"""


													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    []
													__lowercase      =    [self.eos_token_id, self.cur_lang_code]

													__lowercase      =    self.convert_ids_to_tokens(self.prefix_tokens				)
													__lowercase      =    self.convert_ids_to_tokens(self.suffix_tokens				)

													__lowercase      =    processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens				)				)				,							)






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     str				)    ->						None:
													"""simple docstring"""


													__lowercase      =    self.convert_tokens_to_ids(lowerCamelCase__				)
													__lowercase      =    []
													__lowercase      =    [self.eos_token_id, self.cur_lang_code]

													__lowercase      =    self.convert_ids_to_tokens(self.prefix_tokens				)
													__lowercase      =    self.convert_ids_to_tokens(self.suffix_tokens				)

													__lowercase      =    processors.TemplateProcessing(
													    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens				)				)				,							)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Optional[str] = None				)    ->						Tuple[str]:
													"""simple docstring"""


													if not self.can_save_slow_tokenizer:
																				raise ValueError(
																				    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																				    '''tokenizer.'''				)

													if not os.path.isdir(lowerCamelCase__				):
																				logger.error(f'Vocabulary path ({save_directory}) should be a directory.'				)
																				return
													__lowercase      =    os.path.join(
													    lowerCamelCase__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

													if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase__				):
																				copyfile(self.vocab_file				,							lowerCamelCase__				)

													return (out_vocab_file,)






