
'''simple docstring'''







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_lowercase			:  Any   							=	logging.get_logger(__name__)

_lowercase			:  List[Any]   							=	"▁"

_lowercase			:  Any   							=	{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_lowercase			:  int   							=	{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_lowercase			:  List[Any]   							=	{
    "facebook/m2m100_418M": 1_0_2_4,
}

# fmt: off
_lowercase			:  List[str]   							=	{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']

					lowerCAmelCase_					       =					[]
					lowerCAmelCase_					       =					[]






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="m2m100"    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE=8    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	{} if sp_model_kwargs is None else sp_model_kwargs

											lowercase_     :					Optional[int]													=	language_codes
											lowercase_     :					Union[str, Any]													=	FAIRSEQ_LANGUAGE_CODES[language_codes]
											lowercase_     :					Optional[Any]													=	{lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

											lowercase_     :					Dict													=	kwargs.get('''additional_special_tokens'''    ,							[]		)
											kwargs["additional_special_tokens"] += [
											    self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											    for lang_code in fairseq_language_code
											    if self.get_lang_token(__SCREAMING_SNAKE_CASE		) not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    src_lang=__SCREAMING_SNAKE_CASE    ,							tgt_lang=__SCREAMING_SNAKE_CASE    ,							bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							language_codes=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							num_madeup_words=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Tuple													=	vocab_file
											lowercase_     :					str													=	load_json(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					List[Any]													=	{v: k for k, v in self.encoder.items()}
											lowercase_     :					List[str]													=	spm_file
											lowercase_     :					int													=	load_spm(__SCREAMING_SNAKE_CASE    ,							self.sp_model_kwargs		)

											lowercase_     :					Tuple													=	len(self.encoder		)

											lowercase_     :					Tuple													=	{
											    self.get_lang_token(__SCREAMING_SNAKE_CASE		): self.encoder_size + i for i, lang_code in enumerate(__SCREAMING_SNAKE_CASE		)
											}
											lowercase_     :					Tuple													=	{lang_code: self.encoder_size + i for i, lang_code in enumerate(__SCREAMING_SNAKE_CASE		)}
											lowercase_     :					Optional[int]													=	{v: k for k, v in self.lang_token_to_id.items()}

											lowercase_     :					List[Any]													=	src_lang if src_lang is not None else '''en'''
											lowercase_     :					List[Any]													=	tgt_lang
											lowercase_     :					int													=	self.get_lang_id(self._src_lang		)
											self.set_src_lang_special_tokens(self._src_lang		)

											lowercase_     :					Union[str, Any]													=	num_madeup_words






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return len(self.encoder		) + len(self.lang_token_to_id		)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return self._src_lang






					@src_lang.setter
					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					str													=	new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.lang_token_to_id:
																	return self.lang_token_to_id[token]
											return self.encoder.get(__SCREAMING_SNAKE_CASE    ,							self.encoder[self.unk_token]		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.id_to_lang_token:
																	return self.id_to_lang_token[index]
											return self.decoder.get(__SCREAMING_SNAKE_CASE    ,							self.unk_token		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	[]
											lowercase_     :					Optional[int]													=	''''''
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		) + token
																							lowercase_     :					Optional[int]													=	[]
																	else:
																							current_sub_tokens.append(__SCREAMING_SNAKE_CASE		)
											out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		)
											return out_string.strip()






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE    ,							token_ids_a=__SCREAMING_SNAKE_CASE    ,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[Any]													=	[1] * len(self.prefix_tokens		)
											lowercase_     :					Union[str, Any]													=	[1] * len(self.suffix_tokens		)
											if token_ids_a is None:
																	return prefix_ones + ([0] * len(__SCREAMING_SNAKE_CASE		)) + suffix_ones
											return prefix_ones + ([0] * len(__SCREAMING_SNAKE_CASE		)) + ([0] * len(__SCREAMING_SNAKE_CASE		)) + suffix_ones






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	self.__dict__.copy()
											lowercase_     :					Optional[int]													=	None
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					int													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					List[str]													=	{}

											lowercase_     :					Union[str, Any]													=	load_spm(self.spm_file    ,							self.sp_model_kwargs		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	Path(__SCREAMING_SNAKE_CASE		)
											if not save_dir.is_dir():
																	raise OSError(F'''{save_directory} should be a directory'''		)
											lowercase_     :					Tuple													=	save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
											)
											lowercase_     :					Union[str, Any]													=	save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
											)

											save_json(self.encoder    ,							__SCREAMING_SNAKE_CASE		)

											if os.path.abspath(self.spm_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.spm_file		):
																	copyfile(self.spm_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.spm_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					Optional[int]													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (str(__SCREAMING_SNAKE_CASE		), str(__SCREAMING_SNAKE_CASE		))






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = "en"    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = "ro"    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					str													=	src_lang
											lowercase_     :					List[Any]													=	tgt_lang
											self.set_src_lang_special_tokens(self.src_lang		)
											return super().prepare_seqaseq_batch(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if src_lang is None or tgt_lang is None:
																	raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
											lowercase_     :					str													=	src_lang
											lowercase_     :					Tuple													=	self(__SCREAMING_SNAKE_CASE    ,							add_special_tokens=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[Any]													=	self.get_lang_id(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Any													=	tgt_lang_id
											return inputs






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.set_src_lang_special_tokens(self.src_lang		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.set_tgt_lang_special_tokens(self.tgt_lang		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Dict													=	self.lang_token_to_id[lang_token]
											lowercase_     :					List[Any]													=	[self.cur_lang_id]
											lowercase_     :					str													=	[self.eos_token_id]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Dict													=	self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					List[Any]													=	self.lang_token_to_id[lang_token]
											lowercase_     :					Tuple													=	[self.cur_lang_id]
											lowercase_     :					Optional[Any]													=	[self.eos_token_id]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.lang_code_to_token[lang]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					str													=	self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											return self.lang_token_to_id[lang_token]







def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			Dict[str, Any]							):



						"""simple docstring"""





						lowercase_     :					int													=	sentencepiece.SentencePieceProcessor(**__SCREAMING_SNAKE_CASE							)
						spm.Load(str(__SCREAMING_SNAKE_CASE							)							)
						return spm
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str							):



						"""simple docstring"""





						with open(__SCREAMING_SNAKE_CASE				,				'''r'''							) as f:
												return json.load(__SCREAMING_SNAKE_CASE							)




def  snake_case_		(    __SCREAMING_SNAKE_CASE :			int				,				__SCREAMING_SNAKE_CASE :			str							):



						"""simple docstring"""





						with open(__SCREAMING_SNAKE_CASE				,				'''w'''							) as f:
												json.dump(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				indent=2							)


'''simple docstring'''







import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


_lowercase			:  Tuple   							=	logging.getLogger(__name__)






@dataclass
@add_start_docstrings(TrainingArguments.__doc__					)
class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					field(default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use SortishSampler or not.'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `max_length` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `num_beams` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
					    }   ,						)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	super().to_dict()
											for k, v in d.items():
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					str													=	v.to_dict()
											return d



