
"""simple docstring"""






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbart import MBartTokenizer
else:
					UpperCAmelCase        =				None


UpperCAmelCase        =				logging.get_logger(__name__)


UpperCAmelCase        =				{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase        =				{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase        =				{
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
UpperCAmelCase        =				["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]



class 			lowercase					( lowercase__		):
			lowercase        =     VOCAB_FILES_NAMES
			lowercase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase        =     PRETRAINED_VOCAB_FILES_MAP
			lowercase        =     ['''input_ids''', '''attention_mask''']
			lowercase        =     MBartTokenizer

			lowercase        =     []
			lowercase        =     []






			def __init__(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	int=None      ,SCREAMING_SNAKE_CASE_  :	Optional[int]=None      ,SCREAMING_SNAKE_CASE_  :	List[str]="<s>"      ,SCREAMING_SNAKE_CASE_  :	str="</s>"      ,SCREAMING_SNAKE_CASE_  :	Tuple="</s>"      ,SCREAMING_SNAKE_CASE_  :	List[str]="<s>"      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]="<unk>"      ,SCREAMING_SNAKE_CASE_  :	Any="<pad>"      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]="<mask>"      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]=None      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]=None      ,SCREAMING_SNAKE_CASE_  :	Dict=None      ,**SCREAMING_SNAKE_CASE_  :	List[Any]      ,)       ->	List[Any]:





				"""simple docstring"""


				lowerCAmelCase												=  AddedToken(SCREAMING_SNAKE_CASE_      ,lstrip=SCREAMING_SNAKE_CASE_      ,rstrip=SCREAMING_SNAKE_CASE_			) if isinstance(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			) else mask_token

				super().__init__(
				    vocab_file=SCREAMING_SNAKE_CASE_      ,tokenizer_file=SCREAMING_SNAKE_CASE_      ,bos_token=SCREAMING_SNAKE_CASE_      ,eos_token=SCREAMING_SNAKE_CASE_      ,sep_token=SCREAMING_SNAKE_CASE_      ,cls_token=SCREAMING_SNAKE_CASE_      ,unk_token=SCREAMING_SNAKE_CASE_      ,pad_token=SCREAMING_SNAKE_CASE_      ,mask_token=SCREAMING_SNAKE_CASE_      ,src_lang=SCREAMING_SNAKE_CASE_      ,tgt_lang=SCREAMING_SNAKE_CASE_      ,additional_special_tokens=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      ,)

				lowerCAmelCase												=  vocab_file
				lowerCAmelCase												=  False if not self.vocab_file else True

				lowerCAmelCase												=  FAIRSEQ_LANGUAGE_CODES.copy()

				if additional_special_tokens is not None:
					# Only add those special tokens if they are not already there.
					_additional_special_tokens.extend(
					    [t for t in additional_special_tokens if t not in _additional_special_tokens]			)

				self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}			)
				lowerCAmelCase												=  {
				    lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				lowerCAmelCase												=  src_lang if src_lang is not None else '''en_XX'''
				lowerCAmelCase												=  self.convert_tokens_to_ids(self._src_lang			)
				lowerCAmelCase												=  tgt_lang
				self.set_src_lang_special_tokens(self._src_lang			)






			@property
			def 					UpperCAmelCase							(self  :	int			)       ->	str:





				"""simple docstring"""


				return self._src_lang






			@src_lang.setter
			def 					UpperCAmelCase							(self  :	Optional[int]      ,SCREAMING_SNAKE_CASE_  :	str			)       ->	None:





				"""simple docstring"""


				lowerCAmelCase												=  new_src_lang
				self.set_src_lang_special_tokens(self._src_lang			)






			def 					UpperCAmelCase							(self  :	List[str]      ,SCREAMING_SNAKE_CASE_  :	List[int]      ,SCREAMING_SNAKE_CASE_  :	Optional[List[int]] = None			)       ->	List[int]:





				"""simple docstring"""


				if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






			def 					UpperCAmelCase							(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	List[int]      ,SCREAMING_SNAKE_CASE_  :	Optional[List[int]] = None			)       ->	List[int]:





				"""simple docstring"""


				lowerCAmelCase												=  [self.sep_token_id]
				lowerCAmelCase												=  [self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep			) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






			def 					UpperCAmelCase							(self  :	Optional[int]      ,SCREAMING_SNAKE_CASE_  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	str      ,SCREAMING_SNAKE_CASE_  :	Optional[str]      ,SCREAMING_SNAKE_CASE_  :	Optional[str]      ,**SCREAMING_SNAKE_CASE_  :	Any			)       ->	Tuple:





				"""simple docstring"""


				if src_lang is None or tgt_lang is None:
					raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''			)
				lowerCAmelCase												=  src_lang
				lowerCAmelCase												=  self(SCREAMING_SNAKE_CASE_      ,add_special_tokens=SCREAMING_SNAKE_CASE_      ,return_tensors=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  tgt_lang_id
				return inputs






			def 					UpperCAmelCase							(self  :	str      ,SCREAMING_SNAKE_CASE_  :	List[str]      ,SCREAMING_SNAKE_CASE_  :	str = "en_XX"      ,SCREAMING_SNAKE_CASE_  :	Optional[List[str]] = None      ,SCREAMING_SNAKE_CASE_  :	str = "ro_RO"      ,**SCREAMING_SNAKE_CASE_  :	Dict      ,)       ->	BatchEncoding:





				"""simple docstring"""


				lowerCAmelCase												=  src_lang
				lowerCAmelCase												=  tgt_lang
				return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	str			)       ->	Optional[Any]:





				"""simple docstring"""


				return self.set_src_lang_special_tokens(self.src_lang			)






			def 					UpperCAmelCase							(self  :	Dict			)       ->	int:





				"""simple docstring"""


				return self.set_tgt_lang_special_tokens(self.tgt_lang			)






			def 					UpperCAmelCase							(self  :	Dict      ,SCREAMING_SNAKE_CASE_  :	List[Any]			)       ->	None:





				"""simple docstring"""


				lowerCAmelCase												=  self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  []
				lowerCAmelCase												=  [self.eos_token_id, self.cur_lang_code]

				lowerCAmelCase												=  self.convert_ids_to_tokens(self.prefix_tokens			)
				lowerCAmelCase												=  self.convert_ids_to_tokens(self.suffix_tokens			)

				lowerCAmelCase												=  processors.TemplateProcessing(
				    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str      ,pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str      ,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,self.prefix_tokens + self.suffix_tokens			)			)      ,)






			def 					UpperCAmelCase							(self  :	Any      ,SCREAMING_SNAKE_CASE_  :	str			)       ->	None:





				"""simple docstring"""


				lowerCAmelCase												=  self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  []
				lowerCAmelCase												=  [self.eos_token_id, self.cur_lang_code]

				lowerCAmelCase												=  self.convert_ids_to_tokens(self.prefix_tokens			)
				lowerCAmelCase												=  self.convert_ids_to_tokens(self.suffix_tokens			)

				lowerCAmelCase												=  processors.TemplateProcessing(
				    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str      ,pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str      ,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,self.prefix_tokens + self.suffix_tokens			)			)      ,)






			def 					UpperCAmelCase							(self  :	str      ,SCREAMING_SNAKE_CASE_  :	str      ,SCREAMING_SNAKE_CASE_  :	Optional[str] = None			)       ->	Tuple[str]:





				"""simple docstring"""


				if not self.can_save_slow_tokenizer:
					raise ValueError(
					    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
					    '''tokenizer.'''			)

				if not os.path.isdir(SCREAMING_SNAKE_CASE_			):
					logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""			)
					return
				lowerCAmelCase												=  os.path.join(
				    SCREAMING_SNAKE_CASE_      ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

				if os.path.abspath(self.vocab_file			) != os.path.abspath(SCREAMING_SNAKE_CASE_			):
					copyfile(self.vocab_file      ,SCREAMING_SNAKE_CASE_			)

				return (out_vocab_file,)




"""simple docstring"""






import timeit

import numpy as np

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features.features import _ArrayXD




def __magic_name__	(				_lowerCamelCase:		Optional[Any]			)					->						Dict:

	'''simple docstring'''




	def wrapper(*_lowerCamelCase:		Any,					**_lowerCamelCase:		Union[str, Any]			):
		lowerCAmelCase												=  timeit.default_timer()
		lowerCAmelCase												=  func(*_lowerCamelCase,					**_lowerCamelCase			)
		lowerCAmelCase												=  timeit.default_timer() - starttime
		return delta

	lowerCAmelCase												=  func.__name__

	return wrapper




def __magic_name__	(				_lowerCamelCase:		dict,					_lowerCamelCase:		List[Any]=100,					_lowerCamelCase:		int=None			)					->						List[Any]:

	'''simple docstring'''




	lowerCAmelCase												=  []
	lowerCAmelCase												=  seq_shapes or {}
	for i in range(_lowerCamelCase			):
		lowerCAmelCase												=  {}
		for col_id, (k, v) in enumerate(features.items()			):
			if isinstance(_lowerCamelCase,					_ArrayXD			):
				lowerCAmelCase												=  np.random.rand(*v.shape			).astype(v.dtype			)
			elif isinstance(_lowerCamelCase,					datasets.Value			):
				if v.dtype == "string":
					lowerCAmelCase												=  '''The small grey turtle was surprisingly fast when challenged.'''
				else:
					lowerCAmelCase												=  np.random.randint(10,					size=1			).astype(v.dtype			).item()
			elif isinstance(_lowerCamelCase,					datasets.Sequence			):
				while isinstance(_lowerCamelCase,					datasets.Sequence			):
					lowerCAmelCase												=  v.feature
				lowerCAmelCase												=  seq_shapes[k]
				lowerCAmelCase												=  np.random.rand(*_lowerCamelCase			).astype(v.dtype			)
			lowerCAmelCase												=  data

		dummy_data.append((i, example)			)

	return dummy_data




def __magic_name__	(				_lowerCamelCase:		Tuple,					_lowerCamelCase:		Tuple,					_lowerCamelCase:		Union[str, Any]=100,					_lowerCamelCase:		List[Any]=None			)					->						List[Any]:

	'''simple docstring'''




	lowerCAmelCase												=  generate_examples(_lowerCamelCase,					num_examples=_lowerCamelCase,					seq_shapes=_lowerCamelCase			)

	with ArrowWriter(features=_lowerCamelCase,					path=_lowerCamelCase			) as writer:
		for key, record in dummy_data:
			lowerCAmelCase												=  features.encode_example(_lowerCamelCase			)
			writer.write(_lowerCamelCase			)

		lowerCAmelCase							,       lowerCAmelCase												=  writer.finalize()

	if not num_final_examples == num_examples:
		raise ValueError(
		    F"""Error writing the dataset, wrote {num_final_examples} examples but should have written {num_examples}."""			)

	lowerCAmelCase												=  datasets.Dataset.from_file(filename=_lowerCamelCase,					info=datasets.DatasetInfo(features=_lowerCamelCase			)			)

	return dataset


