




"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbart import MBartTokenizer
else:
					lowerCAmelCase							:    Tuple  							=							None


lowerCAmelCase							:    int  							=							logging.get_logger(__name__)


lowerCAmelCase							:    List[str]  							=							{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:    str  							=							{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase							:    List[str]  							=							{
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
lowerCAmelCase							:    List[str]  							=							["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''
				__UpperCamelCase    						=		VOCAB_FILES_NAMES
				__UpperCamelCase    						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase    						=		PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase    						=		["input_ids", "attention_mask"]
				__UpperCamelCase    						=		MBartTokenizer

				__UpperCamelCase    						=		[]
				__UpperCamelCase    						=		[]





				def __init__( self	,				_a=None	,				_a=None	,				_a="<s>"	,				_a="</s>"	,				_a="</s>"	,				_a="<s>"	,				_a="<unk>"	,				_a="<pad>"	,				_a="<mask>"	,				_a=None	,				_a=None	,				_a=None	,				**_a	,				):





							"""simple docstring"""
							# Mask token behave like a normal word, i.e. include the space before it
							lowerCamelCase       =      AddedToken(_a	,				lstrip=_a	,				rstrip=_a				) if isinstance(_a	,				_a				) else mask_token

							super().__init__(
							    vocab_file=_a	,				tokenizer_file=_a	,				bos_token=_a	,				eos_token=_a	,				sep_token=_a	,				cls_token=_a	,				unk_token=_a	,				pad_token=_a	,				mask_token=_a	,				src_lang=_a	,				tgt_lang=_a	,				additional_special_tokens=_a	,				**_a	,				)

							lowerCamelCase       =      vocab_file
							lowerCamelCase       =      False if not self.vocab_file else True

							lowerCamelCase       =      FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
										# Only add those special tokens if they are not already there.
										_additional_special_tokens.extend(
										    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

							self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}				)
							lowerCamelCase       =      {
							    lang_code: self.convert_tokens_to_ids(_a				) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							lowerCamelCase       =      src_lang if src_lang is not None else """en_XX"""
							lowerCamelCase       =      self.convert_tokens_to_ids(self._src_lang				)
							lowerCamelCase       =      tgt_lang
							self.set_src_lang_special_tokens(self._src_lang				)





				@property
				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return self._src_lang





				@src_lang.setter
				def      _lowerCAmelCase       ( self	,				_a				):





							"""simple docstring"""
							lowerCamelCase       =      new_src_lang
							self.set_src_lang_special_tokens(self._src_lang				)





				def      _lowerCAmelCase       ( self	,				_a	,				_a = None				):





							"""simple docstring"""
							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





				def      _lowerCAmelCase       ( self	,				_a	,				_a = None				):





							"""simple docstring"""
							lowerCamelCase       =      [self.sep_token_id]
							lowerCamelCase       =      [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]





				def      _lowerCAmelCase       ( self	,				_a	,				_a	,				_a	,				_a	,				**_a				):





							"""simple docstring"""
							if src_lang is None or tgt_lang is None:
										raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""				)
							lowerCamelCase       =      src_lang
							lowerCamelCase       =      self(_a	,				add_special_tokens=_a	,				return_tensors=_a	,				**_a				)
							lowerCamelCase       =      self.convert_tokens_to_ids(_a				)
							lowerCamelCase       =      tgt_lang_id
							return inputs





				def      _lowerCAmelCase       ( self	,				_a	,				_a = "en_XX"	,				_a = None	,				_a = "ro_RO"	,				**_a	,				):





							"""simple docstring"""
							lowerCamelCase       =      src_lang
							lowerCamelCase       =      tgt_lang
							return super().prepare_seqaseq_batch(_a	,				_a	,				**_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return self.set_src_lang_special_tokens(self.src_lang				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return self.set_tgt_lang_special_tokens(self.tgt_lang				)





				def      _lowerCAmelCase       ( self	,				_a				):





							"""simple docstring"""
							lowerCamelCase       =      self.convert_tokens_to_ids(_a				)
							lowerCamelCase       =      []
							lowerCamelCase       =      [self.eos_token_id, self.cur_lang_code]

							lowerCamelCase       =      self.convert_ids_to_tokens(self.prefix_tokens				)
							lowerCamelCase       =      self.convert_ids_to_tokens(self.suffix_tokens				)

							lowerCamelCase       =      processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str	,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str	,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,				self.prefix_tokens + self.suffix_tokens				)				)	,				)





				def      _lowerCAmelCase       ( self	,				_a				):





							"""simple docstring"""
							lowerCamelCase       =      self.convert_tokens_to_ids(_a				)
							lowerCamelCase       =      []
							lowerCamelCase       =      [self.eos_token_id, self.cur_lang_code]

							lowerCamelCase       =      self.convert_ids_to_tokens(self.prefix_tokens				)
							lowerCamelCase       =      self.convert_ids_to_tokens(self.suffix_tokens				)

							lowerCamelCase       =      processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str	,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str	,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,				self.prefix_tokens + self.suffix_tokens				)				)	,				)





				def      _lowerCAmelCase       ( self	,				_a	,				_a = None				):





							"""simple docstring"""
							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
										    """tokenizer."""				)

							if not os.path.isdir(_a				):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory.'				)
										return
							lowerCamelCase       =      os.path.join(
							    _a	,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

							if os.path.abspath(self.vocab_file				) != os.path.abspath(_a				):
										copyfile(self.vocab_file	,				_a				)

							return (out_vocab_file,)









"""simple docstring"""




from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


lowerCAmelCase							:    List[Any]  							=							logging.get_logger(__name__)


lowerCAmelCase							:    int  							=							TypeVar("""DatasetType""", Dataset, IterableDataset)



def 					a__				(       snake_case__ ,				snake_case__ = None ,				snake_case__ = None ,				snake_case__ = None ,				snake_case__ = None ,				snake_case__ = "first_exhausted" ,				)		->      DatasetType:
			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError("""Unable to interleave an empty list of datasets."""       )
			for i, dataset in enumerate(snake_case__       ):
						if not isinstance(snake_case__ ,				(Dataset, IterableDataset)       ):
									if isinstance(snake_case__ ,				(DatasetDict, IterableDatasetDict)       ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    """is an empty dataset dictionary."""       )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(snake_case__       )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(snake_case__       )       )}\']'       )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(snake_case__       ).__name__}.'       )
						if i == 0:
									lowerCamelCase   ,     lowerCamelCase       =      (
									    (Dataset, IterableDataset) if isinstance(snake_case__ ,				snake_case__       ) else (IterableDataset, Dataset)
									)
						elif not isinstance(snake_case__ ,				snake_case__       ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'       )
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(F'{stopping_strategy} is not supported. Please enter a valid stopping_strategy.'       )
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    snake_case__ ,				snake_case__ ,				snake_case__ ,				info=snake_case__ ,				split=snake_case__ ,				stopping_strategy=snake_case__       )
			else:
						return _interleave_iterable_datasets(
						    snake_case__ ,				snake_case__ ,				snake_case__ ,				info=snake_case__ ,				split=snake_case__ ,				stopping_strategy=snake_case__       )



def 					a__				(       snake_case__ ,				snake_case__ = None ,				snake_case__ = None ,				snake_case__ = 0 ,				)		->      DatasetType:

			if not dsets:
						raise ValueError("""Unable to concatenate an empty list of datasets."""       )
			for i, dataset in enumerate(snake_case__       ):
						if not isinstance(snake_case__ ,				(Dataset, IterableDataset)       ):
									if isinstance(snake_case__ ,				(DatasetDict, IterableDatasetDict)       ):
												if not dataset:
															raise ValueError(
															    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} '
															    """is an empty dataset dictionary."""       )
												raise ValueError(
												    F'Dataset at position {i} has at least one split: {list(snake_case__       )}\n'
												    F'Please pick one to interleave with the other datasets, for example: dataset[\'{next(iter(snake_case__       )       )}\']'       )
									raise ValueError(
									    F'Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(snake_case__       ).__name__}.'       )
						if i == 0:
									lowerCamelCase   ,     lowerCamelCase       =      (
									    (Dataset, IterableDataset) if isinstance(snake_case__ ,				snake_case__       ) else (IterableDataset, Dataset)
									)
						elif not isinstance(snake_case__ ,				snake_case__       ):
									raise ValueError(
									    F'Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects.'       )
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(snake_case__ ,				info=snake_case__ ,				split=snake_case__ ,				axis=snake_case__       )
			else:
						return _concatenate_iterable_datasets(snake_case__ ,				info=snake_case__ ,				split=snake_case__ ,				axis=snake_case__       )



