


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			__snake_case      = None


__snake_case      = logging.get_logger(__name__)


__snake_case      = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__snake_case      = {
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

__snake_case      = {
    "facebook/mbart-large-en-ro": 1_024,
    "facebook/mbart-large-cc25": 1_024,
}

# fmt: off
__snake_case      = ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]




class  UpperCAmelCase       (							_snake_case     ):
			lowercase						       =			VOCAB_FILES_NAMES
			lowercase						       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase						       =			PRETRAINED_VOCAB_FILES_MAP
			lowercase						       =			["""input_ids""", """attention_mask"""]
			lowercase						       =			MBartTokenizer

			lowercase						       =			[]
			lowercase						       =			[]




			def __init__(    self				:		Any							,							__magic_name__				:		List[Any]=None							,							__magic_name__				:		Optional[Any]=None							,							__magic_name__				:		Dict="<s>"							,							__magic_name__				:		Union[str, Any]="</s>"							,							__magic_name__				:		int="</s>"							,							__magic_name__				:		int="<s>"							,							__magic_name__				:		Dict="<unk>"							,							__magic_name__				:		Tuple="<pad>"							,							__magic_name__				:		Union[str, Any]="<mask>"							,							__magic_name__				:		Union[str, Any]=None							,							__magic_name__				:		int=None							,							__magic_name__				:		Optional[int]=None							,							**__magic_name__				:		List[Any]							,							):







						"""simple docstring"""







						UpperCamelCase								=       AddedToken(UpperCamelCase__							,							lstrip=UpperCamelCase__							,							rstrip=UpperCamelCase__    ) if isinstance(UpperCamelCase__							,							UpperCamelCase__    ) else mask_token

						super().__init__(
						    vocab_file=UpperCamelCase__							,							tokenizer_file=UpperCamelCase__							,							bos_token=UpperCamelCase__							,							eos_token=UpperCamelCase__							,							sep_token=UpperCamelCase__							,							cls_token=UpperCamelCase__							,							unk_token=UpperCamelCase__							,							pad_token=UpperCamelCase__							,							mask_token=UpperCamelCase__							,							src_lang=UpperCamelCase__							,							tgt_lang=UpperCamelCase__							,							additional_special_tokens=UpperCamelCase__							,							**UpperCamelCase__							,							)

						UpperCamelCase								=       vocab_file
						UpperCamelCase								=       False if not self.vocab_file else True

						UpperCamelCase								=       FAIRSEQ_LANGUAGE_CODES.copy()

						if additional_special_tokens is not None:
									# Only add those special tokens if they are not already there.
									_additional_special_tokens.extend(
									    [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

						self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
						UpperCamelCase								=       {
						    lang_code: self.convert_tokens_to_ids(UpperCamelCase__    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
						}

						UpperCamelCase								=       src_lang if src_lang is not None else """en_XX"""
						UpperCamelCase								=       self.convert_tokens_to_ids(self._src_lang    )
						UpperCamelCase								=       tgt_lang
						self.set_src_lang_special_tokens(self._src_lang    )




			@property
			def 	lowerCamelCase_       (    self				:		List[str]    ):







						"""simple docstring"""







						return self._src_lang




			@src_lang.setter
			def 	lowerCamelCase_       (    self				:		Tuple							,							__magic_name__				:		Optional[Any]    ):







						"""simple docstring"""







						UpperCamelCase								=       new_src_lang
						self.set_src_lang_special_tokens(self._src_lang    )




			def 	lowerCamelCase_       (    self				:		Tuple							,							__magic_name__				:		Optional[Any]							,							__magic_name__				:		Any = None    ):







						"""simple docstring"""







						if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




			def 	lowerCamelCase_       (    self				:		Dict							,							__magic_name__				:		Any							,							__magic_name__				:		Tuple = None    ):







						"""simple docstring"""







						UpperCamelCase								=       [self.sep_token_id]
						UpperCamelCase								=       [self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep    ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




			def 	lowerCamelCase_       (    self				:		int							,							__magic_name__				:		Union[str, Any]							,							__magic_name__				:		Union[str, Any]							,							__magic_name__				:		str							,							__magic_name__				:		Tuple							,							**__magic_name__				:		str    ):







						"""simple docstring"""







						if src_lang is None or tgt_lang is None:
									raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
						UpperCamelCase								=       src_lang
						UpperCamelCase								=       self(UpperCamelCase__							,							add_special_tokens=UpperCamelCase__							,							return_tensors=UpperCamelCase__							,							**UpperCamelCase__    )
						UpperCamelCase								=       self.convert_tokens_to_ids(UpperCamelCase__    )
						UpperCamelCase								=       tgt_lang_id
						return inputs




			def 	lowerCamelCase_       (    self				:		List[Any]							,							__magic_name__				:		Any							,							__magic_name__				:		Any = "en_XX"							,							__magic_name__				:		Optional[int] = None							,							__magic_name__				:		List[str] = "ro_RO"							,							**__magic_name__				:		Optional[int]							,							):







						"""simple docstring"""







						UpperCamelCase								=       src_lang
						UpperCamelCase								=       tgt_lang
						return super().prepare_seqaseq_batch(UpperCamelCase__							,							UpperCamelCase__							,							**UpperCamelCase__    )




			def 	lowerCamelCase_       (    self				:		int    ):







						"""simple docstring"""







						return self.set_src_lang_special_tokens(self.src_lang    )




			def 	lowerCamelCase_       (    self				:		str    ):







						"""simple docstring"""







						return self.set_tgt_lang_special_tokens(self.tgt_lang    )




			def 	lowerCamelCase_       (    self				:		int							,							__magic_name__				:		Union[str, Any]    ):







						"""simple docstring"""







						UpperCamelCase								=       self.convert_tokens_to_ids(UpperCamelCase__    )
						UpperCamelCase								=       []
						UpperCamelCase								=       [self.eos_token_id, self.cur_lang_code]

						UpperCamelCase								=       self.convert_ids_to_tokens(self.prefix_tokens    )
						UpperCamelCase								=       self.convert_ids_to_tokens(self.suffix_tokens    )

						UpperCamelCase								=       processors.TemplateProcessing(
						    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,							self.prefix_tokens + self.suffix_tokens    )    )							,							)




			def 	lowerCamelCase_       (    self				:		str							,							__magic_name__				:		Optional[Any]    ):







						"""simple docstring"""







						UpperCamelCase								=       self.convert_tokens_to_ids(UpperCamelCase__    )
						UpperCamelCase								=       []
						UpperCamelCase								=       [self.eos_token_id, self.cur_lang_code]

						UpperCamelCase								=       self.convert_ids_to_tokens(self.prefix_tokens    )
						UpperCamelCase								=       self.convert_ids_to_tokens(self.suffix_tokens    )

						UpperCamelCase								=       processors.TemplateProcessing(
						    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,							self.prefix_tokens + self.suffix_tokens    )    )							,							)





			def 	lowerCamelCase_       (    self				:		Union[str, Any]							,							__magic_name__				:		Union[str, Any]							,							__magic_name__				:		Optional[Any] = None    ):







						"""simple docstring"""







						if not self.can_save_slow_tokenizer:
									raise ValueError(
									    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
									    """tokenizer."""    )

						if not os.path.isdir(UpperCamelCase__    ):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory.'    )
									return
						UpperCamelCase								=       os.path.join(
						    UpperCamelCase__							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

						if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCamelCase__    ):
									copyfile(self.vocab_file							,							UpperCamelCase__    )

						return (out_vocab_file,)







'''simple docstring'''







import pytest

import datasets


# Import fixture modules as plugins
__lowerCamelCase									=     ['''tests.fixtures.files''', '''tests.fixtures.hub''', '''tests.fixtures.fsspec''']
def    UpperCAmelCase__     (   UpperCAmelCase__,							UpperCAmelCase__					)      ->      Optional[int]:
       # Mark tests as "unit" by default if not marked as "integration" (or already marked as "unit")
       for item in items:
              if any(marker in item.keywords for marker in ["""integration""", """unit"""]					):
                     continue
              item.add_marker(pytest.mark.unit					)
def    UpperCAmelCase__     (   UpperCAmelCase__					)      ->      Tuple:
       config.addinivalue_line("""markers""",							"""torchaudio_latest: mark test to run with torchaudio>=0.12"""					)
@pytest.fixture(autouse=UpperCAmelCase__					)
def    UpperCAmelCase__     (   UpperCAmelCase__,							UpperCAmelCase__					)      ->      int:
       # test_hf_cache_home = tmp_path_factory.mktemp("cache")  # TODO: why a cache dir per test function does not work?
       A_						       =   tmp_path_factory.getbasetemp() / """cache"""
       A_						       =   test_hf_cache_home / """datasets"""
       A_						       =   test_hf_cache_home / """metrics"""
       A_						       =   test_hf_cache_home / """modules"""
       monkeypatch.setattr("""datasets.config.HF_DATASETS_CACHE""",							str(UpperCAmelCase__					)					)
       monkeypatch.setattr("""datasets.config.HF_METRICS_CACHE""",							str(UpperCAmelCase__					)					)
       monkeypatch.setattr("""datasets.config.HF_MODULES_CACHE""",							str(UpperCAmelCase__					)					)
       A_						       =   test_hf_datasets_cache / """downloads"""
       monkeypatch.setattr("""datasets.config.DOWNLOADED_DATASETS_PATH""",							str(UpperCAmelCase__					)					)
       A_						       =   test_hf_datasets_cache / """downloads""" / """extracted"""
       monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH""",							str(UpperCAmelCase__					)					)
@pytest.fixture(autouse=UpperCAmelCase__,							scope="""session"""					)
def    UpperCAmelCase__     (   )      ->      str:
       datasets.disable_progress_bar()
@pytest.fixture(autouse=UpperCAmelCase__					)
def    UpperCAmelCase__     (   UpperCAmelCase__					)      ->      Tuple:
       # don't take tests into account when counting downloads
       monkeypatch.setattr("""datasets.config.HF_UPDATE_DOWNLOAD_COUNTS""",							UpperCAmelCase__					)







@pytest.fixture
def    UpperCAmelCase__     (   UpperCAmelCase__					)      ->      Any:
       # Required to suppress RemovedIn20Warning when feature(s) are not compatible with SQLAlchemy 2.0
       # To be removed once SQLAlchemy 2.0 supported
       monkeypatch.setattr("""sqlalchemy.util.deprecations.SILENCE_UBER_WARNING""",							UpperCAmelCase__					)
