






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbart import MBartTokenizer
else:
					SCREAMING_SNAKE_CASE__   :      Optional[int]				=		None


SCREAMING_SNAKE_CASE__   :      Any				=		logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__   :      str				=		{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__   :      Tuple				=		{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

SCREAMING_SNAKE_CASE__   :      Dict				=		{
    """facebook/mbart-large-en-ro""": 10_24,
    """facebook/mbart-large-cc25""": 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE__   :      Union[str, Any]				=		["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]
class     lowerCamelCase_					(   lowerCamelCase			):
				a__       							=	VOCAB_FILES_NAMES
				a__       							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__       							=	PRETRAINED_VOCAB_FILES_MAP
				a__       							=	['''input_ids''', '''attention_mask''']
				a__       							=	MBartTokenizer

				a__       							=	[]
				a__       							=	[]







				def __init__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase="<s>"							,       __lowerCAmelCase="</s>"							,       __lowerCAmelCase="</s>"							,       __lowerCAmelCase="<s>"							,       __lowerCAmelCase="<unk>"							,       __lowerCAmelCase="<pad>"							,       __lowerCAmelCase="<mask>"							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       **__lowerCAmelCase							,       ):
								"""simple docstring"""
								# Mask token behave like a normal word, i.e. include the space before it
								__magic_name__		:Optional[int]   =   AddedToken(__lowerCAmelCase							,       lstrip=__lowerCAmelCase							,       rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase							,       __lowerCAmelCase  ) else mask_token

								super().__init__(
								    vocab_file=__lowerCAmelCase							,       tokenizer_file=__lowerCAmelCase							,       bos_token=__lowerCAmelCase							,       eos_token=__lowerCAmelCase							,       sep_token=__lowerCAmelCase							,       cls_token=__lowerCAmelCase							,       unk_token=__lowerCAmelCase							,       pad_token=__lowerCAmelCase							,       mask_token=__lowerCAmelCase							,       src_lang=__lowerCAmelCase							,       tgt_lang=__lowerCAmelCase							,       additional_special_tokens=__lowerCAmelCase							,       **__lowerCAmelCase							,       )

								__magic_name__		:List[Any]   =   vocab_file
								__magic_name__		:Any   =   False if not self.vocab_file else True

								__magic_name__		:Union[str, Any]   =   FAIRSEQ_LANGUAGE_CODES.copy()

								if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												_additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in _additional_special_tokens]  )

								self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}  )
								__magic_name__		:str   =   {
								    lang_code: self.convert_tokens_to_ids(__lowerCAmelCase  ) for lang_code in FAIRSEQ_LANGUAGE_CODES
								}

								__magic_name__		:Optional[int]   =   src_lang if src_lang is not None else '''en_XX'''
								__magic_name__		:int   =   self.convert_tokens_to_ids(self._src_lang  )
								__magic_name__		:str   =   tgt_lang
								self.set_src_lang_special_tokens(self._src_lang  )







				@property
				def       A						(    self  ):
								"""simple docstring"""
								return self._src_lang







				@src_lang.setter
				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   new_src_lang
								self.set_src_lang_special_tokens(self._src_lang  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								__magic_name__		:Any   =   [self.sep_token_id]
								__magic_name__		:List[Any]   =   [self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								if src_lang is None or tgt_lang is None:
												raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''  )
								__magic_name__		:Union[str, Any]   =   src_lang
								__magic_name__		:Union[str, Any]   =   self(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase							,       return_tensors=__lowerCAmelCase							,       **__lowerCAmelCase  )
								__magic_name__		:Any   =   self.convert_tokens_to_ids(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   tgt_lang_id
								return inputs







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = "en_XX"							,       __lowerCAmelCase = None							,       __lowerCAmelCase = "ro_RO"							,       **__lowerCAmelCase							,       ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   src_lang
								__magic_name__		:Dict   =   tgt_lang
								return super().prepare_seqaseq_batch(__lowerCAmelCase							,       __lowerCAmelCase							,       **__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								return self.set_src_lang_special_tokens(self.src_lang  )







				def       A						(    self  ):
								"""simple docstring"""
								return self.set_tgt_lang_special_tokens(self.tgt_lang  )







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:str   =   self.convert_tokens_to_ids(__lowerCAmelCase  )
								__magic_name__		:str   =   []
								__magic_name__		:str   =   [self.eos_token_id, self.cur_lang_code]

								__magic_name__		:Optional[int]   =   self.convert_ids_to_tokens(self.prefix_tokens  )
								__magic_name__		:List[Any]   =   self.convert_ids_to_tokens(self.suffix_tokens  )

								__magic_name__		:List[Any]   =   processors.TemplateProcessing(
								    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str							,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str							,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,       self.prefix_tokens + self.suffix_tokens  )  )							,       )







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   self.convert_tokens_to_ids(__lowerCAmelCase  )
								__magic_name__		:Optional[int]   =   []
								__magic_name__		:List[str]   =   [self.eos_token_id, self.cur_lang_code]

								__magic_name__		:Optional[int]   =   self.convert_ids_to_tokens(self.prefix_tokens  )
								__magic_name__		:List[Any]   =   self.convert_ids_to_tokens(self.suffix_tokens  )

								__magic_name__		:Optional[int]   =   processors.TemplateProcessing(
								    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str							,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str							,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,       self.prefix_tokens + self.suffix_tokens  )  )							,       )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
												    '''tokenizer.'''  )

								if not os.path.isdir(__lowerCAmelCase  ):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''  )
												return
								__magic_name__		:Dict   =   os.path.join(
								    __lowerCAmelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCAmelCase  ):
												copyfile(self.vocab_file							,       __lowerCAmelCase  )

								return (out_vocab_file,)














import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
					import jax.numpy as jnp

if is_tf_available():
					import tensorflow as tf

if is_torch_available():
					import torch
class     lowerCamelCase_					(   unittest.TestCase			):







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   {
								    '''task_specific_params''': {
								        '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 1_2_8, '''min_length''': 1_2, '''num_beams''': 4},
								        '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 1_4_2, '''min_length''': 5_6, '''num_beams''': 4},
								        '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 6_2, '''min_length''': 1_1, '''num_beams''': 6},
								    }
								}
								__magic_name__		:List[str]   =   {
								    '''task_specific_params.summarization.length_penalty''': 1.0,
								    '''task_specific_params.summarization.max_length''': 1_2_8,
								    '''task_specific_params.summarization.min_length''': 1_2,
								    '''task_specific_params.summarization.num_beams''': 4,
								    '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
								    '''task_specific_params.summarization_cnn.max_length''': 1_4_2,
								    '''task_specific_params.summarization_cnn.min_length''': 5_6,
								    '''task_specific_params.summarization_cnn.num_beams''': 4,
								    '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
								    '''task_specific_params.summarization_xsum.max_length''': 6_2,
								    '''task_specific_params.summarization_xsum.min_length''': 1_1,
								    '''task_specific_params.summarization_xsum.num_beams''': 6,
								}

								self.assertEqual(flatten_dict(__lowerCAmelCase  )							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       x.transpose()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       x.transpose((1, 2, 0)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:int   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Union[str, Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       np.asarray(transpose(__lowerCAmelCase  )  )  )  )

								__magic_name__		:Dict   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       np.asarray(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.reshape(__lowerCAmelCase							,       (4, 3)  )  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (4, 3)  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(1							,       3							,       4  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.squeeze(__lowerCAmelCase  )  )  )

								__magic_name__		:Optional[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.squeeze(__lowerCAmelCase							,       axis=2  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:List[Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:str   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Tuple   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:Tuple   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.asarray(squeeze(__lowerCAmelCase  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.asarray(squeeze(__lowerCAmelCase							,       axis=2  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.expand_dims(__lowerCAmelCase							,       axis=1  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.asarray(expand_dims(__lowerCAmelCase							,       axis=1  )  )  )  )






