




'''simple docstring'''


import inspect
import tempfile
from collections import OrderedDict, UserDict
from collections.abc import MutableMapping
from contextlib import ExitStack, contextmanager
from dataclasses import fields
from enum import Enum
from typing import Any, ContextManager, List, Tuple

import numpy as np

from .import_utils import is_flax_available, is_tf_available, is_torch_available, is_torch_fx_proxy


if is_flax_available():
				import jax.numpy as jnp

class     A					(	_a					):







							def __get__(						self :						str							,    lowerCAmelCase_ :						str							,    lowerCAmelCase_ :						Optional[Any]=None      )    ->							str:


														"""simple docstring"""


														if obj is None:
																					return self
														if self.fget is None:
																					raise AttributeError('''unreadable attribute'''      )
														_a			  =       '''__cached_''' + self.fget.__name__
														_a			  =       getattr(lowerCAmelCase_							,    lowerCAmelCase_							,    lowerCAmelCase_      )
														if cached is None:
																					_a			  =       self.fget(lowerCAmelCase_      )
																					setattr(lowerCAmelCase_							,    lowerCAmelCase_							,    lowerCAmelCase_      )
														return cached



def 	snake_case_					(UpperCamelCase :						str   ):





							'''simple docstring'''


							_a			  =       val.lower()
							if val in {"y", "yes", "t", "true", "on", "1"}:
														return 1
							if val in {"n", "no", "f", "false", "off", "0"}:
														return 0
							raise ValueError(f'invalid truth value {val!r}'   )
def 	snake_case_					(UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							if is_torch_fx_proxy(UpperCamelCase   ):
														return True
							if is_torch_available():
														import torch

														if isinstance(UpperCamelCase				,		torch.Tensor   ):
																					return True
							if is_tf_available():
														import tensorflow as tf

														if isinstance(UpperCamelCase				,		tf.Tensor   ):
																					return True

							if is_flax_available():
														import jax.numpy as jnp
														from jax.core import Tracer

														if isinstance(UpperCamelCase				,		(jnp.ndarray, Tracer)   ):
																					return True

							return isinstance(UpperCamelCase				,		np.ndarray   )
def 	snake_case_					(UpperCamelCase :						List[str]   ):





							'''simple docstring'''


							return isinstance(UpperCamelCase				,		np.ndarray   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]   ):





							'''simple docstring'''


							return _is_numpy(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							import torch

							return isinstance(UpperCamelCase				,		torch.Tensor   )
def 	snake_case_					(UpperCamelCase :						int   ):





							'''simple docstring'''


							return False if not is_torch_available() else _is_torch(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Tuple   ):





							'''simple docstring'''


							import torch

							return isinstance(UpperCamelCase				,		torch.device   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]   ):





							'''simple docstring'''


							return False if not is_torch_available() else _is_torch_device(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Union[str, Any]   ):





							'''simple docstring'''


							import torch

							if isinstance(UpperCamelCase				,		UpperCamelCase   ):
														if hasattr(UpperCamelCase				,		UpperCamelCase   ):
																					_a			  =       getattr(UpperCamelCase				,		UpperCamelCase   )
														else:
																					return False
							return isinstance(UpperCamelCase				,		torch.dtype   )
def 	snake_case_					(UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							return False if not is_torch_available() else _is_torch_dtype(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							import tensorflow as tf

							return isinstance(UpperCamelCase				,		tf.Tensor   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]   ):





							'''simple docstring'''


							return False if not is_tf_available() else _is_tensorflow(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							import tensorflow as tf

							# the `is_symbolic_tensor` predicate is only available starting with TF 2.14
							if hasattr(UpperCamelCase				,		'''is_symbolic_tensor'''   ):
														return tf.is_symbolic_tensor(UpperCamelCase   )
							return type(UpperCamelCase   ) == tf.Tensor
def 	snake_case_					(UpperCamelCase :						Dict   ):





							'''simple docstring'''


							return False if not is_tf_available() else _is_tf_symbolic_tensor(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						List[str]   ):





							'''simple docstring'''


							import jax.numpy as jnp  # noqa: F811

							return isinstance(UpperCamelCase				,		jnp.ndarray   )
def 	snake_case_					(UpperCamelCase :						Union[str, Any]   ):





							'''simple docstring'''


							return False if not is_flax_available() else _is_jax(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						List[str]   ):





							'''simple docstring'''


							if isinstance(UpperCamelCase				,		(dict, UserDict)   ):
														return {k: to_py_obj(UpperCamelCase   ) for k, v in obj.items()}
							elif isinstance(UpperCamelCase				,		(list, tuple)   ):
														return [to_py_obj(UpperCamelCase   ) for o in obj]
							elif is_tf_tensor(UpperCamelCase   ):
														return obj.numpy().tolist()
							elif is_torch_tensor(UpperCamelCase   ):
														return obj.detach().cpu().tolist()
							elif is_jax_tensor(UpperCamelCase   ):
														return np.asarray(UpperCamelCase   ).tolist()
							elif isinstance(UpperCamelCase				,		(np.ndarray, np.number)   ):  # tolist also works on 0d np arrays
														return obj.tolist()
							else:
														return obj
def 	snake_case_					(UpperCamelCase :						Optional[Any]   ):





							'''simple docstring'''


							if isinstance(UpperCamelCase				,		(dict, UserDict)   ):
														return {k: to_numpy(UpperCamelCase   ) for k, v in obj.items()}
							elif isinstance(UpperCamelCase				,		(list, tuple)   ):
														return np.array(UpperCamelCase   )
							elif is_tf_tensor(UpperCamelCase   ):
														return obj.numpy()
							elif is_torch_tensor(UpperCamelCase   ):
														return obj.detach().cpu().numpy()
							elif is_jax_tensor(UpperCamelCase   ):
														return np.asarray(UpperCamelCase   )
							else:
														return obj

class     A					(	_a					):







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							List[Any]:


														"""simple docstring"""


														_a			  =       fields(self      )

														# Safety and consistency checks
														if not len(lowerCAmelCase_      ):
																					raise ValueError(F'{self.__class__.__name__} has no fields.'      )
														if not all(field.default is None for field in class_fields[1:]      ):
																					raise ValueError(F'{self.__class__.__name__} should not have more than one required field.'      )

														_a			  =       getattr(self							,    class_fields[0].name      )
														_a			  =       all(getattr(self							,    field.name      ) is None for field in class_fields[1:]      )

														if other_fields_are_none and not is_tensor(lowerCAmelCase_      ):
																					if isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																												_a			  =       first_field.items()
																												_a			  =       True
																					else:
																												try:
																																			_a			  =       iter(lowerCAmelCase_      )
																																			_a			  =       True
																												except TypeError:
																																			_a			  =       False

            # if we provided an iterator as first field and the iterator is a (key, value) iterator
            # set the associated fields
																					if first_field_iterator:
																												for idx, element in enumerate(lowerCAmelCase_      ):
																																			if (
																																			    not isinstance(lowerCAmelCase_							,    (list, tuple)      )
																																			    or not len(lowerCAmelCase_      ) == 2
																																			    or not isinstance(element[0]							,    lowerCAmelCase_      )
																																			):
																																										if idx == 0:
																																																	# If we do not have an iterator of key/values, set it as attribute
																																																	_a			  =       first_field
																																										else:
																																																	# If we have a mixed iterator, raise an error
																																																	raise ValueError(
																																																	    F'Cannot set key/value for {element}. It needs to be a tuple (key, value).'      )
																																										break
																																			setattr(self							,    element[0]							,    element[1]      )
																																			if element[1] is not None:
																																										_a			  =       element[1]
																					elif first_field is not None:
																												_a			  =       first_field
														else:
																					for field in class_fields:
																												_a			  =       getattr(self							,    field.name      )
																												if v is not None:
																																			_a			  =       v







							def __delitem__(						self :						Any							,    *lowerCAmelCase_ :						Any							,    **lowerCAmelCase_ :						Optional[int]      )    ->							Optional[Any]:


														"""simple docstring"""


														raise Exception(F'You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.'      )







							def      __lowerCAmelCase    (						self :						int							,    *lowerCAmelCase_ :						Dict							,    **lowerCAmelCase_ :						List[Any]      )    ->							List[Any]:


														"""simple docstring"""


														raise Exception(F'You cannot use ``setdefault`` on a {self.__class__.__name__} instance.'      )







							def      __lowerCAmelCase    (						self :						str							,    *lowerCAmelCase_ :						Tuple							,    **lowerCAmelCase_ :						Optional[Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														raise Exception(F'You cannot use ``pop`` on a {self.__class__.__name__} instance.'      )







							def      __lowerCAmelCase    (						self :						List[str]							,    *lowerCAmelCase_ :						Any							,    **lowerCAmelCase_ :						List[Any]      )    ->							Optional[Any]:


														"""simple docstring"""


														raise Exception(F'You cannot use ``update`` on a {self.__class__.__name__} instance.'      )







							def __getitem__(						self :						List[str]							,    lowerCAmelCase_ :						int      )    ->							Optional[Any]:


														"""simple docstring"""


														if isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																					_a			  =       dict(self.items()      )
																					return inner_dict[k]
														else:
																					return self.to_tuple()[k]







							def __setattr__(						self :						Union[str, Any]							,    lowerCAmelCase_ :						List[str]							,    lowerCAmelCase_ :						Union[str, Any]      )    ->							Tuple:


														"""simple docstring"""


														if name in self.keys() and value is not None:
																					# Don't call self.__setitem__ to avoid recursion errors
																					super().__setitem__(lowerCAmelCase_							,    lowerCAmelCase_      )
														super().__setattr__(lowerCAmelCase_							,    lowerCAmelCase_      )







							def __setitem__(						self :						Any							,    lowerCAmelCase_ :						Optional[int]							,    lowerCAmelCase_ :						str      )    ->							Union[str, Any]:


														"""simple docstring"""


														super().__setitem__(lowerCAmelCase_							,    lowerCAmelCase_      )
														# Don't call self.__setattr__ to avoid recursion errors
														super().__setattr__(lowerCAmelCase_							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Dict      )    ->							Tuple[Any]:


														"""simple docstring"""


														return tuple(self[k] for k in self.keys()      )



class     A					(	_a  ,_a					):







							@classmethod
							def      __lowerCAmelCase    (						cls :						Union[str, Any]							,    lowerCAmelCase_ :						Dict      )    ->							Tuple:


														"""simple docstring"""


														raise ValueError(
														    F'{value} is not a valid {cls.__name__}, please select one of {list(cls._valueamember_map_.keys()      )}'      )



class     A					(	_a					):
							lowercase_           =       'longest'
							lowercase_           =       'max_length'
							lowercase_           =       'do_not_pad'



class     A					(	_a					):
							lowercase_           =       'pt'
							lowercase_           =       'tf'
							lowercase_           =       'np'
							lowercase_           =       'jax'



class     A					:







							def __init__(						self :						int							,    lowerCAmelCase_ :						List[ContextManager]      )    ->							Any:


														"""simple docstring"""


														_a			  =       context_managers
														_a			  =       ExitStack()







							def __enter__(						self :						Union[str, Any]      )    ->							int:


														"""simple docstring"""


														for context_manager in self.context_managers:
																					self.stack.enter_context(lowerCAmelCase_      )







							def __exit__(						self :						int							,    *lowerCAmelCase_ :						str							,    **lowerCAmelCase_ :						Any      )    ->							Tuple:


														"""simple docstring"""


														self.stack.__exit__(*lowerCAmelCase_							,    **lowerCAmelCase_      )



def 	snake_case_					(UpperCamelCase :						int   ):





							'''simple docstring'''


							_a			  =       infer_framework(UpperCamelCase   )
							if framework == "tf":
														_a			  =       inspect.signature(model_class.call   )  # TensorFlow models
							elif framework == "pt":
														_a			  =       inspect.signature(model_class.forward   )  # PyTorch models
							else:
														_a			  =       inspect.signature(model_class.__call__   )  # Flax models

							for p in signature.parameters:
														if p == "return_loss" and signature.parameters[p].default is True:
																					return True

							return False
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							_a			  =       model_class.__name__
							_a			  =       infer_framework(UpperCamelCase   )
							if framework == "tf":
														_a			  =       inspect.signature(model_class.call   )  # TensorFlow models
							elif framework == "pt":
														_a			  =       inspect.signature(model_class.forward   )  # PyTorch models
							else:
														_a			  =       inspect.signature(model_class.__call__   )  # Flax models

							if "QuestionAnswering" in model_name:
														return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")]
							else:
														return [p for p in signature.parameters if "label" in p]
def 	snake_case_					(UpperCamelCase :						MutableMapping				,		UpperCamelCase :						str = ""				,		UpperCamelCase :						str = "."   ):





							'''simple docstring'''


							def _flatten_dict(UpperCamelCase :						str				,		UpperCamelCase :						Union[str, Any]=""				,		UpperCamelCase :						Union[str, Any]="."   ):
														for k, v in d.items():
																					_a			  =       str(UpperCamelCase   ) + delimiter + str(UpperCamelCase   ) if parent_key else k
																					if v and isinstance(UpperCamelCase				,		UpperCamelCase   ):
																												yield from flatten_dict(UpperCamelCase				,		UpperCamelCase				,		delimiter=UpperCamelCase   ).items()
																					else:
																												yield key, v

							return dict(_flatten_dict(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )   )
@contextmanager
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						bool = False   ):





							'''simple docstring'''


							if use_temp_dir:
														with tempfile.TemporaryDirectory() as tmp_dir:
																					yield tmp_dir
							else:
														yield working_dir
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						Optional[int]=None   ):





							'''simple docstring'''


							if is_numpy_array(UpperCamelCase   ):
														return np.transpose(UpperCamelCase				,		axes=UpperCamelCase   )
							elif is_torch_tensor(UpperCamelCase   ):
														return array.T if axes is None else array.permute(*UpperCamelCase   )
							elif is_tf_tensor(UpperCamelCase   ):
														import tensorflow as tf

														return tf.transpose(UpperCamelCase				,		perm=UpperCamelCase   )
							elif is_jax_tensor(UpperCamelCase   ):
														return jnp.transpose(UpperCamelCase				,		axes=UpperCamelCase   )
							else:
														raise ValueError(f'Type not supported for transpose: {type(UpperCamelCase   )}.'   )
def 	snake_case_					(UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						Dict   ):





							'''simple docstring'''


							if is_numpy_array(UpperCamelCase   ):
														return np.reshape(UpperCamelCase				,		UpperCamelCase   )
							elif is_torch_tensor(UpperCamelCase   ):
														return array.reshape(*UpperCamelCase   )
							elif is_tf_tensor(UpperCamelCase   ):
														import tensorflow as tf

														return tf.reshape(UpperCamelCase				,		UpperCamelCase   )
							elif is_jax_tensor(UpperCamelCase   ):
														return jnp.reshape(UpperCamelCase				,		UpperCamelCase   )
							else:
														raise ValueError(f'Type not supported for reshape: {type(UpperCamelCase   )}.'   )
def 	snake_case_					(UpperCamelCase :						Tuple				,		UpperCamelCase :						Optional[Any]=None   ):





							'''simple docstring'''


							if is_numpy_array(UpperCamelCase   ):
														return np.squeeze(UpperCamelCase				,		axis=UpperCamelCase   )
							elif is_torch_tensor(UpperCamelCase   ):
														return array.squeeze() if axis is None else array.squeeze(dim=UpperCamelCase   )
							elif is_tf_tensor(UpperCamelCase   ):
														import tensorflow as tf

														return tf.squeeze(UpperCamelCase				,		axis=UpperCamelCase   )
							elif is_jax_tensor(UpperCamelCase   ):
														return jnp.squeeze(UpperCamelCase				,		axis=UpperCamelCase   )
							else:
														raise ValueError(f'Type not supported for squeeze: {type(UpperCamelCase   )}.'   )
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						str   ):





							'''simple docstring'''


							if is_numpy_array(UpperCamelCase   ):
														return np.expand_dims(UpperCamelCase				,		UpperCamelCase   )
							elif is_torch_tensor(UpperCamelCase   ):
														return array.unsqueeze(dim=UpperCamelCase   )
							elif is_tf_tensor(UpperCamelCase   ):
														import tensorflow as tf

														return tf.expand_dims(UpperCamelCase				,		axis=UpperCamelCase   )
							elif is_jax_tensor(UpperCamelCase   ):
														return jnp.expand_dims(UpperCamelCase				,		axis=UpperCamelCase   )
							else:
														raise ValueError(f'Type not supported for expand_dims: {type(UpperCamelCase   )}.'   )
def 	snake_case_					(UpperCamelCase :						Dict   ):





							'''simple docstring'''


							if is_numpy_array(UpperCamelCase   ):
														return np.size(UpperCamelCase   )
							elif is_torch_tensor(UpperCamelCase   ):
														return array.numel()
							elif is_tf_tensor(UpperCamelCase   ):
														import tensorflow as tf

														return tf.size(UpperCamelCase   )
							elif is_jax_tensor(UpperCamelCase   ):
														return array.size
							else:
														raise ValueError(f'Type not supported for expand_dims: {type(UpperCamelCase   )}.'   )
def 	snake_case_					(UpperCamelCase :						List[str]				,		UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							for key, value in auto_map.items():
														if isinstance(UpperCamelCase				,		(tuple, list)   ):
																					_a			  =       [f'{repo_id}--{v}' if (v is not None and '''--''' not in v) else v for v in value]
														elif value is not None and "--" not in value:
																					_a			  =       f'{repo_id}--{value}'

							return auto_map

def 	snake_case_					(UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							for base_class in inspect.getmro(UpperCamelCase   ):
														_a			  =       base_class.__module__
														_a			  =       base_class.__name__
														if module.startswith('''tensorflow'''   ) or module.startswith('''keras'''   ) or name == "TFPreTrainedModel":
																					return "tf"
														elif module.startswith('''torch'''   ) or name == "PreTrainedModel":
																					return "pt"
														elif module.startswith('''flax'''   ) or module.startswith('''jax'''   ) or name == "FlaxPreTrainedModel":
																					return "flax"
							else:
														raise TypeError(f'Could not infer framework from class {model_class}.'   )







'''simple docstring'''


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case   :   str     							=	get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
				from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_snake_case   :   Union[str, Any]     							=	256047
_snake_case   :   Tuple     							=	256145

@require_sentencepiece
@require_tokenizers
class     A					(	_a  ,unittest.TestCase					):
							lowercase_           =       NllbTokenizer
							lowercase_           =       NllbTokenizerFast
							lowercase_           =       True
							lowercase_           =       True
							lowercase_           =       {}







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							int:


														"""simple docstring"""


														super().setUp()

														# We have a SentencePiece fixture for testing
														_a			  =       NllbTokenizer(lowerCAmelCase_							,    keep_accents=lowerCAmelCase_      )
														tokenizer.save_pretrained(self.tmpdirname      )







							def      __lowerCAmelCase    (						self :						Tuple      )    ->							Union[str, Any]:


														"""simple docstring"""


														_a			  =       NllbTokenizer(lowerCAmelCase_							,    keep_accents=lowerCAmelCase_      )

														_a			  =       tokenizer.tokenize('''This is a test'''      )
														self.assertListEqual(lowerCAmelCase_							,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )							,    [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]							,    )

														_a			  =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [
														        SPIECE_UNDERLINE + '''I''',
														        SPIECE_UNDERLINE + '''was''',
														        SPIECE_UNDERLINE + '''b''',
														        '''or''',
														        '''n''',
														        SPIECE_UNDERLINE + '''in''',
														        SPIECE_UNDERLINE + '''''',
														        '''9''',
														        '''2''',
														        '''0''',
														        '''0''',
														        '''0''',
														        ''',''',
														        SPIECE_UNDERLINE + '''and''',
														        SPIECE_UNDERLINE + '''this''',
														        SPIECE_UNDERLINE + '''is''',
														        SPIECE_UNDERLINE + '''f''',
														        '''al''',
														        '''s''',
														        '''é''',
														        '''.''',
														    ]							,    )
														_a			  =       tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [
														        value + tokenizer.fairseq_offset
														        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
														    ]							,    )

														_a			  =       tokenizer.convert_ids_to_tokens(lowerCAmelCase_      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [
														        SPIECE_UNDERLINE + '''I''',
														        SPIECE_UNDERLINE + '''was''',
														        SPIECE_UNDERLINE + '''b''',
														        '''or''',
														        '''n''',
														        SPIECE_UNDERLINE + '''in''',
														        SPIECE_UNDERLINE + '''''',
														        '''<unk>''',
														        '''2''',
														        '''0''',
														        '''0''',
														        '''0''',
														        ''',''',
														        SPIECE_UNDERLINE + '''and''',
														        SPIECE_UNDERLINE + '''this''',
														        SPIECE_UNDERLINE + '''is''',
														        SPIECE_UNDERLINE + '''f''',
														        '''al''',
														        '''s''',
														        '''<unk>''',
														        '''.''',
														    ]							,    )







							def      __lowerCAmelCase    (						self :						Optional[int]      )    ->							Union[str, Any]:


														"""simple docstring"""


														_a			  =       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																												_a			  =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )
																												_a			  =       self.tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )

																												_a			  =       tempfile.mkdtemp()

																												_a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_      )
																												_a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

																												# Checks it save with the same files + the tokenizer.json file for the fast one
																												self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )
																												_a			  =       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f      )
																												self.assertSequenceEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

																												# Checks everything loads correctly in the same way
																												_a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
																												_a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )

																												shutil.rmtree(lowerCAmelCase_      )

																												# Save tokenizer rust, legacy_format=True
																												_a			  =       tempfile.mkdtemp()

																												_a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_							,    legacy_format=lowerCAmelCase_      )
																												_a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

																												# Checks it save with the same files
																												self.assertSequenceEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

																												# Checks everything loads correctly in the same way
																												_a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
																												_a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )

																												shutil.rmtree(lowerCAmelCase_      )

																												# Save tokenizer rust, legacy_format=False
																												_a			  =       tempfile.mkdtemp()

																												_a			  =       tokenizer_r.save_pretrained(lowerCAmelCase_							,    legacy_format=lowerCAmelCase_      )
																												_a			  =       tokenizer_p.save_pretrained(lowerCAmelCase_      )

																												# Checks it saved the tokenizer.json file
																												self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )

																												# Checks everything loads correctly in the same way
																												_a			  =       tokenizer_r.from_pretrained(lowerCAmelCase_      )
																												_a			  =       tokenizer_p.from_pretrained(lowerCAmelCase_      )

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(lowerCAmelCase_							,    lowerCAmelCase_      )      )

																												shutil.rmtree(lowerCAmelCase_      )







							@require_torch
							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Dict:


														"""simple docstring"""


														if not self.test_seqaseq:
																					return

														_a			  =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'{tokenizer.__class__.__name__}'      ):
																												# Longer text that will definitely require truncation.
																												_a			  =       [
																												    ''' UN Chief Says There Is No Military Solution in Syria''',
																												    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
																												    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
																												    ''' will only worsen the violence and misery for millions of people.''',
																												]
																												_a			  =       [
																												    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
																												    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
																												    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
																												    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
																												]
																												try:
																																			_a			  =       tokenizer.prepare_seqaseq_batch(
																																			    src_texts=lowerCAmelCase_							,    tgt_texts=lowerCAmelCase_							,    max_length=3							,    max_target_length=10							,    return_tensors='''pt'''							,    src_lang='''eng_Latn'''							,    tgt_lang='''ron_Latn'''							,    )
																												except NotImplementedError:
																																			return
																												self.assertEqual(batch.input_ids.shape[1]							,    3      )
																												self.assertEqual(batch.labels.shape[1]							,    10      )
																												# max_target_length will default to max_length if not specified
																												_a			  =       tokenizer.prepare_seqaseq_batch(
																												    lowerCAmelCase_							,    tgt_texts=lowerCAmelCase_							,    max_length=3							,    return_tensors='''pt'''      )
																												self.assertEqual(batch.input_ids.shape[1]							,    3      )
																												self.assertEqual(batch.labels.shape[1]							,    3      )

																												_a			  =       tokenizer.prepare_seqaseq_batch(
																												    src_texts=lowerCAmelCase_							,    max_length=3							,    max_target_length=10							,    return_tensors='''pt'''      )
																												self.assertEqual(batch_encoder_only.input_ids.shape[1]							,    3      )
																												self.assertEqual(batch_encoder_only.attention_mask.shape[1]							,    3      )
																												self.assertNotIn('''decoder_input_ids'''							,    lowerCAmelCase_      )







							@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''      )
							def      __lowerCAmelCase    (						self :						Dict      )    ->							Tuple:


														"""simple docstring"""


														pass







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							Optional[Any]:


														"""simple docstring"""


														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																												_a			  =       [AddedToken('''<special>'''							,    lstrip=lowerCAmelCase_      )]

																												_a			  =       self.rust_tokenizer_class.from_pretrained(
																												    lowerCAmelCase_							,    additional_special_tokens=lowerCAmelCase_							,    **lowerCAmelCase_      )
																												_a			  =       tokenizer_r.encode('''Hey this is a <special> token'''      )

																												_a			  =       tokenizer_r.encode('''<special>'''							,    add_special_tokens=lowerCAmelCase_      )[0]

																												self.assertTrue(special_token_id in r_output      )

																												if self.test_slow_tokenizer:
																																			_a			  =       self.rust_tokenizer_class.from_pretrained(
																																			    lowerCAmelCase_							,    additional_special_tokens=lowerCAmelCase_							,    **lowerCAmelCase_							,    )
																																			_a			  =       self.tokenizer_class.from_pretrained(
																																			    lowerCAmelCase_							,    additional_special_tokens=lowerCAmelCase_							,    **lowerCAmelCase_      )

																																			_a			  =       tokenizer_p.encode('''Hey this is a <special> token'''      )

																																			_a			  =       tokenizer_cr.encode('''Hey this is a <special> token'''      )

																																			self.assertEqual(lowerCAmelCase_							,    lowerCAmelCase_      )
																																			self.assertEqual(lowerCAmelCase_							,    lowerCAmelCase_      )
																																			self.assertTrue(special_token_id in p_output      )
																																			self.assertTrue(special_token_id in cr_output      )



@require_torch
@require_sentencepiece
@require_tokenizers
class     A					(	unittest.TestCase					):
							lowercase_           =       'facebook/nllb-200-distilled-600M'
							lowercase_           =       [
							    ' UN Chief Says There Is No Military Solution in Syria',
							    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
							]
							lowercase_           =       [
							    'Şeful ONU declară că nu există o soluţie militară în Siria',
							    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
							    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
							    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
							]
							lowercase_           =       [
							    25_6047,
							    1_6297,
							    13_4408,
							    8165,
							    24_8066,
							    1_4734,
							    950,
							    1135,
							    10_5721,
							    3573,
							    83,
							    2_7352,
							    108,
							    4_9486,
							    2,
							]







							@classmethod
							def      __lowerCAmelCase    (						cls :						Tuple      )    ->							Optional[int]:


														"""simple docstring"""


														_a			  =       NllbTokenizer.from_pretrained(
														    cls.checkpoint_name							,    src_lang='''eng_Latn'''							,    tgt_lang='''ron_Latn'''      )
														_a			  =       1
														return cls







							def      __lowerCAmelCase    (						self :						str      )    ->							Union[str, Any]:


														"""simple docstring"""


														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']							,    25_60_01      )
														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']							,    25_60_02      )
														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']							,    25_60_57      )







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							int:


														"""simple docstring"""


														_a			  =       self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
														self.assertListEqual(self.expected_src_tokens							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						int      )    ->							List[Any]:


														"""simple docstring"""


														self.assertIn(lowerCAmelCase_							,    self.tokenizer.all_special_ids      )
														# fmt: off
														_a			  =       [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
														# fmt: on

														_a			  =       self.tokenizer.decode(lowerCAmelCase_							,    skip_special_tokens=lowerCAmelCase_      )
														_a			  =       self.tokenizer.decode(generated_ids[1:]							,    skip_special_tokens=lowerCAmelCase_      )
														self.assertEqual(lowerCAmelCase_							,    lowerCAmelCase_      )
														self.assertNotIn(self.tokenizer.eos_token							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Dict      )    ->							Union[str, Any]:


														"""simple docstring"""


														_a			  =       ['''this is gunna be a long sentence ''' * 20]
														assert isinstance(src_text[0]							,    lowerCAmelCase_      )
														_a			  =       10
														_a			  =       self.tokenizer(lowerCAmelCase_							,    max_length=lowerCAmelCase_							,    truncation=lowerCAmelCase_      ).input_ids[0]
														self.assertEqual(ids[-1]							,    2      )
														self.assertEqual(ids[0]							,    lowerCAmelCase_      )
														self.assertEqual(len(lowerCAmelCase_      )							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							Optional[int]:


														"""simple docstring"""


														self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']      )							,    [25_62_03, 3]      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       tempfile.mkdtemp()
														_a			  =       self.tokenizer.fairseq_tokens_to_ids
														self.tokenizer.save_pretrained(lowerCAmelCase_      )
														_a			  =       NllbTokenizer.from_pretrained(lowerCAmelCase_      )
														self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,    lowerCAmelCase_      )







							@require_torch
							def      __lowerCAmelCase    (						self :						str      )    ->							Optional[int]:


														"""simple docstring"""


														_a			  =       self.tokenizer(
														    self.src_text							,    text_target=self.tgt_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=len(self.expected_src_tokens      )							,    return_tensors='''pt'''							,    )
														_a			  =       shift_tokens_right(
														    batch['''labels''']							,    self.tokenizer.pad_token_id							,    self.tokenizer.lang_code_to_id['''ron_Latn''']      )

														self.assertIsInstance(lowerCAmelCase_							,    lowerCAmelCase_      )

														self.assertEqual((2, 15)							,    batch.input_ids.shape      )
														self.assertEqual((2, 15)							,    batch.attention_mask.shape      )
														_a			  =       batch.input_ids.tolist()[0]
														self.assertListEqual(self.expected_src_tokens							,    lowerCAmelCase_      )
														self.assertEqual(lowerCAmelCase_							,    batch.decoder_input_ids[0, 0]      )  # EOS
														# Test that special tokens are reset
														self.assertEqual(self.tokenizer.prefix_tokens							,    [EN_CODE]      )
														self.assertEqual(self.tokenizer.suffix_tokens							,    [self.tokenizer.eos_token_id]      )







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							Any:


														"""simple docstring"""


														_a			  =       self.tokenizer(self.src_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=3							,    return_tensors='''pt'''      )
														_a			  =       self.tokenizer(
														    text_target=self.tgt_text							,    padding=lowerCAmelCase_							,    truncation=lowerCAmelCase_							,    max_length=10							,    return_tensors='''pt'''      )
														_a			  =       targets['''input_ids''']
														_a			  =       shift_tokens_right(
														    lowerCAmelCase_							,    self.tokenizer.pad_token_id							,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]							,    )

														self.assertEqual(batch.input_ids.shape[1]							,    3      )
														self.assertEqual(batch.decoder_input_ids.shape[1]							,    10      )







							@require_torch
							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       self.tokenizer._build_translation_inputs(
														    '''A test'''							,    return_tensors='''pt'''							,    src_lang='''eng_Latn'''							,    tgt_lang='''fra_Latn'''      )

														self.assertEqual(
														    nested_simplify(lowerCAmelCase_      )							,    {
														        # A, test, EOS, en_XX
														        '''input_ids''': [[25_60_47, 70, 73_56, 2]],
														        '''attention_mask''': [[1, 1, 1, 1]],
														        # ar_AR
														        '''forced_bos_token_id''': 25_60_57,
														    }							,    )







							@require_torch
							def      __lowerCAmelCase    (						self :						Any      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       True
														_a			  =       self.tokenizer(
														    '''UN Chief says there is no military solution in Syria'''							,    src_lang='''eng_Latn'''							,    tgt_lang='''fra_Latn'''      )
														self.assertEqual(
														    inputs.input_ids							,    [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]      )

														_a			  =       False
														_a			  =       self.tokenizer(
														    '''UN Chief says there is no military solution in Syria'''							,    src_lang='''eng_Latn'''							,    tgt_lang='''fra_Latn'''      )
														self.assertEqual(
														    inputs.input_ids							,    [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]      )

