


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase      =   get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
   from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase      =   25_6047
UpperCAmelCase      =   25_6145







@require_sentencepiece
@require_tokenizers
class lowerCAmelCase_						(			lowerCamelCase__ ,						unittest.TestCase	):

 '''simple docstring'''

 __snake_case												=  NllbTokenizer
 __snake_case												=  NllbTokenizerFast
 __snake_case												=  True
 __snake_case												=  True
 __snake_case												=  {}



 def 						UpperCamelCase__					(     self ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       snake_case_	=				NllbTokenizer(_UpperCAmelCase		,				keep_accents=_UpperCAmelCase )
       tokenizer.save_pretrained(self.tmpdirname )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				NllbTokenizer(_UpperCAmelCase		,				keep_accents=_UpperCAmelCase )

       snake_case_	=				tokenizer.tokenize('''This is a test''' )
       self.assertListEqual(_UpperCAmelCase		,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_UpperCAmelCase )		,				[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,				)

       snake_case_	=				tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
       self.assertListEqual(
           _UpperCAmelCase		,				[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]		,				)
       snake_case_	=				tokenizer.convert_tokens_to_ids(_UpperCAmelCase )
       self.assertListEqual(
           _UpperCAmelCase		,				[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]		,				)

       snake_case_	=				tokenizer.convert_ids_to_tokens(_UpperCAmelCase )
       self.assertListEqual(
           _UpperCAmelCase		,				[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ]		,				)



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                   snake_case_	=				self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase		,				**_UpperCAmelCase )
                   snake_case_	=				self.tokenizer_class.from_pretrained(_UpperCAmelCase		,				**_UpperCAmelCase )

                   snake_case_	=				tempfile.mkdtemp()

                   snake_case_	=				tokenizer_r.save_pretrained(_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.save_pretrained(_UpperCAmelCase )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )
                   snake_case_	=				tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f )
                   self.assertSequenceEqual(_UpperCAmelCase		,				_UpperCAmelCase )

                   # Checks everything loads correctly in the same way
                   snake_case_	=				tokenizer_r.from_pretrained(_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.from_pretrained(_UpperCAmelCase )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_UpperCAmelCase		,				_UpperCAmelCase ) )

                   shutil.rmtree(_UpperCAmelCase )

                   # Save tokenizer rust, legacy_format=True
                   snake_case_	=				tempfile.mkdtemp()

                   snake_case_	=				tokenizer_r.save_pretrained(_UpperCAmelCase		,				legacy_format=_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.save_pretrained(_UpperCAmelCase )

                   # Checks it save with the same files
                   self.assertSequenceEqual(_UpperCAmelCase		,				_UpperCAmelCase )

                   # Checks everything loads correctly in the same way
                   snake_case_	=				tokenizer_r.from_pretrained(_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.from_pretrained(_UpperCAmelCase )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_UpperCAmelCase		,				_UpperCAmelCase ) )

                   shutil.rmtree(_UpperCAmelCase )

                   # Save tokenizer rust, legacy_format=False
                   snake_case_	=				tempfile.mkdtemp()

                   snake_case_	=				tokenizer_r.save_pretrained(_UpperCAmelCase		,				legacy_format=_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.save_pretrained(_UpperCAmelCase )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )

                   # Checks everything loads correctly in the same way
                   snake_case_	=				tokenizer_r.from_pretrained(_UpperCAmelCase )
                   snake_case_	=				tokenizer_p.from_pretrained(_UpperCAmelCase )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_UpperCAmelCase		,				_UpperCAmelCase ) )

                   shutil.rmtree(_UpperCAmelCase )



 @require_torch
 def 						UpperCamelCase__					(     self ):
       if not self.test_seqaseq:
             return

       snake_case_	=				self.get_tokenizers()
       for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}''' ):
                   # Longer text that will definitely require truncation.
                   snake_case_	=				[
                       ''' UN Chief Says There Is No Military Solution in Syria''',
                       ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                       ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                       ''' will only worsen the violence and misery for millions of people.''',
                   ]
                   snake_case_	=				[
                       '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                       '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                       ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                       ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                   ]
                   try:
                         snake_case_	=				tokenizer.prepare_seqaseq_batch(
                             src_texts=_UpperCAmelCase		,				tgt_texts=_UpperCAmelCase		,				max_length=3		,				max_target_length=10		,				return_tensors='''pt'''		,				src_lang='''eng_Latn'''		,				tgt_lang='''ron_Latn'''		,				)
                   except NotImplementedError:
                         return
                   self.assertEqual(batch.input_ids.shape[1]		,				3 )
                   self.assertEqual(batch.labels.shape[1]		,				10 )
                   # max_target_length will default to max_length if not specified
                   snake_case_	=				tokenizer.prepare_seqaseq_batch(
                       _UpperCAmelCase		,				tgt_texts=_UpperCAmelCase		,				max_length=3		,				return_tensors='''pt''' )
                   self.assertEqual(batch.input_ids.shape[1]		,				3 )
                   self.assertEqual(batch.labels.shape[1]		,				3 )

                   snake_case_	=				tokenizer.prepare_seqaseq_batch(
                       src_texts=_UpperCAmelCase		,				max_length=3		,				max_target_length=10		,				return_tensors='''pt''' )
                   self.assertEqual(batch_encoder_only.input_ids.shape[1]		,				3 )
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1]		,				3 )
                   self.assertNotIn('''decoder_input_ids'''		,				_UpperCAmelCase )



 @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.''' )
 def 						UpperCamelCase__					(     self ):
       pass





 def 						UpperCamelCase__					(     self ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                   snake_case_	=				[AddedToken('''<special>'''		,				lstrip=_UpperCAmelCase )]

                   snake_case_	=				self.rust_tokenizer_class.from_pretrained(
                       _UpperCAmelCase		,				additional_special_tokens=_UpperCAmelCase		,				**_UpperCAmelCase )
                   snake_case_	=				tokenizer_r.encode('''Hey this is a <special> token''' )

                   snake_case_	=				tokenizer_r.encode('''<special>'''		,				add_special_tokens=_UpperCAmelCase )[0]

                   self.assertTrue(special_token_id in r_output )

                   if self.test_slow_tokenizer:
                         snake_case_	=				self.rust_tokenizer_class.from_pretrained(
                             _UpperCAmelCase		,				additional_special_tokens=_UpperCAmelCase		,				**_UpperCAmelCase		,				)
                         snake_case_	=				self.tokenizer_class.from_pretrained(
                             _UpperCAmelCase		,				additional_special_tokens=_UpperCAmelCase		,				**_UpperCAmelCase )

                         snake_case_	=				tokenizer_p.encode('''Hey this is a <special> token''' )

                         snake_case_	=				tokenizer_cr.encode('''Hey this is a <special> token''' )

                         self.assertEqual(_UpperCAmelCase		,				_UpperCAmelCase )
                         self.assertEqual(_UpperCAmelCase		,				_UpperCAmelCase )
                         self.assertTrue(special_token_id in p_output )
                         self.assertTrue(special_token_id in cr_output )







@require_torch
@require_sentencepiece
@require_tokenizers
class lowerCAmelCase_						(			unittest.TestCase	):

 '''simple docstring'''

 __snake_case												=  "facebook/nllb-200-distilled-600M"
 __snake_case												=  [
     " UN Chief Says There Is No Military Solution in Syria",
     " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
 ]
 __snake_case												=  [
     "Şeful ONU declară că nu există o soluţie militară în Siria",
     "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
     " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
     " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
 ]
 __snake_case												=  [
     2_5_6_0_4_7,
     1_6_2_9_7,
     1_3_4_4_0_8,
     8_1_6_5,
     2_4_8_0_6_6,
     1_4_7_3_4,
     9_5_0,
     1_1_3_5,
     1_0_5_7_2_1,
     3_5_7_3,
     8_3,
     2_7_3_5_2,
     1_0_8,
     4_9_4_8_6,
     2,
 ]



 @classmethod
 def 						UpperCamelCase__					(     cls ):
       snake_case_	=				NllbTokenizer.from_pretrained(
           cls.checkpoint_name		,				src_lang='''eng_Latn'''		,				tgt_lang='''ron_Latn''' )
       snake_case_	=				1
       return cls



 def 						UpperCamelCase__					(     self ):
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']		,				25_60_01 )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']		,				25_60_02 )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']		,				25_60_57 )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       self.assertIn(_UpperCAmelCase		,				self.tokenizer.all_special_ids )
       # fmt: off
       snake_case_	=				[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
       # fmt: on

       snake_case_	=				self.tokenizer.decode(_UpperCAmelCase		,				skip_special_tokens=_UpperCAmelCase )
       snake_case_	=				self.tokenizer.decode(generated_ids[1:]		,				skip_special_tokens=_UpperCAmelCase )
       self.assertEqual(_UpperCAmelCase		,				_UpperCAmelCase )
       self.assertNotIn(self.tokenizer.eos_token		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				['''this is gunna be a long sentence ''' * 20]
       assert isinstance(src_text[0]		,				_UpperCAmelCase )
       snake_case_	=				10
       snake_case_	=				self.tokenizer(_UpperCAmelCase		,				max_length=_UpperCAmelCase		,				truncation=_UpperCAmelCase ).input_ids[0]
       self.assertEqual(ids[-1]		,				2 )
       self.assertEqual(ids[0]		,				_UpperCAmelCase )
       self.assertEqual(len(_UpperCAmelCase )		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR'''] )		,				[25_62_03, 3] )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				tempfile.mkdtemp()
       snake_case_	=				self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(_UpperCAmelCase )
       snake_case_	=				NllbTokenizer.from_pretrained(_UpperCAmelCase )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,				_UpperCAmelCase )



 @require_torch
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.tokenizer(
           self.src_text		,				text_target=self.tgt_text		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				max_length=len(self.expected_src_tokens )		,				return_tensors='''pt'''		,				)
       snake_case_	=				shift_tokens_right(
           batch['''labels''']		,				self.tokenizer.pad_token_id		,				self.tokenizer.lang_code_to_id['''ron_Latn'''] )

       self.assertIsInstance(_UpperCAmelCase		,				_UpperCAmelCase )

       self.assertEqual((2, 15)		,				batch.input_ids.shape )
       self.assertEqual((2, 15)		,				batch.attention_mask.shape )
       snake_case_	=				batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens		,				_UpperCAmelCase )
       self.assertEqual(_UpperCAmelCase		,				batch.decoder_input_ids[0, 0] )  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens		,				[EN_CODE] )
       self.assertEqual(self.tokenizer.suffix_tokens		,				[self.tokenizer.eos_token_id] )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.tokenizer(self.src_text		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				max_length=3		,				return_tensors='''pt''' )
       snake_case_	=				self.tokenizer(
           text_target=self.tgt_text		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				max_length=10		,				return_tensors='''pt''' )
       snake_case_	=				targets['''input_ids''']
       snake_case_	=				shift_tokens_right(
           _UpperCAmelCase		,				self.tokenizer.pad_token_id		,				decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]		,				)

       self.assertEqual(batch.input_ids.shape[1]		,				3 )
       self.assertEqual(batch.decoder_input_ids.shape[1]		,				10 )



 @require_torch
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.tokenizer._build_translation_inputs(
           '''A test'''		,				return_tensors='''pt'''		,				src_lang='''eng_Latn'''		,				tgt_lang='''fra_Latn''' )

       self.assertEqual(
           nested_simplify(_UpperCAmelCase )		,				{
               # A, test, EOS, en_XX
               '''input_ids''': [[25_60_47, 70, 73_56, 2]],
               '''attention_mask''': [[1, 1, 1, 1]],
               # ar_AR
               '''forced_bos_token_id''': 25_60_57,
           }		,				)





 @require_torch
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				True
       snake_case_	=				self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''		,				src_lang='''eng_Latn'''		,				tgt_lang='''fra_Latn''' )
       self.assertEqual(
           inputs.input_ids		,				[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47] )

       snake_case_	=				False
       snake_case_	=				self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''		,				src_lang='''eng_Latn'''		,				tgt_lang='''fra_Latn''' )
       self.assertEqual(
           inputs.input_ids		,				[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2] )



import numpy as np

import datasets


UpperCAmelCase      =   """
Compute the Mahalanobis Distance

Mahalonobis distance is the distance between a point and a distribution.
And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance.
It was introduced by Prof. P. C. Mahalanobis in 1936
and has been used in various statistical applications ever since
[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/]
"""

UpperCAmelCase      =   """\
@article{de2000mahalanobis,
  title={The mahalanobis distance},
  author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L},
  journal={Chemometrics and intelligent laboratory systems},
  volume={50},
  number={1},
  pages={1--18},
  year={2000},
  publisher={Elsevier}
}
"""

UpperCAmelCase      =   """
Args:
    X: List of datapoints to be compared with the `reference_distribution`.
    reference_distribution: List of datapoints from the reference distribution we want to compare to.
Returns:
    mahalanobis: The Mahalonobis distance for each datapoint in `X`.
Examples:

    >>> mahalanobis_metric = datasets.load_metric(\"mahalanobis\")
    >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]])
    >>> print(results)
    {'mahalanobis': array([0.5])}
"""







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,						_KWARGS_DESCRIPTION	)
class lowerCAmelCase_						(			datasets.Metric	):

 '''simple docstring'''

 def 						UpperCamelCase__					(     self ):
       return datasets.MetricInfo(
           description=_DESCRIPTION		,				citation=_CITATION		,				inputs_description=_KWARGS_DESCRIPTION		,				features=datasets.Features(
               {
                   '''X''': datasets.Sequence(datasets.Value('''float'''		,				id='''sequence''' )		,				id='''X''' ),
               } )		,				)





 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase ):
       # convert to numpy arrays
       snake_case_	=				np.array(_UpperCAmelCase )
       snake_case_	=				np.array(_UpperCAmelCase )

       # Assert that arrays are 2D
       if len(X.shape ) != 2:
             raise ValueError('''Expected `X` to be a 2D vector''' )
       if len(reference_distribution.shape ) != 2:
             raise ValueError('''Expected `reference_distribution` to be a 2D vector''' )
       if reference_distribution.shape[0] < 2:
             raise ValueError(
                 '''Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension''' )

       # Get mahalanobis distance for each prediction
       snake_case_	=				X - np.mean(_UpperCAmelCase )
       snake_case_	=				np.cov(reference_distribution.T )
       try:
             snake_case_	=				np.linalg.inv(_UpperCAmelCase )
       except np.linalg.LinAlgError:
             snake_case_	=				np.linalg.pinv(_UpperCAmelCase )
       snake_case_	=				np.dot(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				np.dot(_UpperCAmelCase		,				X_minus_mu.T ).diagonal()

       return {"mahalanobis": mahal_dist}