


'''simple docstring'''
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_ :     Dict								=      get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase_ :     int								=      2_5_6_0_4_7
UpperCAmelCase_ :     Optional[Any]								=      2_5_6_1_4_5

@require_sentencepiece
@require_tokenizers
class 						_lowerCamelCase					(    snake_case_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		List[Any]    							=						NllbTokenizer
 __lowercase				:		Any    							=						NllbTokenizerFast
 __lowercase				:		Dict    							=						True
 __lowercase				:		int    							=						True
 __lowercase				:		Union[str, Any]    							=						{}


 def  snake_case__   (      self						):

  """simple docstring"""







  super().setUp()

  # We have a SentencePiece fixture for testing
  __A  :		Tuple       =			NllbTokenizer(__lowercase      ,      keep_accents=__lowercase						)
  tokenizer.save_pretrained(self.tmpdirname						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Tuple       =			NllbTokenizer(__lowercase      ,      keep_accents=__lowercase						)

  __A  :		Optional[Any]       =			tokenizer.tokenize('This is a test'						)
  self.assertListEqual(__lowercase      ,      ['▁This', '▁is', '▁a', '▁t', 'est']						)

  self.assertListEqual(
      tokenizer.convert_tokens_to_ids(__lowercase						)      ,      [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,      )

  __A  :		str       =			tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
  self.assertListEqual(
      __lowercase      ,      [
          SPIECE_UNDERLINE + 'I',
          SPIECE_UNDERLINE + 'was',
          SPIECE_UNDERLINE + 'b',
          'or',
          'n',
          SPIECE_UNDERLINE + 'in',
          SPIECE_UNDERLINE + '',
          '9',
          '2',
          '0',
          '0',
          '0',
          ',',
          SPIECE_UNDERLINE + 'and',
          SPIECE_UNDERLINE + 'this',
          SPIECE_UNDERLINE + 'is',
          SPIECE_UNDERLINE + 'f',
          'al',
          's',
          'é',
          '.',
      ]      ,      )
  __A  :		Any       =			tokenizer.convert_tokens_to_ids(__lowercase						)
  self.assertListEqual(
      __lowercase      ,      [
          value + tokenizer.fairseq_offset
          for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
      ]      ,      )

  __A  :		Optional[int]       =			tokenizer.convert_ids_to_tokens(__lowercase						)
  self.assertListEqual(
      __lowercase      ,      [
          SPIECE_UNDERLINE + 'I',
          SPIECE_UNDERLINE + 'was',
          SPIECE_UNDERLINE + 'b',
          'or',
          'n',
          SPIECE_UNDERLINE + 'in',
          SPIECE_UNDERLINE + '',
          '<unk>',
          '2',
          '0',
          '0',
          '0',
          ',',
          SPIECE_UNDERLINE + 'and',
          SPIECE_UNDERLINE + 'this',
          SPIECE_UNDERLINE + 'is',
          SPIECE_UNDERLINE + 'f',
          'al',
          's',
          '<unk>',
          '.',
      ]      ,      )


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Dict       =			(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
    __A  :		List[Any]       =			self.rust_tokenizer_class.from_pretrained(__lowercase      ,      **__lowercase						)
    __A  :		Dict       =			self.tokenizer_class.from_pretrained(__lowercase      ,      **__lowercase						)

    __A  :		Any       =			tempfile.mkdtemp()

    __A  :		Union[str, Any]       =			tokenizer_r.save_pretrained(__lowercase						)
    __A  :		Optional[Any]       =			tokenizer_p.save_pretrained(__lowercase						)

    # Checks it save with the same files + the tokenizer.json file for the fast one
    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files						)						)
    __A  :		Optional[Any]       =			tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f						)
    self.assertSequenceEqual(__lowercase      ,      __lowercase						)

    # Checks everything loads correctly in the same way
    __A  :		Optional[int]       =			tokenizer_r.from_pretrained(__lowercase						)
    __A  :		Tuple       =			tokenizer_p.from_pretrained(__lowercase						)

    # Check special tokens are set accordingly on Rust and Python
    for key in tokenizer_pp.special_tokens_map:
     self.assertTrue(hasattr(__lowercase      ,      __lowercase						)						)

    shutil.rmtree(__lowercase						)

    # Save tokenizer rust, legacy_format=True
    __A  :		Optional[int]       =			tempfile.mkdtemp()

    __A  :		Union[str, Any]       =			tokenizer_r.save_pretrained(__lowercase      ,      legacy_format=__lowercase						)
    __A  :		Optional[Any]       =			tokenizer_p.save_pretrained(__lowercase						)

    # Checks it save with the same files
    self.assertSequenceEqual(__lowercase      ,      __lowercase						)

    # Checks everything loads correctly in the same way
    __A  :		str       =			tokenizer_r.from_pretrained(__lowercase						)
    __A  :		Dict       =			tokenizer_p.from_pretrained(__lowercase						)

    # Check special tokens are set accordingly on Rust and Python
    for key in tokenizer_pp.special_tokens_map:
     self.assertTrue(hasattr(__lowercase      ,      __lowercase						)						)

    shutil.rmtree(__lowercase						)

    # Save tokenizer rust, legacy_format=False
    __A  :		List[Any]       =			tempfile.mkdtemp()

    __A  :		Tuple       =			tokenizer_r.save_pretrained(__lowercase      ,      legacy_format=__lowercase						)
    __A  :		Any       =			tokenizer_p.save_pretrained(__lowercase						)

    # Checks it saved the tokenizer.json file
    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files						)						)

    # Checks everything loads correctly in the same way
    __A  :		Any       =			tokenizer_r.from_pretrained(__lowercase						)
    __A  :		Optional[Any]       =			tokenizer_p.from_pretrained(__lowercase						)

    # Check special tokens are set accordingly on Rust and Python
    for key in tokenizer_pp.special_tokens_map:
     self.assertTrue(hasattr(__lowercase      ,      __lowercase						)						)

    shutil.rmtree(__lowercase						)


 @require_torch
 def  snake_case__   (      self						):

  """simple docstring"""







  if not self.test_seqaseq:
   return

  __A  :		Optional[int]       =			self.get_tokenizers()
  for tokenizer in tokenizers:
   with self.subTest(F"""{tokenizer.__class__.__name__}"""						):
    # Longer text that will definitely require truncation.
    __A  :		str       =			[
        ' UN Chief Says There Is No Military Solution in Syria',
        ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
        ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
        ' will only worsen the violence and misery for millions of people.',
    ]
    __A  :		Any       =			[
        'Şeful ONU declară că nu există o soluţie militară în Siria',
        'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
        ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
        ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
    ]
    try:
     __A  :		Optional[Any]       =			tokenizer.prepare_seqaseq_batch(
         src_texts=__lowercase      ,      tgt_texts=__lowercase      ,      max_length=3      ,      max_target_length=10      ,      return_tensors='pt'      ,      src_lang='eng_Latn'      ,      tgt_lang='ron_Latn'      ,      )
    except NotImplementedError:
     return
    self.assertEqual(batch.input_ids.shape[1]      ,      3						)
    self.assertEqual(batch.labels.shape[1]      ,      10						)
    # max_target_length will default to max_length if not specified
    __A  :		List[str]       =			tokenizer.prepare_seqaseq_batch(
        __lowercase      ,      tgt_texts=__lowercase      ,      max_length=3      ,      return_tensors='pt'						)
    self.assertEqual(batch.input_ids.shape[1]      ,      3						)
    self.assertEqual(batch.labels.shape[1]      ,      3						)

    __A  :		List[str]       =			tokenizer.prepare_seqaseq_batch(
        src_texts=__lowercase      ,      max_length=3      ,      max_target_length=10      ,      return_tensors='pt'						)
    self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,      3						)
    self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,      3						)
    self.assertNotIn('decoder_input_ids'      ,      __lowercase						)


 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'						)
 def  snake_case__   (      self						):

  """simple docstring"""







  pass







 def  snake_case__   (      self						):

  """simple docstring"""







  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
    __A  :		Dict       =			[AddedToken('<special>'      ,      lstrip=__lowercase						)]

    __A  :		Dict       =			self.rust_tokenizer_class.from_pretrained(
        __lowercase      ,      additional_special_tokens=__lowercase      ,      **__lowercase						)
    __A  :		Tuple       =			tokenizer_r.encode('Hey this is a <special> token'						)

    __A  :		Any       =			tokenizer_r.encode('<special>'      ,      add_special_tokens=__lowercase						)[0]

    self.assertTrue(special_token_id in r_output						)

    if self.test_slow_tokenizer:
     __A  :		Optional[int]       =			self.rust_tokenizer_class.from_pretrained(
         __lowercase      ,      additional_special_tokens=__lowercase      ,      **__lowercase      ,      )
     __A  :		int       =			self.tokenizer_class.from_pretrained(
         __lowercase      ,      additional_special_tokens=__lowercase      ,      **__lowercase						)

     __A  :		str       =			tokenizer_p.encode('Hey this is a <special> token'						)

     __A  :		Tuple       =			tokenizer_cr.encode('Hey this is a <special> token'						)

     self.assertEqual(__lowercase      ,      __lowercase						)
     self.assertEqual(__lowercase      ,      __lowercase						)
     self.assertTrue(special_token_id in p_output						)
     self.assertTrue(special_token_id in cr_output						)


@require_torch
@require_sentencepiece
@require_tokenizers
class 						_lowerCamelCase					(    unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		Optional[Any]    							=						'''facebook/nllb-200-distilled-600M'''
 __lowercase				:		Tuple    							=						[
     ''' UN Chief Says There Is No Military Solution in Syria''',
     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
 ]
 __lowercase				:		List[Any]    							=						[
     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
     ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
     ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
 ]
 __lowercase				:		Any    							=						[
     2_5_6_0_4_7,
     1_6_2_9_7,
     1_3_4_4_0_8,
     8_1_6_5,
     2_4_8_0_6_6,
     1_4_7_3_4,
     9_5_0,
     1_1_3_5,
     1_0_5_7_2_1,
     3_5_7_3,
     8_3,
     2_7_3_5_2,
     1_0_8,
     4_9_4_8_6,
     2,
 ]


 @classmethod
 def  snake_case__   (      cls						):

  """simple docstring"""







  __A  :		NllbTokenizer       =			NllbTokenizer.from_pretrained(
      cls.checkpoint_name      ,      src_lang='eng_Latn'      ,      tgt_lang='ron_Latn'						)
  __A  :		Optional[Any]       =			1
  return cls


 def  snake_case__   (      self						):

  """simple docstring"""







  self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']      ,      256_001						)
  self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']      ,      256_002						)
  self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']      ,      256_057						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Optional[Any]       =			self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
  self.assertListEqual(self.expected_src_tokens      ,      __lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  self.assertIn(__lowercase      ,      self.tokenizer.all_special_ids						)
  # fmt: off
  __A  :		Optional[Any]       =			[RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
  # fmt: on

  __A  :		Optional[Any]       =			self.tokenizer.decode(__lowercase      ,      skip_special_tokens=__lowercase						)
  __A  :		Any       =			self.tokenizer.decode(generated_ids[1:]      ,      skip_special_tokens=__lowercase						)
  self.assertEqual(__lowercase      ,      __lowercase						)
  self.assertNotIn(self.tokenizer.eos_token      ,      __lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		List[str]       =			['this is gunna be a long sentence ' * 20]
  assert isinstance(src_text[0]      ,      __lowercase						)
  __A  :		Any       =			10
  __A  :		str       =			self.tokenizer(__lowercase      ,      max_length=__lowercase      ,      truncation=__lowercase						).input_ids[0]
  self.assertEqual(ids[-1]      ,      2						)
  self.assertEqual(ids[0]      ,      __lowercase						)
  self.assertEqual(len(__lowercase						)      ,      __lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']						)      ,      [256_203, 3]						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Dict       =			tempfile.mkdtemp()
  __A  :		Union[str, Any]       =			self.tokenizer.fairseq_tokens_to_ids
  self.tokenizer.save_pretrained(__lowercase						)
  __A  :		Optional[Any]       =			NllbTokenizer.from_pretrained(__lowercase						)
  self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,      __lowercase						)


 @require_torch
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Dict       =			self.tokenizer(
      self.src_text      ,      text_target=self.tgt_text      ,      padding=__lowercase      ,      truncation=__lowercase      ,      max_length=len(self.expected_src_tokens						)      ,      return_tensors='pt'      ,      )
  __A  :		Optional[Any]       =			shift_tokens_right(
      batch['labels']      ,      self.tokenizer.pad_token_id      ,      self.tokenizer.lang_code_to_id['ron_Latn']						)

  self.assertIsInstance(__lowercase      ,      __lowercase						)

  self.assertEqual((2, 15)      ,      batch.input_ids.shape						)
  self.assertEqual((2, 15)      ,      batch.attention_mask.shape						)
  __A  :		List[Any]       =			batch.input_ids.tolist()[0]
  self.assertListEqual(self.expected_src_tokens      ,      __lowercase						)
  self.assertEqual(__lowercase      ,      batch.decoder_input_ids[0, 0]						)  # EOS
  # Test that special tokens are reset
  self.assertEqual(self.tokenizer.prefix_tokens      ,      [EN_CODE]						)
  self.assertEqual(self.tokenizer.suffix_tokens      ,      [self.tokenizer.eos_token_id]						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Tuple       =			self.tokenizer(self.src_text      ,      padding=__lowercase      ,      truncation=__lowercase      ,      max_length=3      ,      return_tensors='pt'						)
  __A  :		Any       =			self.tokenizer(
      text_target=self.tgt_text      ,      padding=__lowercase      ,      truncation=__lowercase      ,      max_length=10      ,      return_tensors='pt'						)
  __A  :		List[str]       =			targets['input_ids']
  __A  :		List[Any]       =			shift_tokens_right(
      __lowercase      ,      self.tokenizer.pad_token_id      ,      decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,      )

  self.assertEqual(batch.input_ids.shape[1]      ,      3						)
  self.assertEqual(batch.decoder_input_ids.shape[1]      ,      10						)


 @require_torch
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Dict       =			self.tokenizer._build_translation_inputs(
      'A test'      ,      return_tensors='pt'      ,      src_lang='eng_Latn'      ,      tgt_lang='fra_Latn'						)

  self.assertEqual(
      nested_simplify(__lowercase						)      ,      {
          # A, test, EOS, en_XX
          'input_ids': [[256_047, 70, 7_356, 2]],
          'attention_mask': [[1, 1, 1, 1]],
          # ar_AR
          'forced_bos_token_id': 256_057,
      }      ,      )







 @require_torch
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		List[Any]       =			True
  __A  :		Any       =			self.tokenizer(
      'UN Chief says there is no military solution in Syria'      ,      src_lang='eng_Latn'      ,      tgt_lang='fra_Latn'						)
  self.assertEqual(
      inputs.input_ids      ,      [16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]						)

  __A  :		Any       =			False
  __A  :		List[Any]       =			self.tokenizer(
      'UN Chief says there is no military solution in Syria'      ,      src_lang='eng_Latn'      ,      tgt_lang='fra_Latn'						)
  self.assertEqual(
      inputs.input_ids      ,      [256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]						)










'''simple docstring'''
import math

class 						_lowerCamelCase					:




 '''simple docstring'''


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  __A  :		List[str]       =			0.0
  __A  :		Optional[int]       =			0.0
  for i in range(len(__lowercase						)						):
   da += math.pow((sample[i] - weights[0][i])      ,      2						)
   da += math.pow((sample[i] - weights[1][i])      ,      2						)
   return 0 if da > da else 1
  return 0







 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  for i in range(len(__lowercase						)						):
   weights[j][i] += alpha * (sample[i] - weights[j][i])
  return weights


def 			_lowercase       (				):
 # Training Examples ( m, n )
 __A  :		Union[str, Any]       =			[[1, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 1]]

 # weight initialization ( n, C )
 __A  :		Tuple       =			[[0.2, 0.6, 0.5, 0.9], [0.8, 0.4, 0.7, 0.3]]

 # training
 __A  :		Optional[int]       =			SelfOrganizingMap()
 __A  :		int       =			3
 __A  :		Optional[Any]       =			0.5

 for _ in range(UpperCamelCase__			):
  for j in range(len(UpperCamelCase__			)			):
   # training sample
   __A  :		Dict       =			training_samples[j]

   # Compute the winning vector
   __A  :		List[Any]       =			self_organizing_map.get_winner(UpperCamelCase__,				UpperCamelCase__			)

   # Update the winning vector
   __A  :		Union[str, Any]       =			self_organizing_map.update(UpperCamelCase__,				UpperCamelCase__,				UpperCamelCase__,				UpperCamelCase__			)

    # classify test sample
 __A  :		int       =			[0, 0, 0, 1]
 __A  :		List[Any]       =			self_organizing_map.get_winner(UpperCamelCase__,				UpperCamelCase__			)

 # results
 print(f"""Clusters that the test sample belongs to : {winner}"""			)
 print(f"""Weights that have been trained : {weights}"""			)


# running the main() function
if __name__ == "__main__":
 main()






