





"""simple docstring"""





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__   =  get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
       from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

SCREAMING_SNAKE_CASE__   =  256_047
SCREAMING_SNAKE_CASE__   =  256_145

@require_sentencepiece
@require_tokenizers
class 						lowercase   (   _UpperCAmelCase	,    unittest.TestCase   ):
      _SCREAMING_SNAKE_CASE							      = NllbTokenizer
      _SCREAMING_SNAKE_CASE							      = NllbTokenizerFast
      _SCREAMING_SNAKE_CASE							      = True
      _SCREAMING_SNAKE_CASE							      = True
      _SCREAMING_SNAKE_CASE							      = {}





      def        _snake_case     (			self					)						->	Dict:
            super().setUp()

            # We have a SentencePiece fixture for testing
            lowerCAmelCase    		=			NllbTokenizer(lowercase ,		keep_accents=lowercase					)
            tokenizer.save_pretrained(self.tmpdirname					)





      def        _snake_case     (			self					)						->	Optional[int]:
            lowerCAmelCase    		=			NllbTokenizer(lowercase ,		keep_accents=lowercase					)

            lowerCAmelCase    		=			tokenizer.tokenize("""This is a test"""					)
            self.assertListEqual(lowercase ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(lowercase					) ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,		)

            lowerCAmelCase    		=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
            self.assertListEqual(
                lowercase ,		[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ] ,		)
            lowerCAmelCase    		=			tokenizer.convert_tokens_to_ids(lowercase					)
            self.assertListEqual(
                lowercase ,		[
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                ] ,		)

            lowerCAmelCase    		=			tokenizer.convert_ids_to_tokens(lowercase					)
            self.assertListEqual(
                lowercase ,		[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ] ,		)





      def        _snake_case     (			self					)						->	List[Any]:
            lowerCAmelCase    		=			(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
                        lowerCAmelCase    		=			self.rust_tokenizer_class.from_pretrained(lowercase ,		**lowercase					)
                        lowerCAmelCase    		=			self.tokenizer_class.from_pretrained(lowercase ,		**lowercase					)

                        lowerCAmelCase    		=			tempfile.mkdtemp()

                        lowerCAmelCase    		=			tokenizer_r.save_pretrained(lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.save_pretrained(lowercase					)

                        # Checks it save with the same files + the tokenizer.json file for the fast one
                        self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files					)					)
                        lowerCAmelCase    		=			tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f					)
                        self.assertSequenceEqual(lowercase ,		lowercase					)

                        # Checks everything loads correctly in the same way
                        lowerCAmelCase    		=			tokenizer_r.from_pretrained(lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.from_pretrained(lowercase					)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(lowercase ,		lowercase					)					)

                        shutil.rmtree(lowercase					)

                        # Save tokenizer rust, legacy_format=True
                        lowerCAmelCase    		=			tempfile.mkdtemp()

                        lowerCAmelCase    		=			tokenizer_r.save_pretrained(lowercase ,		legacy_format=lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.save_pretrained(lowercase					)

                        # Checks it save with the same files
                        self.assertSequenceEqual(lowercase ,		lowercase					)

                        # Checks everything loads correctly in the same way
                        lowerCAmelCase    		=			tokenizer_r.from_pretrained(lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.from_pretrained(lowercase					)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(lowercase ,		lowercase					)					)

                        shutil.rmtree(lowercase					)

                        # Save tokenizer rust, legacy_format=False
                        lowerCAmelCase    		=			tempfile.mkdtemp()

                        lowerCAmelCase    		=			tokenizer_r.save_pretrained(lowercase ,		legacy_format=lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.save_pretrained(lowercase					)

                        # Checks it saved the tokenizer.json file
                        self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files					)					)

                        # Checks everything loads correctly in the same way
                        lowerCAmelCase    		=			tokenizer_r.from_pretrained(lowercase					)
                        lowerCAmelCase    		=			tokenizer_p.from_pretrained(lowercase					)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(lowercase ,		lowercase					)					)

                        shutil.rmtree(lowercase					)





      @require_torch
      def        _snake_case     (			self					)						->	int:
            if not self.test_seqaseq:
                  return

            lowerCAmelCase    		=			self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'					):
                        # Longer text that will definitely require truncation.
                        lowerCAmelCase    		=			[
                            """ UN Chief Says There Is No Military Solution in Syria""",
                            """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                            """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                            """ will only worsen the violence and misery for millions of people.""",
                        ]
                        lowerCAmelCase    		=			[
                            """Şeful ONU declară că nu există o soluţie militară în Siria""",
                            """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                            """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                            """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                        ]
                        try:
                              lowerCAmelCase    		=			tokenizer.prepare_seqaseq_batch(
                                  src_texts=lowercase ,		tgt_texts=lowercase ,		max_length=3 ,		max_target_length=10 ,		return_tensors="""pt""" ,		src_lang="""eng_Latn""" ,		tgt_lang="""ron_Latn""" ,		)
                        except NotImplementedError:
                              return
                        self.assertEqual(batch.input_ids.shape[1] ,		3					)
                        self.assertEqual(batch.labels.shape[1] ,		10					)
                        # max_target_length will default to max_length if not specified
                        lowerCAmelCase    		=			tokenizer.prepare_seqaseq_batch(
                            lowercase ,		tgt_texts=lowercase ,		max_length=3 ,		return_tensors="""pt"""					)
                        self.assertEqual(batch.input_ids.shape[1] ,		3					)
                        self.assertEqual(batch.labels.shape[1] ,		3					)

                        lowerCAmelCase    		=			tokenizer.prepare_seqaseq_batch(
                            src_texts=lowercase ,		max_length=3 ,		max_target_length=10 ,		return_tensors="""pt"""					)
                        self.assertEqual(batch_encoder_only.input_ids.shape[1] ,		3					)
                        self.assertEqual(batch_encoder_only.attention_mask.shape[1] ,		3					)
                        self.assertNotIn("""decoder_input_ids""" ,		lowercase					)





      @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""					)
      def        _snake_case     (			self					)						->	Optional[int]:
            pass





      def        _snake_case     (			self					)						->	Dict:
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
                        lowerCAmelCase    		=			[AddedToken("""<special>""" ,		lstrip=lowercase					)]

                        lowerCAmelCase    		=			self.rust_tokenizer_class.from_pretrained(
                            lowercase ,		additional_special_tokens=lowercase ,		**lowercase					)
                        lowerCAmelCase    		=			tokenizer_r.encode("""Hey this is a <special> token"""					)

                        lowerCAmelCase    		=			tokenizer_r.encode("""<special>""" ,		add_special_tokens=lowercase					)[0]

                        self.assertTrue(special_token_id in r_output					)

                        if self.test_slow_tokenizer:
                              lowerCAmelCase    		=			self.rust_tokenizer_class.from_pretrained(
                                  lowercase ,		additional_special_tokens=lowercase ,		**lowercase ,		)
                              lowerCAmelCase    		=			self.tokenizer_class.from_pretrained(
                                  lowercase ,		additional_special_tokens=lowercase ,		**lowercase					)

                              lowerCAmelCase    		=			tokenizer_p.encode("""Hey this is a <special> token"""					)

                              lowerCAmelCase    		=			tokenizer_cr.encode("""Hey this is a <special> token"""					)

                              self.assertEqual(lowercase ,		lowercase					)
                              self.assertEqual(lowercase ,		lowercase					)
                              self.assertTrue(special_token_id in p_output					)
                              self.assertTrue(special_token_id in cr_output					)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						lowercase   (   unittest.TestCase   ):
      _SCREAMING_SNAKE_CASE							      = 'facebook/nllb-200-distilled-600M'
      _SCREAMING_SNAKE_CASE							      = [
          ' UN Chief Says There Is No Military Solution in Syria',
          ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
      ]
      _SCREAMING_SNAKE_CASE							      = [
          'Şeful ONU declară că nu există o soluţie militară în Siria',
          'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
          ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
          ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
      ]
      _SCREAMING_SNAKE_CASE							      = [
          256_047,
          16_297,
          134_408,
          8_165,
          248_066,
          14_734,
          950,
          1_135,
          105_721,
          3_573,
          83,
          27_352,
          108,
          49_486,
          2,
      ]





      @classmethod
      def        _snake_case     (			cls					)						->	Optional[Any]:
            lowerCAmelCase    		=			NllbTokenizer.from_pretrained(
                cls.checkpoint_name ,		src_lang="""eng_Latn""" ,		tgt_lang="""ron_Latn"""					)
            lowerCAmelCase    		=			1
            return cls





      def        _snake_case     (			self					)						->	Optional[Any]:
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""] ,		256_001					)
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""] ,		256_002					)
            self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""] ,		256_057					)





      def        _snake_case     (			self					)						->	int:
            lowerCAmelCase    		=			self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
            self.assertListEqual(self.expected_src_tokens ,		lowercase					)





      def        _snake_case     (			self					)						->	Tuple:
            self.assertIn(lowercase ,		self.tokenizer.all_special_ids					)
            # fmt: off
            lowerCAmelCase    		=			[RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
            # fmt: on

            lowerCAmelCase    		=			self.tokenizer.decode(lowercase ,		skip_special_tokens=lowercase					)
            lowerCAmelCase    		=			self.tokenizer.decode(generated_ids[1:] ,		skip_special_tokens=lowercase					)
            self.assertEqual(lowercase ,		lowercase					)
            self.assertNotIn(self.tokenizer.eos_token ,		lowercase					)





      def        _snake_case     (			self					)						->	Any:
            lowerCAmelCase    		=			["""this is gunna be a long sentence """ * 20]
            assert isinstance(src_text[0] ,		lowercase					)
            lowerCAmelCase    		=			10
            lowerCAmelCase    		=			self.tokenizer(lowercase ,		max_length=lowercase ,		truncation=lowercase					).input_ids[0]
            self.assertEqual(ids[-1] ,		2					)
            self.assertEqual(ids[0] ,		lowercase					)
            self.assertEqual(len(lowercase					) ,		lowercase					)





      def        _snake_case     (			self					)						->	Dict:
            self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]					) ,		[256_203, 3]					)





      def        _snake_case     (			self					)						->	Dict:
            lowerCAmelCase    		=			tempfile.mkdtemp()
            lowerCAmelCase    		=			self.tokenizer.fairseq_tokens_to_ids
            self.tokenizer.save_pretrained(lowercase					)
            lowerCAmelCase    		=			NllbTokenizer.from_pretrained(lowercase					)
            self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,		lowercase					)





      @require_torch
      def        _snake_case     (			self					)						->	List[Any]:
            lowerCAmelCase    		=			self.tokenizer(
                self.src_text ,		text_target=self.tgt_text ,		padding=lowercase ,		truncation=lowercase ,		max_length=len(self.expected_src_tokens					) ,		return_tensors="""pt""" ,		)
            lowerCAmelCase    		=			shift_tokens_right(
                batch["""labels"""] ,		self.tokenizer.pad_token_id ,		self.tokenizer.lang_code_to_id["""ron_Latn"""]					)

            self.assertIsInstance(lowercase ,		lowercase					)

            self.assertEqual((2, 15) ,		batch.input_ids.shape					)
            self.assertEqual((2, 15) ,		batch.attention_mask.shape					)
            lowerCAmelCase    		=			batch.input_ids.tolist()[0]
            self.assertListEqual(self.expected_src_tokens ,		lowercase					)
            self.assertEqual(lowercase ,		batch.decoder_input_ids[0, 0]					)  # EOS
            # Test that special tokens are reset
            self.assertEqual(self.tokenizer.prefix_tokens ,		[EN_CODE]					)
            self.assertEqual(self.tokenizer.suffix_tokens ,		[self.tokenizer.eos_token_id]					)





      def        _snake_case     (			self					)						->	Optional[Any]:
            lowerCAmelCase    		=			self.tokenizer(self.src_text ,		padding=lowercase ,		truncation=lowercase ,		max_length=3 ,		return_tensors="""pt"""					)
            lowerCAmelCase    		=			self.tokenizer(
                text_target=self.tgt_text ,		padding=lowercase ,		truncation=lowercase ,		max_length=10 ,		return_tensors="""pt"""					)
            lowerCAmelCase    		=			targets["""input_ids"""]
            lowerCAmelCase    		=			shift_tokens_right(
                lowercase ,		self.tokenizer.pad_token_id ,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang] ,		)

            self.assertEqual(batch.input_ids.shape[1] ,		3					)
            self.assertEqual(batch.decoder_input_ids.shape[1] ,		10					)





      @require_torch
      def        _snake_case     (			self					)						->	Any:
            lowerCAmelCase    		=			self.tokenizer._build_translation_inputs(
                """A test""" ,		return_tensors="""pt""" ,		src_lang="""eng_Latn""" ,		tgt_lang="""fra_Latn"""					)

            self.assertEqual(
                nested_simplify(lowercase					) ,		{
                    # A, test, EOS, en_XX
                    """input_ids""": [[256_047, 70, 7_356, 2]],
                    """attention_mask""": [[1, 1, 1, 1]],
                    # ar_AR
                    """forced_bos_token_id""": 256_057,
                } ,		)





      @require_torch
      def        _snake_case     (			self					)						->	List[Any]:
            lowerCAmelCase    		=			True
            lowerCAmelCase    		=			self.tokenizer(
                """UN Chief says there is no military solution in Syria""" ,		src_lang="""eng_Latn""" ,		tgt_lang="""fra_Latn"""					)
            self.assertEqual(
                inputs.input_ids ,		[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]					)

            lowerCAmelCase    		=			False
            lowerCAmelCase    		=			self.tokenizer(
                """UN Chief says there is no military solution in Syria""" ,		src_lang="""eng_Latn""" ,		tgt_lang="""fra_Latn"""					)
            self.assertEqual(
                inputs.input_ids ,		[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]					)













"""simple docstring"""





import logging
import os

from .state import PartialState

class 						lowercase   (   logging.LoggerAdapter   ):





      @staticmethod
      def        _snake_case     (			lowercase					)						->	Optional[Any]:
            lowerCAmelCase    		=			PartialState()
            return not main_process_only or (main_process_only and state.is_main_process)





      def        _snake_case     (			self ,		lowercase ,		lowercase ,		*lowercase ,		**lowercase					)						->	Dict:
            if PartialState._shared_state == {}:
                  raise RuntimeError(
                      """You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility."""					)
            lowerCAmelCase    		=			kwargs.pop("""main_process_only""" ,		lowercase					)
            lowerCAmelCase    		=			kwargs.pop("""in_order""" ,		lowercase					)

            if self.isEnabledFor(lowercase					):
                  if self._should_log(lowercase					):
                        lowerCAmelCase		,						lowerCAmelCase    		=			self.process(lowercase ,		lowercase					)
                        self.logger.log(lowercase ,		lowercase ,		*lowercase ,		**lowercase					)

                  elif in_order:
                        lowerCAmelCase    		=			PartialState()
                        for i in range(state.num_processes					):
                              if i == state.process_index:
                                    lowerCAmelCase		,						lowerCAmelCase    		=			self.process(lowercase ,		lowercase					)
                                    self.logger.log(lowercase ,		lowercase ,		*lowercase ,		**lowercase					)
                              state.wait_for_everyone()







def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		str				,		SCREAMING_SNAKE_CASE	:		str = None     ):







      '''simple docstring'''




      if log_level is None:
            lowerCAmelCase    		=			os.environ.get("""ACCELERATE_LOG_LEVEL"""				,		SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			logging.getLogger(SCREAMING_SNAKE_CASE     )
      if log_level is not None:
            logger.setLevel(log_level.upper()     )
            logger.root.setLevel(log_level.upper()     )
      return MultiProcessAdapter(SCREAMING_SNAKE_CASE				,		{}     )






