





"""simple docstring"""





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__				:List[Any]				   =       get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase__				:List[str]				   =       2_5_0_0_0_4
lowercase__				:List[str]				   =       2_5_0_0_2_0




@require_sentencepiece
@require_tokenizers
class 							snake_case							(			__lowerCamelCase      ,  unittest.TestCase ):

      '''simple docstring'''



      _A     :       Any         =       MBartTokenizer
      _A     :       Union[str, Any]         =       MBartTokenizerFast
      _A     :       List[str]         =       True
      _A     :       Tuple         =       True

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        super().setUp()

        # We have a SentencePiece fixture for testing
        __UpperCAmelCase    :						Optional[Any]       =						MBartTokenizer(UpperCamelCase_						,				keep_accents=UpperCamelCase_ )
        tokenizer.save_pretrained(self.tmpdirname )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						MBartTokenizer(UpperCamelCase_						,				keep_accents=UpperCamelCase_ )

        __UpperCAmelCase    :						int       =						tokenizer.tokenize('''This is a test''' )
        self.assertListEqual(UpperCamelCase_						,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(UpperCamelCase_ )						,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,				)

        __UpperCAmelCase    :						List[Any]       =						tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
        self.assertListEqual(
            UpperCamelCase_						,				[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]						,				)
        __UpperCAmelCase    :						str       =						tokenizer.convert_tokens_to_ids(UpperCamelCase_ )
        self.assertListEqual(
            UpperCamelCase_						,				[
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
            ]						,				)

        __UpperCAmelCase    :						List[Any]       =						tokenizer.convert_ids_to_tokens(UpperCamelCase_ )
        self.assertListEqual(
            UpperCamelCase_						,				[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]						,				)







      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        if not self.test_slow_tokenizer:
          # as we don't have a slow version, we can't compare the outputs between slow and fast versions
          return

        __UpperCAmelCase    :						Dict       =						(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
            __UpperCAmelCase    :						Optional[Any]       =						self.rust_tokenizer_class.from_pretrained(UpperCamelCase_						,				**UpperCamelCase_ )
            __UpperCAmelCase    :						List[str]       =						self.tokenizer_class.from_pretrained(UpperCamelCase_						,				**UpperCamelCase_ )

            __UpperCAmelCase    :						Union[str, Any]       =						tempfile.mkdtemp()

            __UpperCAmelCase    :						str       =						tokenizer_r.save_pretrained(UpperCamelCase_ )
            __UpperCAmelCase    :						int       =						tokenizer_p.save_pretrained(UpperCamelCase_ )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )
            __UpperCAmelCase    :						Any       =						tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f )
            self.assertSequenceEqual(UpperCamelCase_						,				UpperCamelCase_ )

            # Checks everything loads correctly in the same way
            __UpperCAmelCase    :						Any       =						tokenizer_r.from_pretrained(UpperCamelCase_ )
            __UpperCAmelCase    :						Union[str, Any]       =						tokenizer_p.from_pretrained(UpperCamelCase_ )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(UpperCamelCase_						,				UpperCamelCase_ ) )
              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

            shutil.rmtree(UpperCamelCase_ )

            # Save tokenizer rust, legacy_format=True
            __UpperCAmelCase    :						str       =						tempfile.mkdtemp()

            __UpperCAmelCase    :						Optional[Any]       =						tokenizer_r.save_pretrained(UpperCamelCase_						,				legacy_format=UpperCamelCase_ )
            __UpperCAmelCase    :						Any       =						tokenizer_p.save_pretrained(UpperCamelCase_ )

            # Checks it save with the same files
            self.assertSequenceEqual(UpperCamelCase_						,				UpperCamelCase_ )

            # Checks everything loads correctly in the same way
            __UpperCAmelCase    :						List[str]       =						tokenizer_r.from_pretrained(UpperCamelCase_ )
            __UpperCAmelCase    :						List[Any]       =						tokenizer_p.from_pretrained(UpperCamelCase_ )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(UpperCamelCase_						,				UpperCamelCase_ ) )

            shutil.rmtree(UpperCamelCase_ )

            # Save tokenizer rust, legacy_format=False
            __UpperCAmelCase    :						Union[str, Any]       =						tempfile.mkdtemp()

            __UpperCAmelCase    :						Optional[int]       =						tokenizer_r.save_pretrained(UpperCamelCase_						,				legacy_format=UpperCamelCase_ )
            __UpperCAmelCase    :						Union[str, Any]       =						tokenizer_p.save_pretrained(UpperCamelCase_ )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )

            # Checks everything loads correctly in the same way
            __UpperCAmelCase    :						Optional[Any]       =						tokenizer_r.from_pretrained(UpperCamelCase_ )
            __UpperCAmelCase    :						Union[str, Any]       =						tokenizer_p.from_pretrained(UpperCamelCase_ )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(UpperCamelCase_						,				UpperCamelCase_ ) )

            shutil.rmtree(UpperCamelCase_ )




@require_torch
@require_sentencepiece
@require_tokenizers
class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      _A     :       Union[str, Any]         =       'facebook/mbart-large-en-ro'
      _A     :       Optional[Any]         =       [
          ' UN Chief Says There Is No Military Solution in Syria',
          ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
      ]
      _A     :       Tuple         =       [
          'Şeful ONU declară că nu există o soluţie militară în Siria',
          'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
          ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
          ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
      ]
      _A     :       int         =       [82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2, EN_CODE]

      @classmethod
      def       A_					(							cls       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						MBartTokenizer.from_pretrained(
            cls.checkpoint_name						,				src_lang='''en_XX'''						,				tgt_lang='''ro_RO''' )
        __UpperCAmelCase    :						List[str]       =						1
        return cls

      def       A_					(							self       :      str ):


        '''simple docstring'''




        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']						,				250_001 )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']						,				250_004 )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']						,				250_020 )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
        self.assertListEqual(self.expected_src_tokens						,				UpperCamelCase_ )

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        self.assertIn(UpperCamelCase_						,				self.tokenizer.all_special_ids )
        __UpperCAmelCase    :						Any       =						[RO_CODE, 884, 9_019, 96, 9, 916, 86_792, 36, 18_743, 15_596, 5, 2]
        __UpperCAmelCase    :						List[str]       =						self.tokenizer.decode(UpperCamelCase_						,				skip_special_tokens=UpperCamelCase_ )
        __UpperCAmelCase    :						List[str]       =						self.tokenizer.decode(generated_ids[1:]						,				skip_special_tokens=UpperCamelCase_ )
        self.assertEqual(UpperCamelCase_						,				UpperCamelCase_ )
        self.assertNotIn(self.tokenizer.eos_token						,				UpperCamelCase_ )

      def       A_					(							self       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						['''this is gunna be a long sentence ''' * 20]
        assert isinstance(src_text[0]						,				UpperCamelCase_ )
        __UpperCAmelCase    :						Tuple       =						10
        __UpperCAmelCase    :						str       =						self.tokenizer(UpperCamelCase_						,				max_length=UpperCamelCase_						,				truncation=UpperCamelCase_ ).input_ids[0]
        self.assertEqual(ids[-2]						,				2 )
        self.assertEqual(ids[-1]						,				UpperCamelCase_ )
        self.assertEqual(len(UpperCamelCase_ )						,				UpperCamelCase_ )

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR'''] )						,				[250_026, 250_001] )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						tempfile.mkdtemp()
        __UpperCAmelCase    :						Optional[int]       =						self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(UpperCamelCase_ )
        __UpperCAmelCase    :						Any       =						MBartTokenizer.from_pretrained(UpperCamelCase_ )
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,				UpperCamelCase_ )

      @require_torch
      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						self.tokenizer(self.src_text						,				text_target=self.tgt_text						,				padding=UpperCamelCase_						,				return_tensors='''pt''' )
        __UpperCAmelCase    :						Optional[Any]       =						shift_tokens_right(batch['''labels''']						,				self.tokenizer.pad_token_id )

        # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
        assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
        assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
        assert batch.decoder_input_ids[1][-1] == 2
        assert batch.labels[1][-2:].tolist() == [2, RO_CODE]

      @require_torch
      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						self.tokenizer(
            self.src_text						,				text_target=self.tgt_text						,				padding=UpperCamelCase_						,				truncation=UpperCamelCase_						,				max_length=len(self.expected_src_tokens )						,				return_tensors='''pt'''						,				)

        __UpperCAmelCase    :						List[Any]       =						shift_tokens_right(batch['''labels''']						,				self.tokenizer.pad_token_id )

        self.assertIsInstance(UpperCamelCase_						,				UpperCamelCase_ )

        self.assertEqual((2, 14)						,				batch.input_ids.shape )
        self.assertEqual((2, 14)						,				batch.attention_mask.shape )
        __UpperCAmelCase    :						Optional[Any]       =						batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens						,				UpperCamelCase_ )
        self.assertEqual(2						,				batch.decoder_input_ids[0, -1] )  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens						,				[] )
        self.assertEqual(self.tokenizer.suffix_tokens						,				[self.tokenizer.eos_token_id, EN_CODE] )

      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						self.tokenizer(self.src_text						,				padding=UpperCamelCase_						,				truncation=UpperCamelCase_						,				max_length=3						,				return_tensors='''pt''' )
        __UpperCAmelCase    :						Optional[int]       =						self.tokenizer(
            text_target=self.tgt_text						,				padding=UpperCamelCase_						,				truncation=UpperCamelCase_						,				max_length=10						,				return_tensors='''pt''' )
        __UpperCAmelCase    :						List[str]       =						targets['''input_ids''']
        __UpperCAmelCase    :						Optional[int]       =						shift_tokens_right(UpperCamelCase_						,				self.tokenizer.pad_token_id )

        self.assertEqual(batch.input_ids.shape[1]						,				3 )
        self.assertEqual(batch.decoder_input_ids.shape[1]						,				10 )







      @require_torch
      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						self.tokenizer._build_translation_inputs(
            '''A test'''						,				return_tensors='''pt'''						,				src_lang='''en_XX'''						,				tgt_lang='''ar_AR''' )

        self.assertEqual(
            nested_simplify(UpperCamelCase_ )						,				{
                # A, test, EOS, en_XX
                '''input_ids''': [[62, 3_034, 2, 250_004]],
                '''attention_mask''': [[1, 1, 1, 1]],
                # ar_AR
                '''forced_bos_token_id''': 250_001,
            }						,				)






"""simple docstring"""





import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowercase__				:int				   =       'src/transformers'
lowercase__				:List[str]				   =       'docs/source/en/tasks'


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->str:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''r'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
    __UpperCAmelCase    :						Union[str, Any]       =						f.readlines()
  # Find the start prompt.
  __UpperCAmelCase    :						Any       =						0
  while not lines[start_index].startswith(UpperCAmelCase_  ):
    start_index += 1
  start_index += 1

  __UpperCAmelCase    :						Optional[Any]       =						start_index
  while not lines[end_index].startswith(UpperCAmelCase_  ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]  ) <= 1:
    start_index += 1
  while len(lines[end_index]  ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowercase__				:Any				   =       direct_transformers_import(TRANSFORMERS_PATH)

lowercase__				:List[Any]				   =       {
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowercase__				:Union[str, Any]				   =       {
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[str, Any]:





  """simple docstring"""


  __UpperCAmelCase    :						List[str]       =						TASK_GUIDE_TO_MODELS[task_guide]
  __UpperCAmelCase    :						Dict       =						SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCAmelCase_  ,			set()  )
  __UpperCAmelCase    :						List[Any]       =						{
      code: name
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if (code in model_maping_names or code in special_model_types)
  }
  return ", ".join([f'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]  ) + "\n"


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_=False  )						->Tuple:





  """simple docstring"""


  __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						_find_text_in_file(
      filename=os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			start_prompt='''<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'''  ,			end_prompt='''<!--End of the generated tip-->'''  ,			)

  __UpperCAmelCase    :						List[str]       =						get_model_list_for_task(UpperCAmelCase_  )

  if current_list != new_list:
    if overwrite:
      with open(os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			'''w'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
        f.writelines(lines[:start_index] + [new_list] + lines[end_index:]  )
    else:
      raise ValueError(
          f'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
          ''' to fix this.'''  )


if __name__ == "__main__":
  lowercase__				:int				   =       argparse.ArgumentParser()
  parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
  lowercase__				:Optional[Any]				   =       parser.parse_args()

  for task_guide in TASK_GUIDE_TO_MODELS.keys():
    check_model_list_for_task(task_guide, args.fix_and_overwrite)