





"""simple docstring"""





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						tempfile.mkdtemp()
        __UpperCAmelCase    :						List[Any]       =						5

        # Realm tok
        __UpperCAmelCase    :						Union[str, Any]       =						[
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''test''',
            '''question''',
            '''this''',
            '''is''',
            '''the''',
            '''first''',
            '''second''',
            '''third''',
            '''fourth''',
            '''fifth''',
            '''record''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        __UpperCAmelCase    :						Optional[int]       =						os.path.join(self.tmpdirname						,				'''realm_tokenizer''' )
        os.makedirs(__lowercase						,				exist_ok=__lowercase )
        __UpperCAmelCase    :						List[Any]       =						os.path.join(__lowercase						,				VOCAB_FILES_NAMES['''vocab_file'''] )
        with open(self.vocab_file						,				'''w'''						,				encoding='''utf-8''' ) as vocab_writer:
          vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

        __UpperCAmelCase    :						Tuple       =						os.path.join(self.tmpdirname						,				'''realm_block_records''' )
        os.makedirs(__lowercase						,				exist_ok=__lowercase )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname						,				'''realm_tokenizer''' ) )

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        shutil.rmtree(self.tmpdirname )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						RealmConfig(num_block_records=self.num_block_records )
        return config

      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						Dataset.from_dict(
            {
                '''id''': ['''0''', '''1'''],
                '''question''': ['''foo''', '''bar'''],
                '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
            } )
        return dataset

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						np.array(
            [
                b'''This is the first record''',
                b'''This is the second record''',
                b'''This is the third record''',
                b'''This is the fourth record''',
                b'''This is the fifth record''',
                b'''This is a longer longer longer record''',
            ]						,				dtype=__lowercase						,				)
        return block_records

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						RealmRetriever(
            block_records=self.get_dummy_block_records()						,				tokenizer=self.get_tokenizer()						,				)
        return retriever

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.get_config()
        __UpperCAmelCase    :						int       =						self.get_dummy_retriever()
        __UpperCAmelCase    :						Optional[int]       =						retriever.tokenizer

        __UpperCAmelCase    :						Union[str, Any]       =						np.array([0, 3]						,				dtype='''long''' )
        __UpperCAmelCase    :						str       =						tokenizer(['''Test question'''] ).input_ids
        __UpperCAmelCase    :						Dict       =						tokenizer(
            ['''the fourth''']						,				add_special_tokens=__lowercase						,				return_token_type_ids=__lowercase						,				return_attention_mask=__lowercase						,				).input_ids
        __UpperCAmelCase    :						Optional[int]       =						config.reader_seq_len

        __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Tuple       =						retriever(
            __lowercase						,				__lowercase						,				answer_ids=__lowercase						,				max_length=__lowercase						,				return_tensors='''np''' )

        self.assertEqual(len(__lowercase )						,				2 )
        self.assertEqual(len(__lowercase )						,				2 )
        self.assertEqual(len(__lowercase )						,				2 )
        self.assertEqual(concat_inputs.input_ids.shape						,				(2, 10) )
        self.assertEqual(concat_inputs.attention_mask.shape						,				(2, 10) )
        self.assertEqual(concat_inputs.token_type_ids.shape						,				(2, 10) )
        self.assertEqual(concat_inputs.special_tokens_mask.shape						,				(2, 10) )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0] )						,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']						,				)
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1] )						,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']						,				)

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						self.get_config()
        __UpperCAmelCase    :						int       =						self.get_dummy_retriever()
        __UpperCAmelCase    :						int       =						retriever.tokenizer

        __UpperCAmelCase    :						List[str]       =						np.array([0, 3, 5]						,				dtype='''long''' )
        __UpperCAmelCase    :						int       =						tokenizer(['''Test question'''] ).input_ids
        __UpperCAmelCase    :						Tuple       =						tokenizer(
            ['''the fourth''', '''longer longer''']						,				add_special_tokens=__lowercase						,				return_token_type_ids=__lowercase						,				return_attention_mask=__lowercase						,				).input_ids
        __UpperCAmelCase    :						Tuple       =						config.reader_seq_len

        __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						retriever(
            __lowercase						,				__lowercase						,				answer_ids=__lowercase						,				max_length=__lowercase						,				return_tensors='''np''' )

        self.assertEqual([False, True, True]						,				__lowercase )
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]						,				__lowercase )
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]						,				__lowercase )







      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname						,				'''realm_block_records''' ) )

        # Test local path
        __UpperCAmelCase    :						List[str]       =						retriever.from_pretrained(os.path.join(self.tmpdirname						,				'''realm_block_records''' ) )
        self.assertEqual(retriever.block_records[0]						,				b'''This is the first record''' )

        # Test mocked remote path
        with patch('''transformers.models.realm.retrieval_realm.hf_hub_download''' ) as mock_hf_hub_download:
          __UpperCAmelCase    :						Dict       =						os.path.join(
              os.path.join(self.tmpdirname						,				'''realm_block_records''' )						,				_REALM_BLOCK_RECORDS_FILENAME )
          __UpperCAmelCase    :						Optional[int]       =						RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa''' )

        self.assertEqual(retriever.block_records[0]						,				b'''This is the first record''' )






"""simple docstring"""





import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowercase__				:int				   =       'src/transformers'
lowercase__				:List[str]				   =       'docs/source/en/tasks'


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->str:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''r'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
    __UpperCAmelCase    :						Union[str, Any]       =						f.readlines()
  # Find the start prompt.
  __UpperCAmelCase    :						Any       =						0
  while not lines[start_index].startswith(UpperCAmelCase_  ):
    start_index += 1
  start_index += 1

  __UpperCAmelCase    :						Optional[Any]       =						start_index
  while not lines[end_index].startswith(UpperCAmelCase_  ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]  ) <= 1:
    start_index += 1
  while len(lines[end_index]  ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowercase__				:Any				   =       direct_transformers_import(TRANSFORMERS_PATH)

lowercase__				:List[Any]				   =       {
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowercase__				:Union[str, Any]				   =       {
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[str, Any]:





  """simple docstring"""


  __UpperCAmelCase    :						List[str]       =						TASK_GUIDE_TO_MODELS[task_guide]
  __UpperCAmelCase    :						Dict       =						SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCAmelCase_  ,			set()  )
  __UpperCAmelCase    :						List[Any]       =						{
      code: name
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if (code in model_maping_names or code in special_model_types)
  }
  return ", ".join([f'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]  ) + "\n"


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_=False  )						->Tuple:





  """simple docstring"""


  __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						_find_text_in_file(
      filename=os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			start_prompt='''<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'''  ,			end_prompt='''<!--End of the generated tip-->'''  ,			)

  __UpperCAmelCase    :						List[str]       =						get_model_list_for_task(UpperCAmelCase_  )

  if current_list != new_list:
    if overwrite:
      with open(os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			'''w'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
        f.writelines(lines[:start_index] + [new_list] + lines[end_index:]  )
    else:
      raise ValueError(
          f'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
          ''' to fix this.'''  )


if __name__ == "__main__":
  lowercase__				:int				   =       argparse.ArgumentParser()
  parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
  lowercase__				:Optional[Any]				   =       parser.parse_args()

  for task_guide in TASK_GUIDE_TO_MODELS.keys():
    check_model_list_for_task(task_guide, args.fix_and_overwrite)