


import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowerCAmelCase__			:      List[Any]								='src/transformers'
lowerCAmelCase__			:      Optional[int]								='docs/source/en/tasks'






def 						a__			(				A__,      A__,      A__				):
			with open(A__,      'r',      encoding='utf-8',      newline='\n'				) as f:
						SCREAMING_SNAKE_CASE_		:     List[Any]					=					f.readlines()
			# Find the start prompt.
			SCREAMING_SNAKE_CASE_		:     List[Any]					=					0
			while not lines[start_index].startswith(A__				):
						start_index += 1
			start_index += 1

			SCREAMING_SNAKE_CASE_		:     List[Any]					=					start_index
			while not lines[end_index].startswith(A__				):
						end_index += 1
			end_index -= 1

			while len(lines[start_index]				) <= 1:
						start_index += 1
			while len(lines[end_index]				) <= 1:
						end_index -= 1
			end_index += 1
			return "".join(lines[start_index:end_index]				), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__			:      List[Any]								=direct_transformers_import(TRANSFORMERS_PATH)

lowerCAmelCase__			:      Dict								={
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowerCAmelCase__			:      Optional[Any]								={
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}






def 						a__			(				A__				):
			SCREAMING_SNAKE_CASE_		:     str					=					TASK_GUIDE_TO_MODELS[task_guide]
			SCREAMING_SNAKE_CASE_		:     List[Any]					=					SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(A__,      set()				)
			SCREAMING_SNAKE_CASE_		:     List[Any]					=					{
			    code: name
			    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
			    if (code in model_maping_names or code in special_model_types)
			}
			return ", ".join([F'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]				) + "\n"






def 						a__			(				A__,      A__=False				):
			SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     Any					=					_find_text_in_file(
			    filename=os.path.join(A__,      A__				),      start_prompt='<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->',      end_prompt='<!--End of the generated tip-->',      )

			SCREAMING_SNAKE_CASE_		:     str					=					get_model_list_for_task(A__				)

			if current_list != new_list:
						if overwrite:
									with open(os.path.join(A__,      A__				),      'w',      encoding='utf-8',      newline='\n'				) as f:
												f.writelines(lines[:start_index] + [new_list] + lines[end_index:]				)
						else:
									raise ValueError(
									    F'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
									    ' to fix this.'				)


if __name__ == "__main__":
				lowerCAmelCase__			:      Any								=argparse.ArgumentParser()
				parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
				lowerCAmelCase__			:      Optional[Any]								=parser.parse_args()

				for task_guide in TASK_GUIDE_TO_MODELS.keys():
								check_model_list_for_task(task_guide, args.fix_and_overwrite)









import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class 	__lowercase    (__SCREAMING_SNAKE_CASE  ):





							"""simple docstring"""







							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Tuple					=					tempfile.mkdtemp()
										SCREAMING_SNAKE_CASE_		:     Dict					=					5

										# Realm tok
										SCREAMING_SNAKE_CASE_		:     Dict					=					[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'test',
										    'question',
										    'this',
										    'is',
										    'the',
										    'first',
										    'second',
										    'third',
										    'fourth',
										    'fifth',
										    'record',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										SCREAMING_SNAKE_CASE_		:     Tuple					=					os.path.join(self.tmpdirname			,						'realm_tokenizer'       )
										os.makedirs(lowerCAmelCase__			,						exist_ok=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Tuple					=					os.path.join(lowerCAmelCase__			,						VOCAB_FILES_NAMES['vocab_file']       )
										with open(self.vocab_file			,						'w'			,						encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )

										SCREAMING_SNAKE_CASE_		:     str					=					os.path.join(self.tmpdirname			,						'realm_block_records'       )
										os.makedirs(lowerCAmelCase__			,						exist_ok=lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname			,						'realm_tokenizer'       )       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										shutil.rmtree(self.tmpdirname       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					RealmConfig(num_block_records=self.num_block_records       )
										return config






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					Dataset.from_dict(
										    {
										        'id': ['0', '1'],
										        'question': ['foo', 'bar'],
										        'answers': [['Foo', 'Bar'], ['Bar']],
										    }       )
										return dataset






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					np.array(
										    [
										        b'This is the first record',
										        b'This is the second record',
										        b'This is the third record',
										        b'This is the fourth record',
										        b'This is the fifth record',
										        b'This is a longer longer longer record',
										    ]			,						dtype=lowerCAmelCase__			,						)
										return block_records






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					RealmRetriever(
										    block_records=self.get_dummy_block_records()			,						tokenizer=self.get_tokenizer()			,						)
										return retriever






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.get_config()
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_dummy_retriever()
										SCREAMING_SNAKE_CASE_		:     List[str]					=					retriever.tokenizer

										SCREAMING_SNAKE_CASE_		:     Any					=					np.array([0, 3]			,						dtype='long'       )
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tokenizer(['Test question']       ).input_ids
										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer(
										    ['the fourth']			,						add_special_tokens=lowerCAmelCase__			,						return_token_type_ids=lowerCAmelCase__			,						return_attention_mask=lowerCAmelCase__			,						).input_ids
										SCREAMING_SNAKE_CASE_		:     Any					=					config.reader_seq_len

										SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     str					=					retriever(
										    lowerCAmelCase__			,						lowerCAmelCase__			,						answer_ids=lowerCAmelCase__			,						max_length=lowerCAmelCase__			,						return_tensors='np'       )

										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(concat_inputs.input_ids.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.attention_mask.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.token_type_ids.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.special_tokens_mask.shape			,						(2, 1_0)       )
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]       )			,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']			,						)
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]       )			,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']			,						)






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_config()
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.get_dummy_retriever()
										SCREAMING_SNAKE_CASE_		:     Tuple					=					retriever.tokenizer

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					np.array([0, 3, 5]			,						dtype='long'       )
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer(['Test question']       ).input_ids
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer(
										    ['the fourth', 'longer longer']			,						add_special_tokens=lowerCAmelCase__			,						return_token_type_ids=lowerCAmelCase__			,						return_attention_mask=lowerCAmelCase__			,						).input_ids
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					config.reader_seq_len

										SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     Tuple					=					retriever(
										    lowerCAmelCase__			,						lowerCAmelCase__			,						answer_ids=lowerCAmelCase__			,						max_length=lowerCAmelCase__			,						return_tensors='np'       )

										self.assertEqual([False, True, True]			,						lowerCAmelCase__       )
										self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]			,						lowerCAmelCase__       )
										self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_dummy_retriever()
										retriever.save_pretrained(os.path.join(self.tmpdirname			,						'realm_block_records'       )       )

										# Test local path
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					retriever.from_pretrained(os.path.join(self.tmpdirname			,						'realm_block_records'       )       )
										self.assertEqual(retriever.block_records[0]			,						b'This is the first record'       )

										# Test mocked remote path
										with patch('transformers.models.realm.retrieval_realm.hf_hub_download'       ) as mock_hf_hub_download:
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					os.path.join(
													    os.path.join(self.tmpdirname			,						'realm_block_records'       )			,						_REALM_BLOCK_RECORDS_FILENAME       )
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'       )

										self.assertEqual(retriever.block_records[0]			,						b'This is the first record'       )





