
import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
__lowerCAmelCase					:			List[str]			      =		"src/transformers"
__lowerCAmelCase					:			Any			      =		"docs/source/en/tasks"


def     UpperCAmelCase_  (						__lowerCAmelCase	,					__lowerCAmelCase	,					__lowerCAmelCase	) ->						List[str]:
    with open(__lowerCAmelCase	,					'''r'''	,					encoding='''utf-8'''	,					newline='''\n'''	) as f:
        __lowercase		:			List[str]	    =       f.readlines()
    # Find the start prompt.
    __lowercase		:			Dict	    =       0
    while not lines[start_index].startswith(__lowerCAmelCase	):
        start_index += 1
    start_index += 1

    __lowercase		:			Union[str, Any]	    =       start_index
    while not lines[end_index].startswith(__lowerCAmelCase	):
        end_index += 1
    end_index -= 1

    while len(lines[start_index]	) <= 1:
        start_index += 1
    while len(lines[end_index]	) <= 1:
        end_index -= 1
    end_index += 1
    return "".join(lines[start_index:end_index]	), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
__lowerCAmelCase					:			str			      =		direct_transformers_import(TRANSFORMERS_PATH)

__lowerCAmelCase					:			Tuple			      =		{
    "asr.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    "audio_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    "language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    "image_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    "masked_language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    "multiple_choice.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    "object_detection.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    "question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    "semantic_segmentation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    "sequence_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    "summarization.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "token_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    "translation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "video_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    "document_question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    "monocular_depth_estimation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
__lowerCAmelCase					:			Tuple			      =		{
    "summarization.md": ("nllb",),
    "translation.md": ("nllb",),
}


def     UpperCAmelCase_  (						__lowerCAmelCase	) ->						Any:
    __lowercase		:			Optional[int]	    =       TASK_GUIDE_TO_MODELS[task_guide]
    __lowercase		:			Any	    =       SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(__lowerCAmelCase	,					set()	)
    __lowercase		:			str	    =       {
        code: name
        for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
        if (code in model_maping_names or code in special_model_types)
    }
    return ", ".join([F'[{name}](../model_doc/{code})' for code, name in model_names.items()]	) + "\n"


def     UpperCAmelCase_  (						__lowerCAmelCase	,					__lowerCAmelCase=False	) ->						List[str]:
    __lowercase    ,      __lowercase    ,      __lowercase    ,      __lowercase		:			Union[str, Any]	    =       _find_text_in_file(
        filename=os.path.join(__lowerCAmelCase	,					__lowerCAmelCase	)	,					start_prompt='''<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'''	,					end_prompt='''<!--End of the generated tip-->'''	,					)

    __lowercase		:			List[str]	    =       get_model_list_for_task(__lowerCAmelCase	)

    if current_list != new_list:
        if overwrite:
            with open(os.path.join(__lowerCAmelCase	,					__lowerCAmelCase	)	,					'''w'''	,					encoding='''utf-8'''	,					newline='''\n'''	) as f:
                f.writelines(lines[:start_index] + [new_list] + lines[end_index:]	)
        else:
            raise ValueError(
                F'The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'
                ''' to fix this.'''	)


if __name__ == "__main__":
 __lowerCAmelCase					:			Union[str, Any]			      =		argparse.ArgumentParser()
 parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
 __lowerCAmelCase					:			Tuple			      =		parser.parse_args()

 for task_guide in TASK_GUIDE_TO_MODELS.keys():
  check_model_list_for_task(task_guide, args.fix_and_overwrite)




import unittest
from queue import Empty
from threading import Thread

from transformers import AutoTokenizer, TextIteratorStreamer, TextStreamer, is_torch_available
from transformers.testing_utils import CaptureStdout, require_torch, torch_device

from ..test_modeling_common import ids_tensor


if is_torch_available():
 import torch

 from transformers import AutoModelForCausalLM


@require_torch
class 						__lowerCAmelCase	(	unittest.TestCase      ):



      """simple docstring"""







      def 		snake_case_			(      self		:       Union[str, Any]				):
          __lowercase		:			Optional[Any]	    =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
          __lowercase		:			Any	    =       AutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				).to(_snake_case				)
          __lowercase		:			List[str]	    =       -1

          __lowercase		:			Optional[Any]	    =       ids_tensor((1, 5)				,       vocab_size=model.config.vocab_size				).to(_snake_case				)
          __lowercase		:			Tuple	    =       model.generate(_snake_case				,       max_new_tokens=10				,       do_sample=_snake_case				)
          __lowercase		:			Optional[int]	    =       tokenizer.decode(greedy_ids[0]				)

          with CaptureStdout() as cs:
              __lowercase		:			Optional[Any]	    =       TextStreamer(_snake_case				)
              model.generate(_snake_case				,       max_new_tokens=10				,       do_sample=_snake_case				,       streamer=_snake_case				)
          # The greedy text should be printed to stdout, except for the final "\n" in the streamer
          __lowercase		:			List[Any]	    =       cs.out[:-1]

          self.assertEqual(_snake_case				,       _snake_case				)




      def 		snake_case_			(      self		:       Optional[int]				):
          __lowercase		:			Optional[int]	    =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
          __lowercase		:			Optional[Any]	    =       AutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				).to(_snake_case				)
          __lowercase		:			List[str]	    =       -1

          __lowercase		:			Tuple	    =       ids_tensor((1, 5)				,       vocab_size=model.config.vocab_size				).to(_snake_case				)
          __lowercase		:			List[Any]	    =       model.generate(_snake_case				,       max_new_tokens=10				,       do_sample=_snake_case				)
          __lowercase		:			Optional[Any]	    =       tokenizer.decode(greedy_ids[0]				)

          __lowercase		:			int	    =       TextIteratorStreamer(_snake_case				)
          __lowercase		:			int	    =       {'''input_ids''': input_ids, '''max_new_tokens''': 10, '''do_sample''': False, '''streamer''': streamer}
          __lowercase		:			Optional[Any]	    =       Thread(target=model.generate				,       kwargs=_snake_case				)
          thread.start()
          __lowercase		:			Optional[int]	    =       ''''''
          for new_text in streamer:
              streamer_text += new_text

          self.assertEqual(_snake_case				,       _snake_case				)




      def 		snake_case_			(      self		:       List[str]				):
          __lowercase		:			Dict	    =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
          __lowercase		:			int	    =       AutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				).to(_snake_case				)
          __lowercase		:			List[Any]	    =       -1

          __lowercase		:			List[Any]	    =       ids_tensor((1, 5)				,       vocab_size=model.config.vocab_size				).to(_snake_case				)
          __lowercase		:			str	    =       model.generate(_snake_case				,       max_new_tokens=10				,       do_sample=_snake_case				)
          __lowercase		:			List[Any]	    =       greedy_ids[:, input_ids.shape[1] :]
          __lowercase		:			Optional[Any]	    =       tokenizer.decode(new_greedy_ids[0]				)

          with CaptureStdout() as cs:
              __lowercase		:			int	    =       TextStreamer(_snake_case				,       skip_prompt=_snake_case				)
              model.generate(_snake_case				,       max_new_tokens=10				,       do_sample=_snake_case				,       streamer=_snake_case				)
          # The greedy text should be printed to stdout, except for the final "\n" in the streamer
          __lowercase		:			int	    =       cs.out[:-1]

          self.assertEqual(_snake_case				,       _snake_case				)




      def 		snake_case_			(      self		:       Any				):
          # Tests that we can pass `decode_kwargs` to the streamer to control how the tokens are decoded. Must be tested
          # with actual models -- the dummy models' tokenizers are not aligned with their models, and
          # `skip_special_tokens=True` has no effect on them
          __lowercase		:			Dict	    =       AutoTokenizer.from_pretrained('''distilgpt2'''				)
          __lowercase		:			List[str]	    =       AutoModelForCausalLM.from_pretrained('''distilgpt2'''				).to(_snake_case				)
          __lowercase		:			str	    =       -1

          __lowercase		:			Dict	    =       torch.ones((1, 5)				,       device=_snake_case				).long() * model.config.bos_token_id
          with CaptureStdout() as cs:
              __lowercase		:			Optional[Any]	    =       TextStreamer(_snake_case				,       skip_special_tokens=_snake_case				)
              model.generate(_snake_case				,       max_new_tokens=1				,       do_sample=_snake_case				,       streamer=_snake_case				)

          # The prompt contains a special token, so the streamer should not print it. As such, the output text, when
          # re-tokenized, must only contain one token
          __lowercase		:			Dict	    =       cs.out[:-1]  # Remove the final "\n"
          __lowercase		:			Any	    =       tokenizer(_snake_case				,       return_tensors='''pt'''				)
          self.assertEqual(streamer_text_tokenized.input_ids.shape				,       (1, 1)				)




      def 		snake_case_			(      self		:       Any				):
          __lowercase		:			List[str]	    =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
          __lowercase		:			Union[str, Any]	    =       AutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				).to(_snake_case				)
          __lowercase		:			Dict	    =       -1

          __lowercase		:			List[Any]	    =       ids_tensor((1, 5)				,       vocab_size=model.config.vocab_size				).to(_snake_case				)
          __lowercase		:			Tuple	    =       TextIteratorStreamer(_snake_case				,       timeout=0.0_01				)
          __lowercase		:			Tuple	    =       {'''input_ids''': input_ids, '''max_new_tokens''': 10, '''do_sample''': False, '''streamer''': streamer}
          __lowercase		:			Optional[int]	    =       Thread(target=model.generate				,       kwargs=_snake_case				)
          thread.start()

          # The streamer will timeout after 0.001 seconds, so an exception will be raised
          with self.assertRaises(_snake_case				):
              __lowercase		:			List[Any]	    =       ''''''
              for new_text in streamer:
                  streamer_text += new_text


