"""simple docstring"""







import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
_a  :  int=				"src/transformers"
_a  :  Tuple=				"docs/source/en/tasks"
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Union[str, Any]    ,							UpperCAmelCase_					:       int    ,							UpperCAmelCase_					:       Optional[int]						)							->    List[Any]:

	'''simple docstring'''





	with open(UpperCAmelCase_    ,							'r'    ,							encoding='utf-8'    ,							newline='\n'						) as f:
		__snake_case    :							List[Any]   	=				f.readlines()
	# Find the start prompt.
	__snake_case    :							Optional[Any]   	=				0
	while not lines[start_index].startswith(UpperCAmelCase_						):
		start_index += 1
	start_index += 1

	__snake_case    :							Any   	=				start_index
	while not lines[end_index].startswith(UpperCAmelCase_						):
		end_index += 1
	end_index -= 1

	while len(lines[start_index]						) <= 1:
		start_index += 1
	while len(lines[end_index]						) <= 1:
		end_index -= 1
	end_index += 1
	return "".join(lines[start_index:end_index]						), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
_a  :  int=				direct_transformers_import(TRANSFORMERS_PATH)

_a  :  Optional[int]=				{
    "asr.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    "audio_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    "language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    "image_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    "masked_language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    "multiple_choice.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    "object_detection.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    "question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    "semantic_segmentation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    "sequence_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    "summarization.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "token_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    "translation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "video_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    "document_question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    "monocular_depth_estimation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
_a  :  Optional[Any]=				{
    "summarization.md": ("nllb",),
    "translation.md": ("nllb",),
}
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Optional[int]						)							->    int:

	'''simple docstring'''





	__snake_case    :							int   	=				TASK_GUIDE_TO_MODELS[task_guide]
	__snake_case    :							Any   	=				SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCAmelCase_    ,							set()						)
	__snake_case    :							Optional[int]   	=				{
	    code: name
	    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
	    if (code in model_maping_names or code in special_model_types)
	}
	return ", ".join([F"[{name}](../model_doc/{code})" for code, name in model_names.items()]						) + "\n"
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Optional[int]    ,							UpperCAmelCase_					:       int=False						)							->    Union[str, Any]:

	'''simple docstring'''





	__snake_case	,       __snake_case	,       __snake_case	,       __snake_case    :							Union[str, Any]   	=				_find_text_in_file(
	    filename=os.path.join(UpperCAmelCase_    ,							UpperCAmelCase_						)    ,							start_prompt='<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'    ,							end_prompt='<!--End of the generated tip-->'    ,							)

	__snake_case    :							Dict   	=				get_model_list_for_task(UpperCAmelCase_						)

	if current_list != new_list:
		if overwrite:
			with open(os.path.join(UpperCAmelCase_    ,							UpperCAmelCase_						)    ,							'w'    ,							encoding='utf-8'    ,							newline='\n'						) as f:
				f.writelines(lines[:start_index] + [new_list] + lines[end_index:]						)
		else:
			raise ValueError(
			    F"The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`"
			    ' to fix this.'						)


if __name__ == "__main__":
	_a  :  str=				argparse.ArgumentParser()
	parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
	_a  :  Optional[Any]=				parser.parse_args()

	for task_guide in TASK_GUIDE_TO_MODELS.keys():
		check_model_list_for_task(task_guide, args.fix_and_overwrite)






"""simple docstring"""







import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

_a  :  List[str]=				yaml.safe_load(
    "\\nname: \"\"\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: \"Dataset Card for X\" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: \"Table of Contents\"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: \"Dataset Description\"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: \"Dataset Summary\"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: \"Supported Tasks and Leaderboards\"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n"
)


_a  :  Dict=				{
    "name": "root",
    "text": "",
    "is_empty_text": True,
    "subsections": [
        {
            "name": "Dataset Card for My Dataset",
            "text": "",
            "is_empty_text": True,
            "subsections": [
                {"name": "Table of Contents", "text": "Some text here.", "is_empty_text": False, "subsections": []},
                {
                    "name": "Dataset Description",
                    "text": "Some text here.",
                    "is_empty_text": False,
                    "subsections": [
                        {
                            "name": "Dataset Summary",
                            "text": "Some text here.",
                            "is_empty_text": False,
                            "subsections": [],
                        },
                        {
                            "name": "Supported Tasks and Leaderboards",
                            "text": "",
                            "is_empty_text": True,
                            "subsections": [],
                        },
                        {"name": "Languages", "text": "Language Text", "is_empty_text": False, "subsections": []},
                    ],
                },
            ],
        }
    ],
}


_a  :  str=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"


_a  :  Tuple=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  Dict=				{
    "name": "root",
    "text": "",
    "is_empty_text": True,
    "subsections": [
        {
            "name": "Dataset Card for My Dataset",
            "text": "",
            "is_empty_text": True,
            "subsections": [
                {"name": "Table of Contents", "text": "Some text here.", "is_empty_text": False, "subsections": []},
                {
                    "name": "Dataset Description",
                    "text": "Some text here.",
                    "is_empty_text": False,
                    "subsections": [
                        {
                            "name": "Dataset Summary",
                            "text": "Some text here.",
                            "is_empty_text": False,
                            "subsections": [
                                {
                                    "name": "Extra Ignored Subsection",
                                    "text": "",
                                    "is_empty_text": True,
                                    "subsections": [],
                                }
                            ],
                        },
                        {
                            "name": "Supported Tasks and Leaderboards",
                            "text": "",
                            "is_empty_text": True,
                            "subsections": [],
                        },
                        {"name": "Languages", "text": "Language Text", "is_empty_text": False, "subsections": []},
                    ],
                },
            ],
        }
    ],
}

_a  :  str=				"\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  Tuple=				(
    "The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README."
)

_a  :  Optional[int]=				"\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  List[str]=				(
    "The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README."
)

_a  :  str=				"\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  Optional[int]=				"The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README."

_a  :  int=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"
_a  :  str=				"The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored)."


_a  :  int=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n"
_a  :  Optional[Any]=				"The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found 'None'."

_a  :  Dict=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n"

_a  :  Optional[Any]=				"The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`."


_a  :  int=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n"

_a  :  Optional[Any]=				"The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty."

_a  :  Optional[int]=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"
_a  :  Any=				"The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README."

_a  :  Tuple=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n"

_a  :  List[str]=				"The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README."

_a  :  List[Any]=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  Tuple=				"The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."

_a  :  Optional[Any]=				""

_a  :  Tuple=				"The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README."

_a  :  Union[str, Any]=				"\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n"

_a  :  List[Any]=				"The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections."
@pytest.mark.parametrize(
    'readme_md, expected_dict'    ,							[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Tuple    ,							UpperCAmelCase_					:       Tuple						)							->    Tuple:

	'''simple docstring'''





	assert ReadMe.from_string(UpperCAmelCase_    ,							UpperCAmelCase_						).to_dict() == expected_dict
@pytest.mark.parametrize(
    'readme_md, expected_error'    ,							[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       List[str]    ,							UpperCAmelCase_					:       Tuple						)							->    Union[str, Any]:

	'''simple docstring'''





	with pytest.raises(UpperCAmelCase_    ,							match=re.escape(expected_error.format(path='root'						)						)						):
		__snake_case    :							Tuple   	=				ReadMe.from_string(UpperCAmelCase_    ,							UpperCAmelCase_						)
		readme.validate()
@pytest.mark.parametrize(
    'readme_md, expected_error'    ,							[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       List[str]    ,							UpperCAmelCase_					:       int						)							->    Optional[Any]:

	'''simple docstring'''





	with pytest.raises(UpperCAmelCase_    ,							match=re.escape(expected_error.format(path='root'						)						)						):
		ReadMe.from_string(UpperCAmelCase_    ,							UpperCAmelCase_						)
@pytest.mark.parametrize(
    'readme_md,'    ,							[
        (README_MULTIPLE_SAME_HEADING_1),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Optional[int]						)							->    Any:

	'''simple docstring'''





	ReadMe.from_string(UpperCAmelCase_    ,							UpperCAmelCase_    ,							suppress_parsing_errors=UpperCAmelCase_						)
@pytest.mark.parametrize(
    'readme_md, expected_dict'    ,							[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Optional[int]    ,							UpperCAmelCase_					:       Union[str, Any]						)							->    str:

	'''simple docstring'''





	with tempfile.TemporaryDirectory() as tmp_dir:
		__snake_case    :							Union[str, Any]   	=				Path(UpperCAmelCase_						) / 'README.md'
		with open(UpperCAmelCase_    ,							'w+'						) as readme_file:
			readme_file.write(UpperCAmelCase_						)
		__snake_case    :							Optional[Any]   	=				ReadMe.from_readme(UpperCAmelCase_    ,							UpperCAmelCase_						).to_dict()
		assert out["name"] == path
		assert out["text"] == ""
		assert out["is_empty_text"]
		assert out["subsections"] == expected_dict["subsections"]
@pytest.mark.parametrize(
    'readme_md, expected_error'    ,							[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Optional[Any]    ,							UpperCAmelCase_					:       Dict						)							->    int:

	'''simple docstring'''





	with tempfile.TemporaryDirectory() as tmp_dir:
		__snake_case    :							Optional[int]   	=				Path(UpperCAmelCase_						) / 'README.md'
		with open(UpperCAmelCase_    ,							'w+'						) as readme_file:
			readme_file.write(UpperCAmelCase_						)
		__snake_case    :							List[Any]   	=				expected_error.format(path=UpperCAmelCase_						)
		with pytest.raises(UpperCAmelCase_    ,							match=re.escape(UpperCAmelCase_						)						):
			__snake_case    :							Any   	=				ReadMe.from_readme(UpperCAmelCase_    ,							UpperCAmelCase_						)
			readme.validate()
@pytest.mark.parametrize(
    'readme_md, expected_error'    ,							[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Any    ,							UpperCAmelCase_					:       Union[str, Any]						)							->    Any:

	'''simple docstring'''





	with tempfile.TemporaryDirectory() as tmp_dir:
		__snake_case    :							Tuple   	=				Path(UpperCAmelCase_						) / 'README.md'
		with open(UpperCAmelCase_    ,							'w+'						) as readme_file:
			readme_file.write(UpperCAmelCase_						)
		__snake_case    :							List[str]   	=				expected_error.format(path=UpperCAmelCase_						)
		with pytest.raises(UpperCAmelCase_    ,							match=re.escape(UpperCAmelCase_						)						):
			ReadMe.from_readme(UpperCAmelCase_    ,							UpperCAmelCase_						)

@pytest.mark.parametrize(
    'readme_md,'    ,							[
        (README_MULTIPLE_SAME_HEADING_1),
    ]    ,							)
def 						__UpperCAmelCase       (					UpperCAmelCase_					:       Dict						)							->    Optional[Any]:

	'''simple docstring'''





	with tempfile.TemporaryDirectory() as tmp_dir:
		__snake_case    :							Optional[int]   	=				Path(UpperCAmelCase_						) / 'README.md'
		with open(UpperCAmelCase_    ,							'w+'						) as readme_file:
			readme_file.write(UpperCAmelCase_						)
		ReadMe.from_readme(UpperCAmelCase_    ,							UpperCAmelCase_    ,							suppress_parsing_errors=UpperCAmelCase_						)





