



"""simple docstring"""

import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

A_						:    Union[str, Any]				=      yaml.safe_load(
    '\\nname: ""\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: "Dataset Card for X" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: "Table of Contents"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: "Dataset Description"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: "Dataset Summary"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: "Supported Tasks and Leaderboards"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n'
)


A_						:    Optional[int]				=      {
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}


A_						:    Tuple				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'


A_						:    Optional[Any]				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    Optional[Any]				=      {
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [
                                {
                                    'name': 'Extra Ignored Subsection',
                                    'text': '',
                                    'is_empty_text': True,
                                    'subsections': [],
                                }
                            ],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}

A_						:    List[Any]				=      '\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    Dict				=      (
    'The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README.'
)

A_						:    Dict				=      '\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    List[Any]				=      (
    'The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README.'
)

A_						:    Tuple				=      '\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    List[str]				=      'The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README.'

A_						:    List[Any]				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
A_						:    Any				=      'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored).'


A_						:    str				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n'
A_						:    List[str]				=      'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found \'None\'.'

A_						:    Dict				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n'

A_						:    Tuple				=      'The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`.'


A_						:    Optional[Any]				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n'

A_						:    List[str]				=      'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty.'

A_						:    Dict				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
A_						:    Optional[int]				=      'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.'

A_						:    Optional[int]				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n'

A_						:    str				=      'The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README.'

A_						:    Any				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    Dict				=      'The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'

A_						:    Tuple				=      ''

A_						:    List[Any]				=      'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README.'

A_						:    List[str]				=      '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A_						:    int				=      'The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections.'
@pytest.mark.parametrize(
    'readme_md, expected_dict'							,		[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]							,		)
def  __snake_case							(	__A      :						int							,		__A      :						int     )     ->       Dict:






				'''simple docstring'''
				assert ReadMe.from_string(__A							,		__A     ).to_dict() == expected_dict
@pytest.mark.parametrize(
    'readme_md, expected_error'							,		[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]							,		)
def  __snake_case							(	__A      :						Tuple							,		__A      :						int     )     ->       Tuple:






				'''simple docstring'''
				with pytest.raises(__A							,		match=re.escape(expected_error.format(path='root'     )     )     ):
								SCREAMING_SNAKE_CASE						:							List[str]							=     ReadMe.from_string(__A							,		__A     )
								readme.validate()
@pytest.mark.parametrize(
    'readme_md, expected_error'							,		[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]							,		)
def  __snake_case							(	__A      :						int							,		__A      :						Any     )     ->       Any:






				'''simple docstring'''
				with pytest.raises(__A							,		match=re.escape(expected_error.format(path='root'     )     )     ):
								ReadMe.from_string(__A							,		__A     )
@pytest.mark.parametrize(
    'readme_md,'							,		[
        (README_MULTIPLE_SAME_HEADING_1),
    ]							,		)
def  __snake_case							(	__A      :						List[str]     )     ->       str:






				'''simple docstring'''
				ReadMe.from_string(__A							,		__A							,		suppress_parsing_errors=__A     )
@pytest.mark.parametrize(
    'readme_md, expected_dict'							,		[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]							,		)
def  __snake_case							(	__A      :						str							,		__A      :						Dict     )     ->       List[str]:






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								SCREAMING_SNAKE_CASE						:							List[Any]							=     Path(__A     ) / 'README.md'
								with open(__A							,		'w+'     ) as readme_file:
												readme_file.write(__A     )
								SCREAMING_SNAKE_CASE						:							int							=     ReadMe.from_readme(__A							,		__A     ).to_dict()
								assert out["name"] == path
								assert out["text"] == ""
								assert out["is_empty_text"]
								assert out["subsections"] == expected_dict["subsections"]
@pytest.mark.parametrize(
    'readme_md, expected_error'							,		[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]							,		)
def  __snake_case							(	__A      :						str							,		__A      :						List[Any]     )     ->       Any:






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								SCREAMING_SNAKE_CASE						:							int							=     Path(__A     ) / 'README.md'
								with open(__A							,		'w+'     ) as readme_file:
												readme_file.write(__A     )
								SCREAMING_SNAKE_CASE						:							str							=     expected_error.format(path=__A     )
								with pytest.raises(__A							,		match=re.escape(__A     )     ):
												SCREAMING_SNAKE_CASE						:							Dict							=     ReadMe.from_readme(__A							,		__A     )
												readme.validate()
@pytest.mark.parametrize(
    'readme_md, expected_error'							,		[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]							,		)
def  __snake_case							(	__A      :						Dict							,		__A      :						Union[str, Any]     )     ->       Tuple:






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								SCREAMING_SNAKE_CASE						:							Any							=     Path(__A     ) / 'README.md'
								with open(__A							,		'w+'     ) as readme_file:
												readme_file.write(__A     )
								SCREAMING_SNAKE_CASE						:							List[str]							=     expected_error.format(path=__A     )
								with pytest.raises(__A							,		match=re.escape(__A     )     ):
												ReadMe.from_readme(__A							,		__A     )







@pytest.mark.parametrize(
    'readme_md,'							,		[
        (README_MULTIPLE_SAME_HEADING_1),
    ]							,		)
def  __snake_case							(	__A      :						List[Any]     )     ->       List[str]:






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								SCREAMING_SNAKE_CASE						:							Any							=     Path(__A     ) / 'README.md'
								with open(__A							,		'w+'     ) as readme_file:
												readme_file.write(__A     )
								ReadMe.from_readme(__A							,		__A							,		suppress_parsing_errors=__A     )










"""simple docstring"""

import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()
def  __snake_case							(	__A      :						Union[str, Any]							,		__A      :						Any							,		__A      :						Dict     )     ->       Tuple:






				'''simple docstring'''
				# Initialise PyTorch model
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     BertConfig.from_json_file(__A     )
				print(F"""Building PyTorch model from configuration: {config}"""     )
				SCREAMING_SNAKE_CASE						:							List[Any]							=     BertForPreTraining(__A     )

				# Load weights from tf checkpoint
				load_tf_weights_in_bert(__A							,		__A							,		__A     )

				# Save pytorch-model
				print(F"""Save PyTorch model to {pytorch_dump_path}"""     )
				torch.save(model.state_dict()							,		__A     )


if __name__ == "__main__":
							A_						:    int				=      argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
							)
							parser.add_argument(
							    '--bert_config_file',
							    default=None,
							    type=str,
							    required=True,
							    help=(
							        'The config json file corresponding to the pre-trained BERT model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							A_						:    str				=      parser.parse_args()
							convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





