


'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__A	=logging.get_logger(__name__)

__A	={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A	={
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

__A	={
    'bert-base-uncased': 5_12,
    'bert-large-uncased': 5_12,
    'bert-base-cased': 5_12,
    'bert-large-cased': 5_12,
    'bert-base-multilingual-uncased': 5_12,
    'bert-base-multilingual-cased': 5_12,
    'bert-base-chinese': 5_12,
    'bert-base-german-cased': 5_12,
    'bert-large-uncased-whole-word-masking': 5_12,
    'bert-large-cased-whole-word-masking': 5_12,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_12,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_12,
    'bert-base-cased-finetuned-mrpc': 5_12,
    'bert-base-german-dbmdz-cased': 5_12,
    'bert-base-german-dbmdz-uncased': 5_12,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_12,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_12,
    'wietsedv/bert-base-dutch-cased': 5_12,
}

__A	={
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Optional[Any]   =					VOCAB_FILES_NAMES
					lowerCAmelCase      :Any   =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase      :int   =					PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase      :Optional[int]   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase      :List[Any]   =					BertTokenizer




					def __init__(      self							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase="[UNK]"							,							_lowerCamelCase="[SEP]"							,							_lowerCamelCase="[PAD]"							,							_lowerCamelCase="[CLS]"							,							_lowerCamelCase="[MASK]"							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							**_lowerCamelCase							,							):
								super().__init__(
								    _lowerCamelCase							,							tokenizer_file=_lowerCamelCase							,							do_lower_case=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							tokenize_chinese_chars=_lowerCamelCase							,							strip_accents=_lowerCamelCase							,							**_lowerCamelCase							,							)

								UpperCAmelCase__  :					Optional[int]        =     json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get("""lowercase"""							,							_lowerCamelCase) != do_lower_case
								    or normalizer_state.get("""strip_accents"""							,							_lowerCamelCase) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars"""							,							_lowerCamelCase) != tokenize_chinese_chars
								):
											UpperCAmelCase__  :					Tuple        =     getattr(_lowerCamelCase							,							normalizer_state.pop("""type"""))
											UpperCAmelCase__  :					Optional[Any]        =     do_lower_case
											UpperCAmelCase__  :					int        =     strip_accents
											UpperCAmelCase__  :					Union[str, Any]        =     tokenize_chinese_chars
											UpperCAmelCase__  :					List[Any]        =     normalizer_class(**_lowerCamelCase)

								UpperCAmelCase__  :					List[Any]        =     do_lower_case




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase=None):
								UpperCAmelCase__  :					Dict        =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

								return output




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__  :					Optional[Any]        =     [self.sep_token_id]
								UpperCAmelCase__  :					Union[str, Any]        =     [self.cls_token_id]
								if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__  :					Tuple        =     self._tokenizer.model.save(_lowerCamelCase							,							name=_lowerCamelCase)
								return tuple(_lowerCamelCase)



'''simple docstring'''



from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml


class 		_snake_case					(     yaml.SafeLoader	):




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					Dict        =     [self.constructed_objects[key_node] for key_node, _ in node.value]
								UpperCAmelCase__  :					List[str]        =     [tuple(_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else key for key in keys]
								UpperCAmelCase__  :					List[str]        =     Counter(_lowerCamelCase)
								UpperCAmelCase__  :					Any        =     [key for key in counter if counter[key] > 1]
								if duplicate_keys:
											raise TypeError(f'''Got duplicate yaml keys: {duplicate_keys}''')




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase=False):
								UpperCAmelCase__  :					List[str]        =     super().construct_mapping(_lowerCamelCase							,							deep=_lowerCamelCase)
								self._check_no_duplicates_on_constructed_node(_lowerCamelCase)
								return mapping







def     _UpperCamelCase      (		UpperCamelCase__					):
			UpperCAmelCase__  :					Tuple        =     list(readme_content.splitlines()					)
			if full_content and full_content[0] == "---" and "---" in full_content[1:]:
						UpperCAmelCase__  :					Dict        =     full_content[1:].index("""---"""					) + 1
						UpperCAmelCase__  :					List[Any]        =     """\n""".join(full_content[1:sep_idx]					)
						return yamlblock, "\n".join(full_content[sep_idx + 1 :]					)

			return None, "\n".join(UpperCamelCase__					)


class 		_snake_case					(     a__	):
					# class attributes
					lowerCAmelCase      :str   =					{'''train_eval_index'''}  # train-eval-index in the YAML metadata




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase):
								with open(_lowerCamelCase							,							encoding="""utf-8""") as readme_file:
											UpperCAmelCase__   ,       UpperCAmelCase__  :					Any        =     _split_yaml_from_readme(readme_file.read())
								if yaml_string is not None:
											return cls.from_yaml_string(_lowerCamelCase)
								else:
											return cls()




					def 				snake_case__						(      self							,							_lowerCamelCase):
								if path.exists():
											with open(_lowerCamelCase							,							encoding="""utf-8""") as readme_file:
														UpperCAmelCase__  :					int        =     readme_file.read()
								else:
											UpperCAmelCase__  :					List[str]        =     None
								UpperCAmelCase__  :					Union[str, Any]        =     self._to_readme(_lowerCamelCase)
								with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as readme_file:
											readme_file.write(_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase = None):
								if readme_content is not None:
											UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[Any]        =     _split_yaml_from_readme(_lowerCamelCase)
											UpperCAmelCase__  :					str        =     """---\n""" + self.to_yaml_string() + """---\n""" + content
								else:
											UpperCAmelCase__  :					Optional[int]        =     """---\n""" + self.to_yaml_string() + """---\n"""
								return full_content




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase):
								UpperCAmelCase__  :					List[str]        =     yaml.load(_lowerCamelCase							,							Loader=_NoDuplicateSafeLoader) or {}

								# Convert the YAML keys to DatasetMetadata fields
								UpperCAmelCase__  :					List[str]        =     {
								    (key.replace("""-"""							,							"""_""") if key.replace("""-"""							,							"""_""") in cls._FIELDS_WITH_DASHES else key): value
								    for key, value in metadata_dict.items()
								}
								return cls(**_lowerCamelCase)




					def 				snake_case__						(      self):
								return yaml.safe_dump(
								    {
								        (key.replace("""_"""							,							"""-""") if key in self._FIELDS_WITH_DASHES else key): value
								        for key, value in self.items()
								    }							,							sort_keys=_lowerCamelCase							,							allow_unicode=_lowerCamelCase							,							encoding="""utf-8"""							,							).decode("""utf-8""")

__A	={
    'image-classification': [],
    'translation': [],
    'image-segmentation': [],
    'fill-mask': [],
    'automatic-speech-recognition': [],
    'token-classification': [],
    'sentence-similarity': [],
    'audio-classification': [],
    'question-answering': [],
    'summarization': [],
    'zero-shot-classification': [],
    'table-to-text': [],
    'feature-extraction': [],
    'other': [],
    'multiple-choice': [],
    'text-classification': [],
    'text-to-image': [],
    'text2text-generation': [],
    'zero-shot-image-classification': [],
    'tabular-classification': [],
    'tabular-regression': [],
    'image-to-image': [],
    'tabular-to-text': [],
    'unconditional-image-generation': [],
    'text-retrieval': [],
    'text-to-speech': [],
    'object-detection': [],
    'audio-to-audio': [],
    'text-generation': [],
    'conversational': [],
    'table-question-answering': [],
    'visual-question-answering': [],
    'image-to-text': [],
    'reinforcement-learning': [],
    'voice-activity-detection': [],
    'time-series-forecasting': [],
    'document-question-answering': [],
}


if __name__ == "__main__":
		from argparse import ArgumentParser

		__A	=ArgumentParser(usage='Validate the yaml metadata block of a README.md file.')
		ap.add_argument('readme_filepath')
		__A	=ap.parse_args()

		__A	=Path(args.readme_filepath)
		__A	=DatasetMetadata.from_readme(readme_filepath)
		print(dataset_metadata)
		dataset_metadata.to_readme(readme_filepath)