



import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
					import faiss
@require_faiss
class 			_a (					lowerCAmelCase):
			"""simple docstring"""



			def  lowercase__						(		self     :    Dict   )->List[str]:
							_UpperCAmelCase					=    tempfile.mkdtemp()
							_UpperCAmelCase					=    8

							# DPR tok
							_UpperCAmelCase					=    [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''dpr_tokenizer'''   )
							os.makedirs(__UpperCamelCase				,   exist_ok=__UpperCamelCase   )
							_UpperCAmelCase					=    os.path.join(__UpperCamelCase				,   DPR_VOCAB_FILES_NAMES['''vocab_file''']   )
							with open(self.vocab_file				,   '''w'''				,   encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )

							# BART tok
							_UpperCAmelCase					=    [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''\u0120''',
							    '''\u0120l''',
							    '''\u0120n''',
							    '''\u0120lo''',
							    '''\u0120low''',
							    '''er''',
							    '''\u0120lowest''',
							    '''\u0120newer''',
							    '''\u0120wider''',
							    '''<unk>''',
							]
							_UpperCAmelCase					=    dict(zip(__UpperCamelCase				,   range(len(__UpperCamelCase   )   )   )   )
							_UpperCAmelCase					=    ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
							_UpperCAmelCase					=    {'''unk_token''': '''<unk>'''}

							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''bart_tokenizer'''   )
							os.makedirs(__UpperCamelCase				,   exist_ok=__UpperCamelCase   )
							_UpperCAmelCase					=    os.path.join(__UpperCamelCase				,   BART_VOCAB_FILES_NAMES['''vocab_file''']   )
							_UpperCAmelCase					=    os.path.join(__UpperCamelCase				,   BART_VOCAB_FILES_NAMES['''merges_file''']   )
							with open(self.vocab_file				,   '''w'''				,   encoding='''utf-8'''   ) as fp:
											fp.write(json.dumps(__UpperCamelCase   ) + '''\n'''   )
							with open(self.merges_file				,   '''w'''				,   encoding='''utf-8'''   ) as fp:
											fp.write('''\n'''.join(__UpperCamelCase   )   )



			def  lowercase__						(		self     :    Optional[int]   )->DPRQuestionEncoderTokenizer:
							return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname				,   '''dpr_tokenizer'''   )   )



			def  lowercase__						(		self     :    Tuple   )->DPRContextEncoderTokenizer:
							return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname				,   '''dpr_tokenizer'''   )   )



			def  lowercase__						(		self     :    List[Any]   )->BartTokenizer:
							return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname				,   '''bart_tokenizer'''   )   )



			def  lowercase__						(		self     :    List[Any]   )->Tuple:
							shutil.rmtree(self.tmpdirname   )



			def  lowercase__						(		self     :    Tuple   )->Dict:
							_UpperCAmelCase					=    Dataset.from_dict(
							    {
							        '''id''': ['''0''', '''1'''],
							        '''text''': ['''foo''', '''bar'''],
							        '''title''': ['''Foo''', '''Bar'''],
							        '''embeddings''': [np.ones(self.retrieval_vector_size   ), 2 * np.ones(self.retrieval_vector_size   )],
							    }   )
							dataset.add_faiss_index('''embeddings'''				,   string_factory='''Flat'''				,   metric_type=faiss.METRIC_INNER_PRODUCT   )
							return dataset



			def  lowercase__						(		self     :    Optional[Any]   )->List[Any]:
							_UpperCAmelCase					=    self.get_dummy_dataset()
							_UpperCAmelCase					=    RagConfig(
							    retrieval_vector_size=self.retrieval_vector_size				,   question_encoder=DPRConfig().to_dict()				,   generator=BartConfig().to_dict()				,   )
							with patch('''transformers.models.rag.retrieval_rag.load_dataset'''   ) as mock_load_dataset:
											_UpperCAmelCase					=    dataset
											_UpperCAmelCase					=    RagRetriever(
											    __UpperCamelCase				,   question_encoder_tokenizer=self.get_dpr_tokenizer()				,   generator_tokenizer=self.get_bart_tokenizer()				,   )
							return retriever



			def  lowercase__						(		self     :    Optional[int]				,   __UpperCamelCase     :    bool   )->int:
							_UpperCAmelCase					=    self.get_dummy_dataset()
							_UpperCAmelCase					=    RagConfig(
							    retrieval_vector_size=self.retrieval_vector_size				,   question_encoder=DPRConfig().to_dict()				,   generator=BartConfig().to_dict()				,   index_name='''custom'''				,   )
							if from_disk:
											_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''dataset'''   )
											_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''index.faiss'''   )
											dataset.get_index('''embeddings'''   ).save(os.path.join(self.tmpdirname				,   '''index.faiss'''   )   )
											dataset.drop_index('''embeddings'''   )
											dataset.save_to_disk(os.path.join(self.tmpdirname				,   '''dataset'''   )   )
											del dataset
											_UpperCAmelCase					=    RagRetriever(
											    __UpperCamelCase				,   question_encoder_tokenizer=self.get_dpr_tokenizer()				,   generator_tokenizer=self.get_bart_tokenizer()				,   )
							else:
											_UpperCAmelCase					=    RagRetriever(
											    __UpperCamelCase				,   question_encoder_tokenizer=self.get_dpr_tokenizer()				,   generator_tokenizer=self.get_bart_tokenizer()				,   index=CustomHFIndex(config.retrieval_vector_size				,   __UpperCamelCase   )				,   )
							return retriever



			def  lowercase__						(		self     :    List[str]   )->List[str]:
							_UpperCAmelCase					=    Dataset.from_dict(
							    {
							        '''id''': ['''0''', '''1'''],
							        '''text''': ['''foo''', '''bar'''],
							        '''title''': ['''Foo''', '''Bar'''],
							        '''embeddings''': [np.ones(self.retrieval_vector_size + 1   ), 2 * np.ones(self.retrieval_vector_size + 1   )],
							    }   )
							dataset.add_faiss_index('''embeddings'''				,   string_factory='''Flat'''				,   metric_type=faiss.METRIC_INNER_PRODUCT   )

							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''   )
							dataset.save_faiss_index('''embeddings'''				,   index_file_name + '''.index.dpr'''   )
							pickle.dump(dataset['''id''']				,   open(index_file_name + '''.index_meta.dpr'''				,   '''wb'''   )   )

							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   '''psgs_w100.tsv.pkl'''   )
							_UpperCAmelCase					=    {sample['''id''']: [sample['''text'''], sample['''title''']] for sample in dataset}
							pickle.dump(__UpperCamelCase				,   open(__UpperCamelCase				,   '''wb'''   )   )

							_UpperCAmelCase					=    RagConfig(
							    retrieval_vector_size=self.retrieval_vector_size				,   question_encoder=DPRConfig().to_dict()				,   generator=BartConfig().to_dict()				,   index_name='''legacy'''				,   index_path=self.tmpdirname				,   )
							_UpperCAmelCase					=    RagRetriever(
							    __UpperCamelCase				,   question_encoder_tokenizer=self.get_dpr_tokenizer()				,   generator_tokenizer=self.get_bart_tokenizer()   )
							return retriever



			def  lowercase__						(		self     :    Optional[int]   )->List[Any]:
							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_canonical_hf_index_retriever()
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=__UpperCamelCase   )
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertEqual(len(__UpperCamelCase   )				,   2   )
							self.assertEqual(sorted(doc_dicts[0]   )				,   ['''embeddings''', '''id''', '''text''', '''title''']   )
							self.assertEqual(len(doc_dicts[0]['''id''']   )				,   __UpperCamelCase   )
							self.assertEqual(doc_dicts[0]['''id'''][0]				,   '''1'''   )  # max inner product is reached with second doc
							self.assertEqual(doc_dicts[1]['''id'''][0]				,   '''0'''   )  # max inner product is reached with first doc
							self.assertListEqual(doc_ids.tolist()				,   [[1], [0]]   )



			def  lowercase__						(		self     :    Tuple   )->Any:
							_UpperCAmelCase					=    self.get_dummy_canonical_hf_index_retriever()
							with tempfile.TemporaryDirectory() as tmp_dirname:
											with patch('''transformers.models.rag.retrieval_rag.load_dataset'''   ) as mock_load_dataset:
															_UpperCAmelCase					=    self.get_dummy_dataset()
															retriever.save_pretrained(__UpperCamelCase   )
															_UpperCAmelCase					=    RagRetriever.from_pretrained(__UpperCamelCase   )
															self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
															_UpperCAmelCase					=    np.array(
															    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
															_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=1   )
															self.assertTrue(out is not None   )



			def  lowercase__						(		self     :    str   )->Union[str, Any]:
							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCamelCase   )
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=__UpperCamelCase   )
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertEqual(len(__UpperCamelCase   )				,   2   )
							self.assertEqual(sorted(doc_dicts[0]   )				,   ['''embeddings''', '''id''', '''text''', '''title''']   )
							self.assertEqual(len(doc_dicts[0]['''id''']   )				,   __UpperCamelCase   )
							self.assertEqual(doc_dicts[0]['''id'''][0]				,   '''1'''   )  # max inner product is reached with second doc
							self.assertEqual(doc_dicts[1]['''id'''][0]				,   '''0'''   )  # max inner product is reached with first doc
							self.assertListEqual(doc_ids.tolist()				,   [[1], [0]]   )



			def  lowercase__						(		self     :    str   )->Optional[Any]:
							_UpperCAmelCase					=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCamelCase   )
							with tempfile.TemporaryDirectory() as tmp_dirname:
											retriever.save_pretrained(__UpperCamelCase   )
											_UpperCAmelCase					=    RagRetriever.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
											_UpperCAmelCase					=    np.array(
											    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
											_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=1   )
											self.assertTrue(out is not None   )



			def  lowercase__						(		self     :    int   )->str:
							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCamelCase   )
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=__UpperCamelCase   )
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertEqual(len(__UpperCamelCase   )				,   2   )
							self.assertEqual(sorted(doc_dicts[0]   )				,   ['''embeddings''', '''id''', '''text''', '''title''']   )
							self.assertEqual(len(doc_dicts[0]['''id''']   )				,   __UpperCamelCase   )
							self.assertEqual(doc_dicts[0]['''id'''][0]				,   '''1'''   )  # max inner product is reached with second doc
							self.assertEqual(doc_dicts[1]['''id'''][0]				,   '''0'''   )  # max inner product is reached with first doc
							self.assertListEqual(doc_ids.tolist()				,   [[1], [0]]   )



			def  lowercase__						(		self     :    int   )->List[str]:
							_UpperCAmelCase					=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCamelCase   )
							with tempfile.TemporaryDirectory() as tmp_dirname:
											retriever.save_pretrained(__UpperCamelCase   )
											_UpperCAmelCase					=    RagRetriever.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
											_UpperCAmelCase					=    np.array(
											    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
											_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=1   )
											self.assertTrue(out is not None   )



			def  lowercase__						(		self     :    str   )->Union[str, Any]:
							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_legacy_index_retriever()
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=__UpperCamelCase   )
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertEqual(len(__UpperCamelCase   )				,   2   )
							self.assertEqual(sorted(doc_dicts[0]   )				,   ['''text''', '''title''']   )
							self.assertEqual(len(doc_dicts[0]['''text''']   )				,   __UpperCamelCase   )
							self.assertEqual(doc_dicts[0]['''text'''][0]				,   '''bar'''   )  # max inner product is reached with second doc
							self.assertEqual(doc_dicts[1]['''text'''][0]				,   '''foo'''   )  # max inner product is reached with first doc
							self.assertListEqual(doc_ids.tolist()				,   [[1], [0]]   )



			def  lowercase__						(		self     :    str   )->List[str]:
							_UpperCAmelCase					=    self.get_dummy_legacy_index_retriever()
							with tempfile.TemporaryDirectory() as tmp_dirname:
											retriever.save_pretrained(__UpperCamelCase   )
											_UpperCAmelCase					=    RagRetriever.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
											_UpperCAmelCase					=    np.array(
											    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
											_UpperCAmelCase					=    retriever.retrieve(__UpperCamelCase				,   n_docs=1   )
											self.assertTrue(out is not None   )



			@require_torch
			@require_tokenizers
			@require_sentencepiece
			def  lowercase__						(		self     :    Tuple   )->str:
							import torch

							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_canonical_hf_index_retriever()
							_UpperCAmelCase					=    [[5, 7], [1_0, 1_1]]
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase					=    retriever(__UpperCamelCase				,   __UpperCamelCase				,   prefix=retriever.config.generator.prefix				,   n_docs=__UpperCamelCase   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    (
							    out['''context_input_ids'''],
							    out['''context_attention_mask'''],
							    out['''retrieved_doc_embeds'''],
							)
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
							self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )
							self.assertIsInstance(__UpperCamelCase				,   np.ndarray   )

							_UpperCAmelCase					=    retriever(
							    __UpperCamelCase				,   __UpperCamelCase				,   prefix=retriever.config.generator.prefix				,   n_docs=__UpperCamelCase				,   return_tensors='''pt'''				,   )
							_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase		,						_UpperCAmelCase					=    (  # noqa: F841
							    out['''context_input_ids'''],
							    out['''context_attention_mask'''],
							    out['''retrieved_doc_embeds'''],
							    out['''doc_ids'''],
							)
							self.assertEqual(retrieved_doc_embeds.shape				,   (2, n_docs, self.retrieval_vector_size)   )
							self.assertIsInstance(__UpperCamelCase				,   torch.Tensor   )
							self.assertIsInstance(__UpperCamelCase				,   torch.Tensor   )
							self.assertIsInstance(__UpperCamelCase				,   torch.Tensor   )



			@require_torch
			@require_tokenizers
			@require_sentencepiece
			def  lowercase__						(		self     :    Tuple   )->Optional[int]:
							_UpperCAmelCase					=    self.get_dpr_ctx_encoder_tokenizer()
							_UpperCAmelCase					=    1
							_UpperCAmelCase					=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCamelCase   )
							retriever.set_ctx_encoder_tokenizer(__UpperCamelCase   )

							_UpperCAmelCase					=    [[5, 7], [1_0, 1_1]]
							_UpperCAmelCase					=    np.array(
							    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]				,   dtype=np.floataa   )
							_UpperCAmelCase					=    retriever(__UpperCamelCase				,   __UpperCamelCase				,   prefix=retriever.config.generator.prefix				,   n_docs=__UpperCamelCase   )

							self.assertEqual(
							    len(__UpperCamelCase   )				,   6   )  # check whether the retriever output consist of 6 attributes including tokenized docs
							self.assertEqual(
							    all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')   )				,   __UpperCamelCase   )  # check for doc token related keys in dictionary.








"""simple docstring"""


from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml
class 			_a (					yaml.SafeLoader):
			"""simple docstring"""



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Any   )->List[Any]:
							_UpperCAmelCase					=    [self.constructed_objects[key_node] for key_node, _ in node.value]
							_UpperCAmelCase					=    [tuple(__UpperCamelCase   ) if isinstance(__UpperCamelCase				,   __UpperCamelCase   ) else key for key in keys]
							_UpperCAmelCase					=    Counter(__UpperCamelCase   )
							_UpperCAmelCase					=    [key for key in counter if counter[key] > 1]
							if duplicate_keys:
											raise TypeError(F'Got duplicate yaml keys: {duplicate_keys}'   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[Any]				,   __UpperCamelCase     :    str=False   )->Dict:
							_UpperCAmelCase					=    super().construct_mapping(__UpperCamelCase				,   deep=__UpperCamelCase   )
							self._check_no_duplicates_on_constructed_node(__UpperCamelCase   )
							return mapping







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						str				):






				'''simple docstring'''



				_UpperCAmelCase					=    list(readme_content.splitlines()				)
				if full_content and full_content[0] == "---" and "---" in full_content[1:]:
								_UpperCAmelCase					=    full_content[1:].index('''---'''				) + 1
								_UpperCAmelCase					=    '''\n'''.join(full_content[1:sep_idx]				)
								return yamlblock, "\n".join(full_content[sep_idx + 1 :]				)

				return None, "\n".join(_SCREAMING_SNAKE_CASE				)
class 			_a (					lowerCAmelCase):
			"""simple docstring"""
			# class attributes
			UpperCamelCase__			      =	{"""train_eval_index"""}  # train-eval-index in the YAML metadata



			@classmethod
			def  lowercase__						(		cls     :    List[Any]				,   __UpperCamelCase     :    Path   )->"DatasetMetadata":
							with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(readme_file.read()   )
							if yaml_string is not None:
											return cls.from_yaml_string(__UpperCamelCase   )
							else:
											return cls()



			def  lowercase__						(		self     :    Tuple				,   __UpperCamelCase     :    Path   )->List[Any]:
							if path.exists():
											with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
															_UpperCAmelCase					=    readme_file.read()
							else:
											_UpperCAmelCase					=    None
							_UpperCAmelCase					=    self._to_readme(__UpperCamelCase   )
							with open(__UpperCamelCase				,   '''w'''				,   encoding='''utf-8'''   ) as readme_file:
											readme_file.write(__UpperCamelCase   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[str] = None   )->str:
							if readme_content is not None:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(__UpperCamelCase   )
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n''' + content
							else:
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n'''
							return full_content



			@classmethod
			def  lowercase__						(		cls     :    str				,   __UpperCamelCase     :    str   )->"DatasetMetadata":
							_UpperCAmelCase					=    yaml.load(__UpperCamelCase				,   Loader=_NoDuplicateSafeLoader   ) or {}

							# Convert the YAML keys to DatasetMetadata fields
							_UpperCAmelCase					=    {
							    (key.replace('''-'''				,   '''_'''   ) if key.replace('''-'''				,   '''_'''   ) in cls._FIELDS_WITH_DASHES else key): value
							    for key, value in metadata_dict.items()
							}
							return cls(**__UpperCamelCase   )



			def  lowercase__						(		self     :    str   )->str:
							return yaml.safe_dump(
							    {
							        (key.replace('''_'''				,   '''-'''   ) if key in self._FIELDS_WITH_DASHES else key): value
							        for key, value in self.items()
							    }				,   sort_keys=__UpperCamelCase				,   allow_unicode=__UpperCamelCase				,   encoding='''utf-8'''				,   ).decode('''utf-8'''   )



__A   :							str										=   {
    "image-classification": [],
    "translation": [],
    "image-segmentation": [],
    "fill-mask": [],
    "automatic-speech-recognition": [],
    "token-classification": [],
    "sentence-similarity": [],
    "audio-classification": [],
    "question-answering": [],
    "summarization": [],
    "zero-shot-classification": [],
    "table-to-text": [],
    "feature-extraction": [],
    "other": [],
    "multiple-choice": [],
    "text-classification": [],
    "text-to-image": [],
    "text2text-generation": [],
    "zero-shot-image-classification": [],
    "tabular-classification": [],
    "tabular-regression": [],
    "image-to-image": [],
    "tabular-to-text": [],
    "unconditional-image-generation": [],
    "text-retrieval": [],
    "text-to-speech": [],
    "object-detection": [],
    "audio-to-audio": [],
    "text-generation": [],
    "conversational": [],
    "table-question-answering": [],
    "visual-question-answering": [],
    "image-to-text": [],
    "reinforcement-learning": [],
    "voice-activity-detection": [],
    "time-series-forecasting": [],
    "document-question-answering": [],
}


if __name__ == "__main__":
					from argparse import ArgumentParser

					__A   :							str										=   ArgumentParser(usage="Validate the yaml metadata block of a README.md file.")
					ap.add_argument("readme_filepath")
					__A   :							Union[str, Any]										=   ap.parse_args()

					__A   :							Dict										=   Path(args.readme_filepath)
					__A   :							Tuple										=   DatasetMetadata.from_readme(readme_filepath)
					print(dataset_metadata)
					dataset_metadata.to_readme(readme_filepath)



