



"""simple docstring"""


import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402
class 			_a (					unittest.TestCase):
			"""simple docstring"""



			def  lowercase__						(		self     :    str   )->str:
							_UpperCAmelCase					=    0



			def  lowercase__						(		self     :    int   )->List[Any]:
							_UpperCAmelCase					=    AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32'''   )
							self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    Dict   )->List[str]:
							with tempfile.TemporaryDirectory() as tmpdirname:
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''preprocessor_config.json'''
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''config.json'''
											json.dump(
											    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}				,   open(__UpperCamelCase				,   '''w'''   )				,   )
											json.dump({'''model_type''': '''clip'''}				,   open(__UpperCamelCase				,   '''w'''   )   )

											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    int   )->List[Any]:
							# Ensure we can load the image processor from the feature extractor config
							with tempfile.TemporaryDirectory() as tmpdirname:
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''preprocessor_config.json'''
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''config.json'''
											json.dump(
											    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}				,   open(__UpperCamelCase				,   '''w'''   )				,   )
											json.dump({'''model_type''': '''clip'''}				,   open(__UpperCamelCase				,   '''w'''   )   )

											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    Optional[Any]   )->Dict:
							with tempfile.TemporaryDirectory() as tmpdirname:
											_UpperCAmelCase					=    CLIPConfig()

											# Create a dummy config file with image_proceesor_type
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''preprocessor_config.json'''
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''config.json'''
											json.dump(
											    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}				,   open(__UpperCamelCase				,   '''w'''   )				,   )
											json.dump({'''model_type''': '''clip'''}				,   open(__UpperCamelCase				,   '''w'''   )   )

											# remove image_processor_type to make sure config.json alone is enough to load image processor locally
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   ).to_dict()

											config_dict.pop('''image_processor_type'''   )
											_UpperCAmelCase					=    CLIPImageProcessor(**__UpperCamelCase   )

											# save in new folder
											model_config.save_pretrained(__UpperCamelCase   )
											config.save_pretrained(__UpperCamelCase   )

											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   )

											# make sure private variable is not incorrectly saved
											_UpperCAmelCase					=    json.loads(config.to_json_string()   )
											self.assertTrue('''_processor_class''' not in dict_as_saved   )

							self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    int   )->Union[str, Any]:
							with tempfile.TemporaryDirectory() as tmpdirname:
											_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''preprocessor_config.json'''
											json.dump(
											    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}				,   open(__UpperCamelCase				,   '''w'''   )				,   )

											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   )
											self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    List[Any]   )->Optional[int]:
							with self.assertRaisesRegex(
							    __UpperCamelCase				,   '''clip-base is not a local folder and is not a valid model identifier'''   ):
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained('''clip-base'''   )



			def  lowercase__						(		self     :    Any   )->Optional[int]:
							with self.assertRaisesRegex(
							    __UpperCamelCase				,   r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''   ):
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase				,   revision='''aaaaaa'''   )



			def  lowercase__						(		self     :    str   )->str:
							with self.assertRaisesRegex(
							    __UpperCamelCase				,   '''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''				,   ):
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model'''   )



			def  lowercase__						(		self     :    Optional[int]   )->Optional[int]:
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(__UpperCamelCase   ):
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''   )
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__UpperCamelCase   ):
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(
											    '''hf-internal-testing/test_dynamic_image_processor'''				,   trust_remote_code=__UpperCamelCase   )

							_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(
							    '''hf-internal-testing/test_dynamic_image_processor'''				,   trust_remote_code=__UpperCamelCase   )
							self.assertEqual(image_processor.__class__.__name__				,   '''NewImageProcessor'''   )

							# Test image processor can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
											image_processor.save_pretrained(__UpperCamelCase   )
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase				,   trust_remote_code=__UpperCamelCase   )
							self.assertEqual(reloaded_image_processor.__class__.__name__				,   '''NewImageProcessor'''   )



			def  lowercase__						(		self     :    str   )->List[str]:
							try:
											AutoConfig.register('''custom'''				,   __UpperCamelCase   )
											AutoImageProcessor.register(__UpperCamelCase				,   __UpperCamelCase   )
											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(__UpperCamelCase   ):
															AutoImageProcessor.register(__UpperCamelCase				,   __UpperCamelCase   )

											with tempfile.TemporaryDirectory() as tmpdirname:
															_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''preprocessor_config.json'''
															_UpperCAmelCase					=    Path(__UpperCamelCase   ) / '''config.json'''
															json.dump(
															    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}				,   open(__UpperCamelCase				,   '''w'''   )				,   )
															json.dump({'''model_type''': '''clip'''}				,   open(__UpperCamelCase				,   '''w'''   )   )

															_UpperCAmelCase					=    CustomImageProcessor.from_pretrained(__UpperCamelCase   )

											# Now that the config is registered, it can be used as any other config with the auto-API
											with tempfile.TemporaryDirectory() as tmp_dir:
															image_processor.save_pretrained(__UpperCamelCase   )
															_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(__UpperCamelCase   )
															self.assertIsInstance(__UpperCamelCase				,   __UpperCamelCase   )

							finally:
											if "custom" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
															del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]



			def  lowercase__						(		self     :    List[str]   )->Union[str, Any]:
							class 			_a (					lowerCAmelCase):
										"""simple docstring"""
										UpperCamelCase__			      =	True

							try:
											AutoConfig.register('''custom'''				,   __UpperCamelCase   )
											AutoImageProcessor.register(__UpperCamelCase				,   __UpperCamelCase   )
											# If remote code is not set, the default is to use local
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''   )
											self.assertEqual(image_processor.__class__.__name__				,   '''NewImageProcessor'''   )
											self.assertTrue(image_processor.is_local   )

											# If remote code is disabled, we load the local one.
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(
											    '''hf-internal-testing/test_dynamic_image_processor'''				,   trust_remote_code=__UpperCamelCase   )
											self.assertEqual(image_processor.__class__.__name__				,   '''NewImageProcessor'''   )
											self.assertTrue(image_processor.is_local   )

											# If remote is enabled, we load from the Hub
											_UpperCAmelCase					=    AutoImageProcessor.from_pretrained(
											    '''hf-internal-testing/test_dynamic_image_processor'''				,   trust_remote_code=__UpperCamelCase   )
											self.assertEqual(image_processor.__class__.__name__				,   '''NewImageProcessor'''   )
											self.assertTrue(not hasattr(__UpperCamelCase				,   '''is_local'''   )   )

							finally:
											if "custom" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
															del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]








"""simple docstring"""


from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml
class 			_a (					yaml.SafeLoader):
			"""simple docstring"""



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Any   )->List[Any]:
							_UpperCAmelCase					=    [self.constructed_objects[key_node] for key_node, _ in node.value]
							_UpperCAmelCase					=    [tuple(__UpperCamelCase   ) if isinstance(__UpperCamelCase				,   __UpperCamelCase   ) else key for key in keys]
							_UpperCAmelCase					=    Counter(__UpperCamelCase   )
							_UpperCAmelCase					=    [key for key in counter if counter[key] > 1]
							if duplicate_keys:
											raise TypeError(F'Got duplicate yaml keys: {duplicate_keys}'   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[Any]				,   __UpperCamelCase     :    str=False   )->Dict:
							_UpperCAmelCase					=    super().construct_mapping(__UpperCamelCase				,   deep=__UpperCamelCase   )
							self._check_no_duplicates_on_constructed_node(__UpperCamelCase   )
							return mapping







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						str				):






				'''simple docstring'''



				_UpperCAmelCase					=    list(readme_content.splitlines()				)
				if full_content and full_content[0] == "---" and "---" in full_content[1:]:
								_UpperCAmelCase					=    full_content[1:].index('''---'''				) + 1
								_UpperCAmelCase					=    '''\n'''.join(full_content[1:sep_idx]				)
								return yamlblock, "\n".join(full_content[sep_idx + 1 :]				)

				return None, "\n".join(_SCREAMING_SNAKE_CASE				)
class 			_a (					lowerCAmelCase):
			"""simple docstring"""
			# class attributes
			UpperCamelCase__			      =	{"""train_eval_index"""}  # train-eval-index in the YAML metadata



			@classmethod
			def  lowercase__						(		cls     :    List[Any]				,   __UpperCamelCase     :    Path   )->"DatasetMetadata":
							with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(readme_file.read()   )
							if yaml_string is not None:
											return cls.from_yaml_string(__UpperCamelCase   )
							else:
											return cls()



			def  lowercase__						(		self     :    Tuple				,   __UpperCamelCase     :    Path   )->List[Any]:
							if path.exists():
											with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
															_UpperCAmelCase					=    readme_file.read()
							else:
											_UpperCAmelCase					=    None
							_UpperCAmelCase					=    self._to_readme(__UpperCamelCase   )
							with open(__UpperCamelCase				,   '''w'''				,   encoding='''utf-8'''   ) as readme_file:
											readme_file.write(__UpperCamelCase   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[str] = None   )->str:
							if readme_content is not None:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(__UpperCamelCase   )
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n''' + content
							else:
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n'''
							return full_content



			@classmethod
			def  lowercase__						(		cls     :    str				,   __UpperCamelCase     :    str   )->"DatasetMetadata":
							_UpperCAmelCase					=    yaml.load(__UpperCamelCase				,   Loader=_NoDuplicateSafeLoader   ) or {}

							# Convert the YAML keys to DatasetMetadata fields
							_UpperCAmelCase					=    {
							    (key.replace('''-'''				,   '''_'''   ) if key.replace('''-'''				,   '''_'''   ) in cls._FIELDS_WITH_DASHES else key): value
							    for key, value in metadata_dict.items()
							}
							return cls(**__UpperCamelCase   )



			def  lowercase__						(		self     :    str   )->str:
							return yaml.safe_dump(
							    {
							        (key.replace('''_'''				,   '''-'''   ) if key in self._FIELDS_WITH_DASHES else key): value
							        for key, value in self.items()
							    }				,   sort_keys=__UpperCamelCase				,   allow_unicode=__UpperCamelCase				,   encoding='''utf-8'''				,   ).decode('''utf-8'''   )



__A   :							str										=   {
    "image-classification": [],
    "translation": [],
    "image-segmentation": [],
    "fill-mask": [],
    "automatic-speech-recognition": [],
    "token-classification": [],
    "sentence-similarity": [],
    "audio-classification": [],
    "question-answering": [],
    "summarization": [],
    "zero-shot-classification": [],
    "table-to-text": [],
    "feature-extraction": [],
    "other": [],
    "multiple-choice": [],
    "text-classification": [],
    "text-to-image": [],
    "text2text-generation": [],
    "zero-shot-image-classification": [],
    "tabular-classification": [],
    "tabular-regression": [],
    "image-to-image": [],
    "tabular-to-text": [],
    "unconditional-image-generation": [],
    "text-retrieval": [],
    "text-to-speech": [],
    "object-detection": [],
    "audio-to-audio": [],
    "text-generation": [],
    "conversational": [],
    "table-question-answering": [],
    "visual-question-answering": [],
    "image-to-text": [],
    "reinforcement-learning": [],
    "voice-activity-detection": [],
    "time-series-forecasting": [],
    "document-question-answering": [],
}


if __name__ == "__main__":
					from argparse import ArgumentParser

					__A   :							str										=   ArgumentParser(usage="Validate the yaml metadata block of a README.md file.")
					ap.add_argument("readme_filepath")
					__A   :							Union[str, Any]										=   ap.parse_args()

					__A   :							Dict										=   Path(args.readme_filepath)
					__A   :							Tuple										=   DatasetMetadata.from_readme(readme_filepath)
					print(dataset_metadata)
					dataset_metadata.to_readme(readme_filepath)



