
'''simple docstring'''





from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml






class 							UpperCAmelCase_							(    yaml.SafeLoader				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  List[str]			,				snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[Any]				       =							[self.constructed_objects[key_node] for key_node, _ in node.value]
   UpperCAmelCase__      :					Tuple				       =							[tuple(snake_case__ ) if isinstance(snake_case__			,				snake_case__ ) else key for key in keys]
   UpperCAmelCase__      :					Dict				       =							Counter(snake_case__ )
   UpperCAmelCase__      :					int				       =							[key for key in counter if counter[key] > 1]
   if duplicate_keys:
     raise TypeError(F"""Got duplicate yaml keys: {duplicate_keys}""" )







 def 							UpperCamelCase  (					self		:  int			,				snake_case__		:  Tuple			,				snake_case__		:  Any=False ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							super().construct_mapping(snake_case__			,				deep=snake_case__ )
   self._check_no_duplicates_on_constructed_node(snake_case__ )
   return mapping







def snake_case_     (					lowercase__		):
  UpperCAmelCase__      :					Optional[Any]				       =							list(readme_content.splitlines()		)
  if full_content and full_content[0] == "---" and "---" in full_content[1:]:
    UpperCAmelCase__      :					Dict				       =							full_content[1:].index("---"		) + 1
    UpperCAmelCase__      :					Optional[int]				       =							"\n".join(full_content[1:sep_idx]		)
    return yamlblock, "\n".join(full_content[sep_idx + 1 :]		)

  return None, "\n".join(lowercase__		)






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 # class attributes
 lowercase_					:    Union[str, Any]				=							{"train_eval_index"}  # train-eval-index in the YAML metadata







 @classmethod
 def 							UpperCamelCase  (					cls		:  Optional[Any]			,				snake_case__		:  Path ):



   '''simple docstring'''




   with open(snake_case__			,				encoding="utf-8" ) as readme_file:
     UpperCAmelCase__       ,			UpperCAmelCase__      :					str				       =							_split_yaml_from_readme(readme_file.read() )
   if yaml_string is not None:
     return cls.from_yaml_string(snake_case__ )
   else:
     return cls()







 def 							UpperCamelCase  (					self		:  Any			,				snake_case__		:  Path ):



   '''simple docstring'''




   if path.exists():
     with open(snake_case__			,				encoding="utf-8" ) as readme_file:
       UpperCAmelCase__      :					Optional[int]				       =							readme_file.read()
   else:
     UpperCAmelCase__      :					List[str]				       =							None
   UpperCAmelCase__      :					Optional[int]				       =							self._to_readme(snake_case__ )
   with open(snake_case__			,				"w"			,				encoding="utf-8" ) as readme_file:
     readme_file.write(snake_case__ )







 def 							UpperCamelCase  (					self		:  List[Any]			,				snake_case__		:  Optional[str] = None ):



   '''simple docstring'''




   if readme_content is not None:
     UpperCAmelCase__       ,			UpperCAmelCase__      :					Optional[int]				       =							_split_yaml_from_readme(snake_case__ )
     UpperCAmelCase__      :					Optional[int]				       =							"---\n" + self.to_yaml_string() + "---\n" + content
   else:
     UpperCAmelCase__      :					int				       =							"---\n" + self.to_yaml_string() + "---\n"
   return full_content







 @classmethod
 def 							UpperCamelCase  (					cls		:  Any			,				snake_case__		:  str ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							yaml.load(snake_case__			,				Loader=_NoDuplicateSafeLoader ) or {}

   # Convert the YAML keys to DatasetMetadata fields
   UpperCAmelCase__      :					str				       =							{
       (key.replace("-"			,				"_" ) if key.replace("-"			,				"_" ) in cls._FIELDS_WITH_DASHES else key): value
       for key, value in metadata_dict.items()
   }
   return cls(**snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   return yaml.safe_dump(
       {
           (key.replace("_"			,				"-" ) if key in self._FIELDS_WITH_DASHES else key): value
           for key, value in self.items()
       }			,				sort_keys=snake_case__			,				allow_unicode=snake_case__			,				encoding="utf-8"			,				).decode("utf-8" )



SCREAMING_SNAKE_CASE     =       {
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
      from argparse import ArgumentParser

      SCREAMING_SNAKE_CASE     =       ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
      ap.add_argument("""readme_filepath""")
      SCREAMING_SNAKE_CASE     =       ap.parse_args()

      SCREAMING_SNAKE_CASE     =       Path(args.readme_filepath)
      SCREAMING_SNAKE_CASE     =       DatasetMetadata.from_readme(readme_filepath)
      print(dataset_metadata)
      dataset_metadata.to_readme(readme_filepath)





'''simple docstring'''





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures""")
SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures/dummy-config.json""")






class 							UpperCAmelCase_							(    unittest.TestCase				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							0







 def 							UpperCamelCase  (					self		:  Union[str, Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h" )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   with tempfile.TemporaryDirectory() as tmpdirname:
     UpperCAmelCase__      :					Optional[Any]				       =							WavaVecaConfig()

     # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__ ).to_dict()

     config_dict.pop("feature_extractor_type" )
     UpperCAmelCase__      :					Union[str, Any]				       =							WavaVecaFeatureExtractor(**snake_case__ )

     # save in new folder
     model_config.save_pretrained(snake_case__ )
     config.save_pretrained(snake_case__ )

     UpperCAmelCase__      :					Tuple				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )

     # make sure private variable is not incorrectly saved
     UpperCAmelCase__      :					List[Any]				       =							json.loads(config.to_json_string() )
     self.assertTrue("_processor_class" not in dict_as_saved )

   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				"bert-base is not a local folder and is not a valid model identifier" ):
     UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained("bert-base" )







 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)" ):
     UpperCAmelCase__      :					List[str]				       =							AutoFeatureExtractor.from_pretrained(snake_case__			,				revision="aaaaaa" )







 def 							UpperCamelCase  (					self		:  Union[str, Any] ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				"hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."			,				):
     UpperCAmelCase__      :					Optional[Any]				       =							AutoFeatureExtractor.from_pretrained("hf-internal-testing/config-no-model" )







 def 							UpperCamelCase  (					self		:  Optional[Any] ):



   '''simple docstring'''




   with self.assertRaises(snake_case__ ):
     UpperCAmelCase__      :					List[str]				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor" )
   # If remote code is disabled, we can't load this config.
   with self.assertRaises(snake_case__ ):
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )

   UpperCAmelCase__      :					Optional[Any]				       =							AutoFeatureExtractor.from_pretrained(
       "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
   self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )

   # Test feature extractor can be reloaded.
   with tempfile.TemporaryDirectory() as tmp_dir:
     feature_extractor.save_pretrained(snake_case__ )
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__			,				trust_remote_code=snake_case__ )
   self.assertEqual(reloaded_feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )







 def 							UpperCamelCase  (					self		:  str ):



   '''simple docstring'''




   try:
     AutoConfig.register("custom"			,				snake_case__ )
     AutoFeatureExtractor.register(snake_case__			,				snake_case__ )
     # Trying to register something existing in the Transformers library will raise an error
     with self.assertRaises(snake_case__ ):
       AutoFeatureExtractor.register(snake_case__			,				snake_case__ )

     # Now that the config is registered, it can be used as any other config with the auto-API
     UpperCAmelCase__      :					str				       =							CustomFeatureExtractor.from_pretrained(snake_case__ )
     with tempfile.TemporaryDirectory() as tmp_dir:
       feature_extractor.save_pretrained(snake_case__ )
       UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
       self.assertIsInstance(snake_case__			,				snake_case__ )

   finally:
     if "custom" in CONFIG_MAPPING._extra_content:
       del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
       del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







 def 							UpperCamelCase  (					self		:  str ):



   '''simple docstring'''




   class 							UpperCAmelCase_							(    A				):

    '''simple docstring'''
    lowercase_					:    Optional[Any]				=							True

   try:
     AutoConfig.register("custom"			,				snake_case__ )
     AutoFeatureExtractor.register(snake_case__			,				snake_case__ )
     # If remote code is not set, the default is to use local
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor" )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(feature_extractor.is_local )

     # If remote code is disabled, we load the local one.
     UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(feature_extractor.is_local )

     # If remote is enabled, we load from the Hub
     UpperCAmelCase__      :					int				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(not hasattr(snake_case__			,				"is_local" ) )

   finally:
     if "custom" in CONFIG_MAPPING._extra_content:
       del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
       del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]



