





import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
     from transformers import WhisperFeatureExtractor

if is_torch_available():
     import torch

__lowercase	: Optional[Any]       =random.Random()




def      a__			(       lowercase__	,     lowercase__=1.0	,     lowercase__=None	,     lowercase__=None	):






   '''simple docstring'''






   if rng is None:
      UpperCAmelCase_           =global_rng

   UpperCAmelCase_           =[]
   for batch_idx in range(shape[0]	):
      values.append([]	)
      for _ in range(shape[1]	):
         values[-1].append(rng.random() * scale	)

   return values

@require_torch
@require_torchaudio
class   A							(						unittest.TestCase    ):



      def __init__(						self:    Optional[int]		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    str=7		,	_lowerCAmelCase:    Optional[int]=400		,	_lowerCAmelCase:    Optional[Any]=2000		,	_lowerCAmelCase:    Union[str, Any]=10		,	_lowerCAmelCase:    Optional[int]=160		,	_lowerCAmelCase:    Any=8		,	_lowerCAmelCase:    str=0.0		,	_lowerCAmelCase:    Optional[int]=4000		,	_lowerCAmelCase:    Dict=False		,	_lowerCAmelCase:    Optional[Any]=True		,	)    ->     Dict:
         '''simple docstring'''

         UpperCAmelCase_           =parent
         UpperCAmelCase_           =batch_size
         UpperCAmelCase_           =min_seq_length
         UpperCAmelCase_           =max_seq_length
         UpperCAmelCase_           =(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
         UpperCAmelCase_           =padding_value
         UpperCAmelCase_           =sampling_rate
         UpperCAmelCase_           =return_attention_mask
         UpperCAmelCase_           =do_normalize
         UpperCAmelCase_           =feature_size
         UpperCAmelCase_           =chunk_length
         UpperCAmelCase_           =hop_length



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     int:
         '''simple docstring'''

         return {
             "feature_size": self.feature_size,
             "hop_length": self.hop_length,
             "chunk_length": self.chunk_length,
             "padding_value": self.padding_value,
             "sampling_rate": self.sampling_rate,
             "return_attention_mask": self.return_attention_mask,
             "do_normalize": self.do_normalize,
         }






      def 						lowerCAmelCase__     (						self:    List[Any]		,	_lowerCAmelCase:    List[str]=False		,	_lowerCAmelCase:    Dict=False							)    ->     str:
         '''simple docstring'''

         def _flatten(_lowerCAmelCase:    List[str]							):
            return list(itertools.chain(*_lowerCAmelCase							)							)

         if equal_length:
            UpperCAmelCase_           =[floats_list((self.max_seq_length, self.feature_size)							) for _ in range(self.batch_size							)]
         else:
            # make sure that inputs increase in size
            UpperCAmelCase_           =[
                floats_list((x, self.feature_size)							)
                for x in range(self.min_seq_length		,	self.max_seq_length		,	self.seq_length_diff							)
            ]
         if numpify:
            UpperCAmelCase_           =[np.asarray(_lowerCAmelCase							) for x in speech_inputs]
         return speech_inputs

@require_torch
@require_torchaudio
class   A							(						__lowercase		,   unittest.TestCase    ):
      _snake_case				  =WhisperFeatureExtractor if is_speech_available() else None



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_           =WhisperFeatureExtractionTester(self							)



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =self.feature_extraction_class(**self.feat_extract_dict							)

         with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_           =feat_extract_first.save_pretrained(_lowerCAmelCase							)[0]
            check_json_file_has_correct_format(_lowerCAmelCase							)
            UpperCAmelCase_           =self.feature_extraction_class.from_pretrained(_lowerCAmelCase							)

         UpperCAmelCase_           =feat_extract_first.to_dict()
         UpperCAmelCase_           =feat_extract_second.to_dict()
         UpperCAmelCase_           =feat_extract_first.mel_filters
         UpperCAmelCase_           =feat_extract_second.mel_filters
         self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase							)							)
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     Dict:
         '''simple docstring'''

         UpperCAmelCase_           =self.feature_extraction_class(**self.feat_extract_dict							)

         with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_           =os.path.join(_lowerCAmelCase		,	"feat_extract.json"							)
            feat_extract_first.to_json_file(_lowerCAmelCase							)
            UpperCAmelCase_           =self.feature_extraction_class.from_json_file(_lowerCAmelCase							)

         UpperCAmelCase_           =feat_extract_first.to_dict()
         UpperCAmelCase_           =feat_extract_second.to_dict()
         UpperCAmelCase_           =feat_extract_first.mel_filters
         UpperCAmelCase_           =feat_extract_second.mel_filters
         self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase							)							)
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
         # create three inputs of length 800, 1000, and 1200
         UpperCAmelCase_           =[floats_list((1, x)							)[0] for x in range(800		,	1400		,	200							)]
         UpperCAmelCase_           =[np.asarray(_lowerCAmelCase							) for speech_input in speech_inputs]

         # Test feature size
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	padding="max_length"		,	return_tensors="np"							).input_features
         self.assertTrue(input_features.ndim == 3							)
         self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames							)
         self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size							)

         # Test not batched input
         UpperCAmelCase_           =feature_extractor(speech_inputs[0]		,	return_tensors="np"							).input_features
         UpperCAmelCase_           =feature_extractor(np_speech_inputs[0]		,	return_tensors="np"							).input_features
         self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase		,	atol=1e-3							)							)

         # Test batched
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         for enc_seq_a, enc_seq_a in zip(_lowerCAmelCase		,	_lowerCAmelCase							):
            self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase		,	atol=1e-3							)							)

         # Test 2-D numpy arrays are batched.
         UpperCAmelCase_           =[floats_list((1, x)							)[0] for x in (800, 800, 800)]
         UpperCAmelCase_           =np.asarray(_lowerCAmelCase							)
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         for enc_seq_a, enc_seq_a in zip(_lowerCAmelCase		,	_lowerCAmelCase							):
            self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase		,	atol=1e-3							)							)

         # Test truncation required
         UpperCAmelCase_           =[floats_list((1, x)							)[0] for x in range(200		,	(feature_extractor.n_samples + 500)		,	200							)]
         UpperCAmelCase_           =[np.asarray(_lowerCAmelCase							) for speech_input in speech_inputs]

         UpperCAmelCase_           =[x[: feature_extractor.n_samples] for x in speech_inputs]
         UpperCAmelCase_           =[np.asarray(_lowerCAmelCase							) for speech_input in speech_inputs_truncated]

         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="np"							).input_features
         for enc_seq_a, enc_seq_a in zip(_lowerCAmelCase		,	_lowerCAmelCase							):
            self.assertTrue(np.allclose(_lowerCAmelCase		,	_lowerCAmelCase		,	atol=1e-3							)							)



      def 						lowerCAmelCase__     (						self:    str							)    ->     Tuple:
         '''simple docstring'''

         import torch

         UpperCAmelCase_           =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
         UpperCAmelCase_           =np.random.rand(100		,	32							).astype(np.floataa							)
         UpperCAmelCase_           =np_speech_inputs.tolist()

         for inputs in [py_speech_inputs, np_speech_inputs]:
            UpperCAmelCase_           =feature_extractor.pad([{"input_features": inputs}]		,	return_tensors="np"							)
            self.assertTrue(np_processed.input_features.dtype == np.floataa							)
            UpperCAmelCase_           =feature_extractor.pad([{"input_features": inputs}]		,	return_tensors="pt"							)
            self.assertTrue(pt_processed.input_features.dtype == torch.floataa							)



      def 						lowerCAmelCase__     (						self:    int		,	_lowerCAmelCase:    Optional[int]							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =load_dataset("hf-internal-testing/librispeech_asr_dummy"		,	"clean"		,	split="validation"							)
         # automatic decoding with librispeech
         UpperCAmelCase_           =ds.sort("id"							).select(range(_lowerCAmelCase							)							)[:num_samples]["audio"]

         return [x["array"] for x in speech_samples]



      def 						lowerCAmelCase__     (						self:    str							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =torch.tensor(
             [
                 0.11_93, -0.09_46, -0.10_98, -0.01_96, 0.02_25, -0.06_90, -0.17_36, 0.09_51,
                 0.09_71, -0.08_17, -0.07_02, 0.01_62, 0.02_60, 0.00_17, -0.01_92, -0.16_78,
                 0.07_09, -0.18_67, -0.06_55, -0.02_74, -0.02_34, -0.18_84, -0.05_16, -0.05_54,
                 -0.02_74, -0.14_25, -0.14_23, 0.08_37, 0.03_77, -0.08_54
             ]							)
         # fmt: on

         UpperCAmelCase_           =self._load_datasamples(1							)
         UpperCAmelCase_           =WhisperFeatureExtractor()
         UpperCAmelCase_           =feature_extractor(_lowerCAmelCase		,	return_tensors="pt"							).input_features
         self.assertEqual(input_features.shape		,	(1, 80, 3000)							)
         self.assertTrue(torch.allclose(input_features[0, 0, :30]		,	_lowerCAmelCase		,	atol=1e-4							)							)






      def 						lowerCAmelCase__     (						self:    str							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
         UpperCAmelCase_           =self._load_datasamples(1							)[0]
         UpperCAmelCase_           =((audio - audio.min()) / (audio.max() - audio.min())) * 6_5535  # Rescale to [0, 65535] to show issue
         UpperCAmelCase_           =feat_extract.zero_mean_unit_var_norm([audio]		,	attention_mask=_lowerCAmelCase							)[0]

         self.assertTrue(np.all(np.mean(_lowerCAmelCase							) < 1e-3							)							)
         self.assertTrue(np.all(np.abs(np.var(_lowerCAmelCase							) - 1							) < 1e-3							)							)









"""simple docstring"""






from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml



class     UpperCAmelCase    (	yaml.SafeLoader  ):



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      [self.constructed_objects[key_node] for key_node, _ in node.value]
     _snake_case   	=      [tuple(__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else key for key in keys]
     _snake_case   	=      Counter(__lowerCamelCase				)
     _snake_case   	=      [key for key in counter if counter[key] > 1]
     if duplicate_keys:
       raise TypeError(f"""Got duplicate yaml keys: {duplicate_keys}"""				)





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	Optional[Any]=False				):



     """simple docstring"""
     _snake_case   	=      super().construct_mapping(__lowerCamelCase							,       deep=__lowerCamelCase				)
     self._check_no_duplicates_on_constructed_node(__lowerCamelCase				)
     return mapping




def 			snake_case		(		lowerCAmelCase_	)						->				Tuple[Optional[str], str]:
  _snake_case   	=      list(readme_content.splitlines()	)
  if full_content and full_content[0] == "---" and "---" in full_content[1:]:
    _snake_case   	=      full_content[1:].index('''---'''	) + 1
    _snake_case   	=      '''\n'''.join(full_content[1:sep_idx]	)
    return yamlblock, "\n".join(full_content[sep_idx + 1 :]	)

  return None, "\n".join(lowerCAmelCase_	)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   # class attributes
   A__		:	int           =  {'''train_eval_index'''}  # train-eval-index in the YAML metadata



   @classmethod
   def    __UpperCAmelCase    (       cls    :	str							,       __lowerCamelCase    :	Path				):



     """simple docstring"""
     with open(__lowerCamelCase							,       encoding='''utf-8'''				) as readme_file:
       _snake_case	,     _snake_case   	=      _split_yaml_from_readme(readme_file.read()				)
     if yaml_string is not None:
       return cls.from_yaml_string(__lowerCamelCase				)
     else:
       return cls()



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	Path				):



     """simple docstring"""
     if path.exists():
       with open(__lowerCamelCase							,       encoding='''utf-8'''				) as readme_file:
         _snake_case   	=      readme_file.read()
     else:
       _snake_case   	=      None
     _snake_case   	=      self._to_readme(__lowerCamelCase				)
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as readme_file:
       readme_file.write(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if readme_content is not None:
       _snake_case	,     _snake_case   	=      _split_yaml_from_readme(__lowerCamelCase				)
       _snake_case   	=      '''---\n''' + self.to_yaml_string() + '''---\n''' + content
     else:
       _snake_case   	=      '''---\n''' + self.to_yaml_string() + '''---\n'''
     return full_content



   @classmethod
   def    __UpperCAmelCase    (       cls    :	Tuple							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      yaml.load(__lowerCamelCase							,       Loader=_NoDuplicateSafeLoader				) or {}

     # Convert the YAML keys to DatasetMetadata fields
     _snake_case   	=      {
         (key.replace('''-'''							,       '''_'''				) if key.replace('''-'''							,       '''_'''				) in cls._FIELDS_WITH_DASHES else key): value
         for key, value in metadata_dict.items()
     }
     return cls(**__lowerCamelCase				)





   def    __UpperCAmelCase    (       self    :	Dict				):



     """simple docstring"""
     return yaml.safe_dump(
         {
             (key.replace('''_'''							,       '''-'''				) if key in self._FIELDS_WITH_DASHES else key): value
             for key, value in self.items()
         }							,       sort_keys=__lowerCamelCase							,       allow_unicode=__lowerCamelCase							,       encoding='''utf-8'''							,       ).decode('''utf-8'''				)

snake_case   					=				{
    '''image-classification''': [],
    '''translation''': [],
    '''image-segmentation''': [],
    '''fill-mask''': [],
    '''automatic-speech-recognition''': [],
    '''token-classification''': [],
    '''sentence-similarity''': [],
    '''audio-classification''': [],
    '''question-answering''': [],
    '''summarization''': [],
    '''zero-shot-classification''': [],
    '''table-to-text''': [],
    '''feature-extraction''': [],
    '''other''': [],
    '''multiple-choice''': [],
    '''text-classification''': [],
    '''text-to-image''': [],
    '''text2text-generation''': [],
    '''zero-shot-image-classification''': [],
    '''tabular-classification''': [],
    '''tabular-regression''': [],
    '''image-to-image''': [],
    '''tabular-to-text''': [],
    '''unconditional-image-generation''': [],
    '''text-retrieval''': [],
    '''text-to-speech''': [],
    '''object-detection''': [],
    '''audio-to-audio''': [],
    '''text-generation''': [],
    '''conversational''': [],
    '''table-question-answering''': [],
    '''visual-question-answering''': [],
    '''image-to-text''': [],
    '''reinforcement-learning''': [],
    '''voice-activity-detection''': [],
    '''time-series-forecasting''': [],
    '''document-question-answering''': [],
}


if __name__ == "__main__":
     from argparse import ArgumentParser

     snake_case   					=				ArgumentParser(usage='''Validate the yaml metadata block of a README.md file.''')
     ap.add_argument('''readme_filepath''')
     snake_case   					=				ap.parse_args()

     snake_case   					=				Path(args.readme_filepath)
     snake_case   					=				DatasetMetadata.from_readme(readme_filepath)
     print(dataset_metadata)
     dataset_metadata.to_readme(readme_filepath)





