'''simple docstring'''





import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





def       lowerCAmelCase_							(   snake_case_			:					Features     )       -> Optional[int]:


     '''simple docstring'''




     UpperCAmelCase_			       =  np.inf

     def set_batch_size(snake_case_			:					FeatureType     ) -> None:
          nonlocal batch_size
          if isinstance(snake_case_   ,       snake_case_     ):
               UpperCAmelCase_			       =  min(snake_case_   ,       config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS     )
          elif isinstance(snake_case_   ,       snake_case_     ):
               UpperCAmelCase_			       =  min(snake_case_   ,       config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS     )
          elif isinstance(snake_case_   ,       snake_case_     ) and feature.dtype == "binary":
               UpperCAmelCase_			       =  min(snake_case_   ,       config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS     )

     _visit(snake_case_   ,       snake_case_     )

     return None if batch_size is np.inf else batch_size




class 			__A (					UpperCamelCase__      ):



    def __init__(self       :       Tuple  ,     __a       :       NestedDataStructureLike[PathLike]  ,     __a       :       Optional[NamedSplit] = None  ,     __a       :       Optional[Features] = None  ,     __a       :       str = None  ,     __a       :       bool = False  ,     __a       :       bool = False  ,     __a       :       Optional[int] = None  ,     **__a       :       Dict  ,     ):
         super().__init__(
             __a  ,     split=__a  ,     features=__a  ,     cache_dir=__a  ,     keep_in_memory=__a  ,     streaming=__a  ,     num_proc=__a  ,     **__a  ,     )
         UpperCAmelCase_			       =  path_or_paths if isinstance(__a  ,     __a					) else {self.split: path_or_paths}
         UpperCAmelCase_			       =  _PACKAGED_DATASETS_MODULES["parquet"][1]
         UpperCAmelCase_			       =  Parquet(
             cache_dir=__a  ,     data_files=__a  ,     features=__a  ,     hash=__a  ,     **__a  ,     )






    def    _lowercase		(self       :       str					):
         # Build iterable dataset
         if self.streaming:
              UpperCAmelCase_			       =  self.builder.as_streaming_dataset(split=self.split					)
         # Build regular (map-style) dataset
         else:
              UpperCAmelCase_			       =  None
              UpperCAmelCase_			       =  None
              UpperCAmelCase_			       =  None
              UpperCAmelCase_			       =  None

              self.builder.download_and_prepare(
                  download_config=__a  ,     download_mode=__a  ,     verification_mode=__a  ,     base_path=__a  ,     num_proc=self.num_proc  ,     )
              UpperCAmelCase_			       =  self.builder.as_dataset(
                  split=self.split  ,     verification_mode=__a  ,     in_memory=self.keep_in_memory					)
         return dataset




class 			__A :



    def __init__(self       :       List[Any]  ,     __a       :       Dataset  ,     __a       :       Union[PathLike, BinaryIO]  ,     __a       :       Optional[int] = None  ,     **__a       :       Optional[int]  ,     ):
         UpperCAmelCase_			       =  dataset
         UpperCAmelCase_			       =  path_or_buf
         UpperCAmelCase_			       =  batch_size or get_writer_batch_size(dataset.features					)
         UpperCAmelCase_			       =  parquet_writer_kwargs



    def    _lowercase		(self       :       str					):
         UpperCAmelCase_			       =  self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

         if isinstance(self.path_or_buf  ,     (str, bytes, os.PathLike)					):
              with open(self.path_or_buf  ,     "wb+"					) as buffer:
                   UpperCAmelCase_			       =  self._write(file_obj=__a  ,     batch_size=__a  ,     **self.parquet_writer_kwargs					)
         else:
              UpperCAmelCase_			       =  self._write(file_obj=self.path_or_buf  ,     batch_size=__a  ,     **self.parquet_writer_kwargs					)
         return written






    def    _lowercase		(self       :       int  ,     __a       :       BinaryIO  ,     __a       :       int  ,     **__a       :       Union[str, Any]					):
         UpperCAmelCase_			       =  0
         UpperCAmelCase_			       =  parquet_writer_kwargs.pop("path_or_buf"  ,     __a					)
         UpperCAmelCase_			       =  self.dataset.features.arrow_schema

         UpperCAmelCase_			       =  pq.ParquetWriter(__a  ,     schema=__a  ,     **__a					)

         for offset in logging.tqdm(
             range(0  ,     len(self.dataset					)  ,     __a					)  ,     unit="ba"  ,     disable=not logging.is_progress_bar_enabled()  ,     desc="Creating parquet from Arrow format"  ,     ):
              UpperCAmelCase_			       =  query_table(
                  table=self.dataset._data  ,     key=slice(__a  ,     offset + batch_size					)  ,     indices=self.dataset._indices if self.dataset._indices is not None else None  ,     )
              writer.write_table(__a					)
              written += batch.nbytes
         writer.close()
         return written





'''simple docstring'''




import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


__lowercase        =    logging.get_logger(__name__)






class 							a__(   lowerCAmelCase__      ):


			'''simple docstring'''






			UpperCAmelCase_     :	Dict						=			['''input_values''', '''attention_mask''']



			def __init__(  self			,       __lowerCAmelCase = 1			,       __lowerCAmelCase = 16000			,       __lowerCAmelCase = 0.0			,       __lowerCAmelCase = False			,       __lowerCAmelCase = 80			,       __lowerCAmelCase = 16			,       __lowerCAmelCase = 64			,       __lowerCAmelCase = "hann_window"			,       __lowerCAmelCase = 1.0			,       __lowerCAmelCase = 80			,       __lowerCAmelCase = 7600			,       __lowerCAmelCase = 1E-1_0			,       __lowerCAmelCase = 2			,       __lowerCAmelCase = True			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					super().__init__(feature_size=__lowerCAmelCase			,       sampling_rate=__lowerCAmelCase			,       padding_value=__lowerCAmelCase			,       **__lowerCAmelCase)
					lowerCAmelCase     =    do_normalize
					lowerCAmelCase     =    return_attention_mask

					lowerCAmelCase     =    num_mel_bins
					lowerCAmelCase     =    hop_length
					lowerCAmelCase     =    win_length
					lowerCAmelCase     =    win_function
					lowerCAmelCase     =    frame_signal_scale
					lowerCAmelCase     =    fmin
					lowerCAmelCase     =    fmax
					lowerCAmelCase     =    mel_floor
					lowerCAmelCase     =    reduction_factor

					lowerCAmelCase     =    win_length * sampling_rate // 1000
					lowerCAmelCase     =    hop_length * sampling_rate // 1000
					lowerCAmelCase     =    optimal_fft_length(self.sample_size)
					lowerCAmelCase     =    (self.n_fft // 2) + 1

					lowerCAmelCase     =    window_function(window_length=self.sample_size			,       name=self.win_function			,       periodic=__lowerCAmelCase)

					lowerCAmelCase     =    mel_filter_bank(
					    num_frequency_bins=self.n_freqs			,       num_mel_filters=self.num_mel_bins			,       min_frequency=self.fmin			,       max_frequency=self.fmax			,       sampling_rate=self.sampling_rate			,       norm="""slaney"""			,       mel_scale="""slaney"""			,       )

					if frame_signal_scale != 1.0:
							warnings.warn(
							    """The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers"""			,       __lowerCAmelCase			,       )
					if reduction_factor != 2.0:
							warnings.warn(
							    """The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers"""			,       __lowerCAmelCase			,       )



			@staticmethod
			# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
			def 				a_			(  __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase = 0.0):




					"""simple docstring"""


					if attention_mask is not None:
							lowerCAmelCase     =    np.array(__lowerCAmelCase			,       np.intaa)
							lowerCAmelCase     =    []

							for vector, length in zip(__lowerCAmelCase			,       attention_mask.sum(-1)):
									lowerCAmelCase     =    (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7)
									if length < normed_slice.shape[0]:
											lowerCAmelCase     =    padding_value

									normed_input_values.append(__lowerCAmelCase)
					else:
							lowerCAmelCase     =    [(x - x.mean()) / np.sqrt(x.var() + 1E-7) for x in input_values]

					return normed_input_values



			def 				a_			(  self			,       __lowerCAmelCase			,       ):




					"""simple docstring"""


					lowerCAmelCase     =    spectrogram(
					    __lowerCAmelCase			,       window=self.window			,       frame_length=self.sample_size			,       hop_length=self.sample_stride			,       fft_length=self.n_fft			,       mel_filters=self.mel_filters			,       mel_floor=self.mel_floor			,       log_mel="""log10"""			,       )
					return log_mel_spec.T



			def __call__(  self			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = False			,       __lowerCAmelCase = None			,       __lowerCAmelCase = False			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					if audio is None and audio_target is None:
							raise ValueError("""You must provide either `audio` or `audio_target` values.""")

					if sampling_rate is not None:
							if sampling_rate != self.sampling_rate:
									raise ValueError(
									    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
									    f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
									    f" {self.sampling_rate} and not {sampling_rate}.")
					else:
							logger.warning(
							    """It is strongly recommended to pass the ``sampling_rate`` argument to this function. """
							    """Failing to do so can result in silent errors that might be hard to debug.""")

					if audio is not None:
							lowerCAmelCase     =    self._process_audio(
							    __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       **__lowerCAmelCase			,       )
					else:
							lowerCAmelCase     =    None

					if audio_target is not None:
							lowerCAmelCase     =    self._process_audio(
							    __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       **__lowerCAmelCase			,       )

							if inputs is None:
									return inputs_target
							else:
									lowerCAmelCase     =    inputs_target["""input_values"""]
									lowerCAmelCase     =    inputs_target.get("""attention_mask""")
									if decoder_attention_mask is not None:
											lowerCAmelCase     =    decoder_attention_mask

					return inputs



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = False			,       __lowerCAmelCase = False			,       __lowerCAmelCase = None			,       __lowerCAmelCase = False			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					lowerCAmelCase     =    isinstance(__lowerCAmelCase			,       np.ndarray) and len(speech.shape) > 1
					if is_batched_numpy and len(speech.shape) > 2:
							raise ValueError(f"Only mono-channel audio is supported for input to {self}")
					lowerCAmelCase     =    is_batched_numpy or (
					    isinstance(__lowerCAmelCase			,       (list, tuple)) and (isinstance(speech[0]			,       (np.ndarray, tuple, list)))
					)

					if is_batched:
							lowerCAmelCase     =    [np.asarray(__lowerCAmelCase			,       dtype=np.floataa) for speech in speech]
					elif not is_batched and not isinstance(__lowerCAmelCase			,       np.ndarray):
							lowerCAmelCase     =    np.asarray(__lowerCAmelCase			,       dtype=np.floataa)
					elif isinstance(__lowerCAmelCase			,       np.ndarray) and speech.dtype is np.dtype(np.floataa):
							lowerCAmelCase     =    speech.astype(np.floataa)

					# always return batch
					if not is_batched:
							lowerCAmelCase     =    [speech]

					# needed to make pad() work on spectrogram inputs
					lowerCAmelCase     =    self.feature_size

					# convert into correct format for padding
					if is_target:
							lowerCAmelCase     =    [self._extract_mel_features(__lowerCAmelCase) for waveform in speech]
							lowerCAmelCase     =    BatchFeature({"""input_values""": features})
							lowerCAmelCase     =    self.num_mel_bins
					else:
							lowerCAmelCase     =    BatchFeature({"""input_values""": speech})

					lowerCAmelCase     =    self.pad(
					    __lowerCAmelCase			,       padding=__lowerCAmelCase			,       max_length=__lowerCAmelCase			,       truncation=__lowerCAmelCase			,       pad_to_multiple_of=__lowerCAmelCase			,       return_attention_mask=__lowerCAmelCase			,       **__lowerCAmelCase			,       )

					lowerCAmelCase     =    feature_size_hack

					# convert input values to correct format
					lowerCAmelCase     =    padded_inputs["""input_values"""]
					if not isinstance(input_values[0]			,       np.ndarray):
							lowerCAmelCase     =    [np.asarray(__lowerCAmelCase			,       dtype=np.floataa) for array in input_values]
					elif (
					    not isinstance(__lowerCAmelCase			,       np.ndarray)
					    and isinstance(input_values[0]			,       np.ndarray)
					    and input_values[0].dtype is np.dtype(np.floataa)
					):
							lowerCAmelCase     =    [array.astype(np.floataa) for array in input_values]
					elif isinstance(__lowerCAmelCase			,       np.ndarray) and input_values.dtype is np.dtype(np.floataa):
							lowerCAmelCase     =    input_values.astype(np.floataa)

					# convert attention_mask to correct format
					lowerCAmelCase     =    padded_inputs.get("""attention_mask""")
					if attention_mask is not None:
							lowerCAmelCase     =    [np.asarray(__lowerCAmelCase			,       dtype=np.intaa) for array in attention_mask]

					# zero-mean and unit-variance normalization
					if not is_target and self.do_normalize:
							lowerCAmelCase     =    (
							    attention_mask
							    if self._get_padding_strategies(__lowerCAmelCase			,       max_length=__lowerCAmelCase) is not PaddingStrategy.DO_NOT_PAD
							    else None
							)
							lowerCAmelCase     =    self.zero_mean_unit_var_norm(
							    padded_inputs["""input_values"""]			,       attention_mask=__lowerCAmelCase			,       padding_value=self.padding_value)

					if return_tensors is not None:
							lowerCAmelCase     =    padded_inputs.convert_to_tensors(__lowerCAmelCase)

					return padded_inputs







			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    super().to_dict()

					# Don't serialize these as they are derived from the other properties.
					lowerCAmelCase     =    ["""window""", """mel_filters""", """sample_size""", """sample_stride""", """n_fft""", """n_freqs"""]
					for name in names:
							if name in output:
									del output[name]

					return output




