
'''simple docstring'''





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def __init__(					self		:  Optional[int]			,				snake_case__		:  NestedDataStructureLike[PathLike]			,				snake_case__		:  Optional[NamedSplit] = None			,				snake_case__		:  Optional[Features] = None			,				snake_case__		:  str = None			,				snake_case__		:  bool = False			,				snake_case__		:  bool = False			,				snake_case__		:  Optional[str] = None			,				snake_case__		:  Optional[int] = None			,				**snake_case__		:  Any			,				):



   '''simple docstring'''




   super().__init__(
       snake_case__			,				split=snake_case__			,				features=snake_case__			,				cache_dir=snake_case__			,				keep_in_memory=snake_case__			,				streaming=snake_case__			,				num_proc=snake_case__			,				**snake_case__			,				)
   UpperCAmelCase__      :					str				       =							field
   UpperCAmelCase__      :					Union[str, Any]				       =							path_or_paths if isinstance(snake_case__			,				snake_case__ ) else {self.split: path_or_paths}
   UpperCAmelCase__      :					str				       =							Json(
       cache_dir=snake_case__			,				data_files=snake_case__			,				features=snake_case__			,				field=snake_case__			,				**snake_case__			,				)







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   if self.streaming:
     UpperCAmelCase__      :					Union[str, Any]				       =							self.builder.as_streaming_dataset(split=self.split )
   # Build regular (map-style) dataset
   else:
     UpperCAmelCase__      :					List[Any]				       =							None
     UpperCAmelCase__      :					Tuple				       =							None
     UpperCAmelCase__      :					Dict				       =							None
     UpperCAmelCase__      :					Tuple				       =							None

     self.builder.download_and_prepare(
         download_config=snake_case__			,				download_mode=snake_case__			,				verification_mode=snake_case__			,				base_path=snake_case__			,				num_proc=self.num_proc			,				)
     UpperCAmelCase__      :					Optional[int]				       =							self.builder.as_dataset(
         split=self.split			,				verification_mode=snake_case__			,				in_memory=self.keep_in_memory )
   return dataset






class 							UpperCAmelCase_							:

 '''simple docstring'''







 def __init__(					self		:  int			,				snake_case__		:  Dataset			,				snake_case__		:  Union[PathLike, BinaryIO]			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  Optional[int] = None			,				**snake_case__		:  int			,				):



   '''simple docstring'''




   if num_proc is not None and num_proc <= 0:
     raise ValueError(F"""num_proc {num_proc} must be an integer > 0.""" )

   UpperCAmelCase__      :					Union[str, Any]				       =							dataset
   UpperCAmelCase__      :					Union[str, Any]				       =							path_or_buf
   UpperCAmelCase__      :					List[str]				       =							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
   UpperCAmelCase__      :					List[Any]				       =							num_proc
   UpperCAmelCase__      :					Any				       =							"utf-8"
   UpperCAmelCase__      :					str				       =							to_json_kwargs







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					int				       =							self.to_json_kwargs.pop("path_or_buf"			,				snake_case__ )
   UpperCAmelCase__      :					int				       =							self.to_json_kwargs.pop("orient"			,				"records" )
   UpperCAmelCase__      :					int				       =							self.to_json_kwargs.pop("lines"			,				True if orient == "records" else False )
   UpperCAmelCase__      :					Dict				       =							self.to_json_kwargs.pop("index"			,				False if orient in ["split", "table"] else True )
   UpperCAmelCase__      :					Optional[int]				       =							self.to_json_kwargs.pop("compression"			,				snake_case__ )

   if compression not in [None, "infer", "gzip", "bz2", "xz"]:
     raise NotImplementedError(F"""`datasets` currently does not support {compression} compression""" )

   if isinstance(self.path_or_buf			,				(str, bytes, os.PathLike) ):
     with fsspec.open(self.path_or_buf			,				"wb"			,				compression=snake_case__ ) as buffer:
       UpperCAmelCase__      :					Optional[Any]				       =							self._write(file_obj=snake_case__			,				orient=snake_case__			,				lines=snake_case__			,				index=snake_case__			,				**self.to_json_kwargs )
   else:
     if compression:
       raise NotImplementedError(
           F"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
           " was passed. Please provide a local path instead." )
     UpperCAmelCase__      :					Union[str, Any]				       =							self._write(
         file_obj=self.path_or_buf			,				orient=snake_case__			,				lines=snake_case__			,				index=snake_case__			,				**self.to_json_kwargs )
   return written







 def 							UpperCamelCase  (					self		:  str			,				snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					List[str]				       =							args

   UpperCAmelCase__      :					Optional[Any]				       =							query_table(
       table=self.dataset.data			,				key=slice(snake_case__			,				offset + self.batch_size )			,				indices=self.dataset._indices			,				)
   UpperCAmelCase__      :					List[str]				       =							batch.to_pandas().to_json(
       path_or_buf=snake_case__			,				orient=snake_case__			,				lines=snake_case__			,				index=snake_case__			,				**snake_case__ )
   if not json_str.endswith("\n" ):
     json_str += "\n"
   return json_str.encode(self.encoding )







 def 							UpperCamelCase  (					self		:  int			,				snake_case__		:  BinaryIO			,				snake_case__		:  Tuple			,				snake_case__		:  Union[str, Any]			,				snake_case__		:  str			,				**snake_case__		:  List[Any]			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					List[str]				       =							0

   if self.num_proc is None or self.num_proc == 1:
     for offset in logging.tqdm(
         range(0			,				len(self.dataset )			,				self.batch_size )			,				unit="ba"			,				disable=not logging.is_progress_bar_enabled()			,				desc="Creating json from Arrow format"			,				):
       UpperCAmelCase__      :					Optional[Any]				       =							self._batch_json((offset, orient, lines, index, to_json_kwargs) )
       written += file_obj.write(snake_case__ )
   else:
     UpperCAmelCase__       ,			UpperCAmelCase__      :					Union[str, Any]				       =							len(self.dataset ), self.batch_size
     with multiprocessing.Pool(self.num_proc ) as pool:
       for json_str in logging.tqdm(
           pool.imap(
               self._batch_json			,				[(offset, orient, lines, index, to_json_kwargs) for offset in range(0			,				snake_case__			,				snake_case__ )]			,				)			,				total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,				unit="ba"			,				disable=not logging.is_progress_bar_enabled()			,				desc="Creating json from Arrow format"			,				):
         written += file_obj.write(snake_case__ )

   return written





'''simple docstring'''





import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    int				=							["input_values", "attention_mask"]







 def __init__(					self		:  Any			,				snake_case__		:  int = 1			,				snake_case__		:  int = 1_60_00			,				snake_case__		:  float = 0.0			,				snake_case__		:  bool = False			,				snake_case__		:  int = 80			,				snake_case__		:  int = 16			,				snake_case__		:  int = 64			,				snake_case__		:  str = "hann_window"			,				snake_case__		:  float = 1.0			,				snake_case__		:  float = 80			,				snake_case__		:  float = 76_00			,				snake_case__		:  float = 1e-10			,				snake_case__		:  int = 2			,				snake_case__		:  bool = True			,				**snake_case__		:  List[Any]			,				):



   '''simple docstring'''




   super().__init__(feature_size=snake_case__			,				sampling_rate=snake_case__			,				padding_value=snake_case__			,				**snake_case__ )
   UpperCAmelCase__      :					int				       =							do_normalize
   UpperCAmelCase__      :					Tuple				       =							return_attention_mask

   UpperCAmelCase__      :					Union[str, Any]				       =							num_mel_bins
   UpperCAmelCase__      :					List[str]				       =							hop_length
   UpperCAmelCase__      :					List[str]				       =							win_length
   UpperCAmelCase__      :					Union[str, Any]				       =							win_function
   UpperCAmelCase__      :					str				       =							frame_signal_scale
   UpperCAmelCase__      :					int				       =							fmin
   UpperCAmelCase__      :					Union[str, Any]				       =							fmax
   UpperCAmelCase__      :					Optional[Any]				       =							mel_floor
   UpperCAmelCase__      :					List[str]				       =							reduction_factor

   UpperCAmelCase__      :					str				       =							win_length * sampling_rate // 10_00
   UpperCAmelCase__      :					Union[str, Any]				       =							hop_length * sampling_rate // 10_00
   UpperCAmelCase__      :					Optional[Any]				       =							optimal_fft_length(self.sample_size )
   UpperCAmelCase__      :					str				       =							(self.n_fft // 2) + 1

   UpperCAmelCase__      :					Dict				       =							window_function(window_length=self.sample_size			,				name=self.win_function			,				periodic=snake_case__ )

   UpperCAmelCase__      :					List[str]				       =							mel_filter_bank(
       num_frequency_bins=self.n_freqs			,				num_mel_filters=self.num_mel_bins			,				min_frequency=self.fmin			,				max_frequency=self.fmax			,				sampling_rate=self.sampling_rate			,				norm="slaney"			,				mel_scale="slaney"			,				)

   if frame_signal_scale != 1.0:
     warnings.warn(
         "The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers"			,				snake_case__			,				)
   if reduction_factor != 2.0:
     warnings.warn(
         "The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers"			,				snake_case__			,				)







 @staticmethod
 # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
 def 							UpperCamelCase  (					snake_case__		:  List[np.ndarray]			,				snake_case__		:  List[np.ndarray]			,				snake_case__		:  float = 0.0 ):



   '''simple docstring'''




   if attention_mask is not None:
     UpperCAmelCase__      :					Tuple				       =							np.array(snake_case__			,				np.intaa )
     UpperCAmelCase__      :					Tuple				       =							[]

     for vector, length in zip(snake_case__			,				attention_mask.sum(-1 ) ):
       UpperCAmelCase__      :					List[str]				       =							(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7 )
       if length < normed_slice.shape[0]:
         UpperCAmelCase__      :					Optional[int]				       =							padding_value

       normed_input_values.append(snake_case__ )
   else:
     UpperCAmelCase__      :					Optional[int]				       =							[(x - x.mean()) / np.sqrt(x.var() + 1e-7 ) for x in input_values]

   return normed_input_values







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  np.ndarray			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							spectrogram(
       snake_case__			,				window=self.window			,				frame_length=self.sample_size			,				hop_length=self.sample_stride			,				fft_length=self.n_fft			,				mel_filters=self.mel_filters			,				mel_floor=self.mel_floor			,				log_mel="log10"			,				)
   return log_mel_spec.T







 def __call__(					self		:  List[Any]			,				snake_case__		:  Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,				snake_case__		:  Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,				snake_case__		:  Union[bool, str, PaddingStrategy] = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  bool = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[Union[str, TensorType]] = None			,				snake_case__		:  Optional[int] = None			,				**snake_case__		:  List[Any]			,				):



   '''simple docstring'''




   if audio is None and audio_target is None:
     raise ValueError("You must provide either `audio` or `audio_target` values." )

   if sampling_rate is not None:
     if sampling_rate != self.sampling_rate:
       raise ValueError(
           F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
           F""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
           F""" {self.sampling_rate} and not {sampling_rate}.""" )
   else:
     logger.warning(
         "It is strongly recommended to pass the ``sampling_rate`` argument to this function. "
         "Failing to do so can result in silent errors that might be hard to debug." )

   if audio is not None:
     UpperCAmelCase__      :					Union[str, Any]				       =							self._process_audio(
         snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				**snake_case__			,				)
   else:
     UpperCAmelCase__      :					str				       =							None

   if audio_target is not None:
     UpperCAmelCase__      :					str				       =							self._process_audio(
         snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				**snake_case__			,				)

     if inputs is None:
       return inputs_target
     else:
       UpperCAmelCase__      :					Union[str, Any]				       =							inputs_target["input_values"]
       UpperCAmelCase__      :					Any				       =							inputs_target.get("attention_mask" )
       if decoder_attention_mask is not None:
         UpperCAmelCase__      :					Dict				       =							decoder_attention_mask

   return inputs







 def 							UpperCamelCase  (					self		:  List[Any]			,				snake_case__		:  Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]			,				snake_case__		:  bool = False			,				snake_case__		:  Union[bool, str, PaddingStrategy] = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  bool = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[Union[str, TensorType]] = None			,				**snake_case__		:  int			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							isinstance(snake_case__			,				np.ndarray ) and len(speech.shape ) > 1
   if is_batched_numpy and len(speech.shape ) > 2:
     raise ValueError(F"""Only mono-channel audio is supported for input to {self}""" )
   UpperCAmelCase__      :					Union[str, Any]				       =							is_batched_numpy or (
       isinstance(snake_case__			,				(list, tuple) ) and (isinstance(speech[0]			,				(np.ndarray, tuple, list) ))
   )

   if is_batched:
     UpperCAmelCase__      :					List[Any]				       =							[np.asarray(snake_case__			,				dtype=np.floataa ) for speech in speech]
   elif not is_batched and not isinstance(snake_case__			,				np.ndarray ):
     UpperCAmelCase__      :					int				       =							np.asarray(snake_case__			,				dtype=np.floataa )
   elif isinstance(snake_case__			,				np.ndarray ) and speech.dtype is np.dtype(np.floataa ):
     UpperCAmelCase__      :					List[Any]				       =							speech.astype(np.floataa )

   # always return batch
   if not is_batched:
     UpperCAmelCase__      :					str				       =							[speech]

   # needed to make pad() work on spectrogram inputs
   UpperCAmelCase__      :					str				       =							self.feature_size

   # convert into correct format for padding
   if is_target:
     UpperCAmelCase__      :					List[Any]				       =							[self._extract_mel_features(snake_case__ ) for waveform in speech]
     UpperCAmelCase__      :					List[Any]				       =							BatchFeature({"input_values": features} )
     UpperCAmelCase__      :					List[str]				       =							self.num_mel_bins
   else:
     UpperCAmelCase__      :					List[str]				       =							BatchFeature({"input_values": speech} )

   UpperCAmelCase__      :					Optional[Any]				       =							self.pad(
       snake_case__			,				padding=snake_case__			,				max_length=snake_case__			,				truncation=snake_case__			,				pad_to_multiple_of=snake_case__			,				return_attention_mask=snake_case__			,				**snake_case__			,				)

   UpperCAmelCase__      :					List[str]				       =							feature_size_hack

   # convert input values to correct format
   UpperCAmelCase__      :					Tuple				       =							padded_inputs["input_values"]
   if not isinstance(input_values[0]			,				np.ndarray ):
     UpperCAmelCase__      :					List[Any]				       =							[np.asarray(snake_case__			,				dtype=np.floataa ) for array in input_values]
   elif (
       not isinstance(snake_case__			,				np.ndarray )
       and isinstance(input_values[0]			,				np.ndarray )
       and input_values[0].dtype is np.dtype(np.floataa )
   ):
     UpperCAmelCase__      :					str				       =							[array.astype(np.floataa ) for array in input_values]
   elif isinstance(snake_case__			,				np.ndarray ) and input_values.dtype is np.dtype(np.floataa ):
     UpperCAmelCase__      :					str				       =							input_values.astype(np.floataa )

   # convert attention_mask to correct format
   UpperCAmelCase__      :					Dict				       =							padded_inputs.get("attention_mask" )
   if attention_mask is not None:
     UpperCAmelCase__      :					int				       =							[np.asarray(snake_case__			,				dtype=np.intaa ) for array in attention_mask]

   # zero-mean and unit-variance normalization
   if not is_target and self.do_normalize:
     UpperCAmelCase__      :					Any				       =							(
         attention_mask
         if self._get_padding_strategies(snake_case__			,				max_length=snake_case__ ) is not PaddingStrategy.DO_NOT_PAD
         else None
     )
     UpperCAmelCase__      :					int				       =							self.zero_mean_unit_var_norm(
         padded_inputs["input_values"]			,				attention_mask=snake_case__			,				padding_value=self.padding_value )

   if return_tensors is not None:
     UpperCAmelCase__      :					Union[str, Any]				       =							padded_inputs.convert_to_tensors(snake_case__ )

   return padded_inputs







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							super().to_dict()

   # Don't serialize these as they are derived from the other properties.
   UpperCAmelCase__      :					Dict				       =							["window", "mel_filters", "sample_size", "sample_stride", "n_fft", "n_freqs"]
   for name in names:
     if name in output:
       del output[name]

   return output



