


'''simple docstring'''






import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


lowercase				=datasets.utils.logging.get_logger(__name__)

lowercase				=['names', 'prefix']
lowercase				=['warn_bad_lines', 'error_bad_lines', 'mangle_dupe_cols']
lowercase				=['encoding_errors', 'on_bad_lines']
lowercase				=['date_format']






@dataclass
class    __magic_name__				(						datasets.BuilderConfig							):
       UpperCAmelCase		 =","
       UpperCAmelCase		 =None
       UpperCAmelCase		 ="infer"
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =True
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =False
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =True
       UpperCAmelCase		 =True
       UpperCAmelCase		 =False
       UpperCAmelCase		 =True
       UpperCAmelCase		 =None
       UpperCAmelCase		 ="."
       UpperCAmelCase		 =None
       UpperCAmelCase		 ='"'
       UpperCAmelCase		 =0
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =None
       UpperCAmelCase		 =True
       UpperCAmelCase		 =True
       UpperCAmelCase		 =0
       UpperCAmelCase		 =True
       UpperCAmelCase		 =False
       UpperCAmelCase		 =None
       UpperCAmelCase		 =1_0_0_0_0
       UpperCAmelCase		 =None
       UpperCAmelCase		 ="strict"
       UpperCAmelCase		 ="error"
       UpperCAmelCase		 =None






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          if self.delimiter is not None:
             _UpperCAmelCase  :  Dict				       =self.delimiter
          if self.column_names is not None:
             _UpperCAmelCase  :  Any				       =self.column_names






       @property
       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       ={
              'sep': self.sep,
              'header': self.header,
              'names': self.names,
              'index_col': self.index_col,
              'usecols': self.usecols,
              'prefix': self.prefix,
              'mangle_dupe_cols': self.mangle_dupe_cols,
              'engine': self.engine,
              'converters': self.converters,
              'true_values': self.true_values,
              'false_values': self.false_values,
              'skipinitialspace': self.skipinitialspace,
              'skiprows': self.skiprows,
              'nrows': self.nrows,
              'na_values': self.na_values,
              'keep_default_na': self.keep_default_na,
              'na_filter': self.na_filter,
              'verbose': self.verbose,
              'skip_blank_lines': self.skip_blank_lines,
              'thousands': self.thousands,
              'decimal': self.decimal,
              'lineterminator': self.lineterminator,
              'quotechar': self.quotechar,
              'quoting': self.quoting,
              'escapechar': self.escapechar,
              'comment': self.comment,
              'encoding': self.encoding,
              'dialect': self.dialect,
              'error_bad_lines': self.error_bad_lines,
              'warn_bad_lines': self.warn_bad_lines,
              'skipfooter': self.skipfooter,
              'doublequote': self.doublequote,
              'memory_map': self.memory_map,
              'float_precision': self.float_precision,
              'chunksize': self.chunksize,
              'encoding_errors': self.encoding_errors,
              'on_bad_lines': self.on_bad_lines,
              'date_format': self.date_format,
          }

          # some kwargs must not be passed if they don't have a default value
          # some others are deprecated and we can also not pass them if they are the default value
          for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
             if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()    ,				snake_case):
                del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
          if not (datasets.config.PANDAS_VERSION.major >= 2):
             for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
                del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
          if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
             for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
                del pd_read_csv_kwargs[pd_read_csv_parameter]

          return pd_read_csv_kwargs






class    __magic_name__				(						datasets.ArrowBasedBuilder							):
       UpperCAmelCase		 =CsvConfig






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          return datasets.DatasetInfo(features=self.config.features)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    List[str]:




          '''simple docstring'''
          if not self.config.data_files:
             raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
          _UpperCAmelCase  :  Optional[Any]				       =dl_manager.download_and_extract(self.config.data_files)
          if isinstance(snake_case    ,				(str, list, tuple)):
             _UpperCAmelCase  :  Optional[int]				       =data_files
             if isinstance(snake_case    ,				snake_case):
                _UpperCAmelCase  :  Tuple				       =[files]
             _UpperCAmelCase  :  str				       =[dl_manager.iter_files(snake_case) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,				gen_kwargs={'files': files})]
          _UpperCAmelCase  :  Union[str, Any]				       =[]
          for split_name, files in data_files.items():
             if isinstance(snake_case    ,				snake_case):
                _UpperCAmelCase  :  Dict				       =[files]
             _UpperCAmelCase  :  Union[str, Any]				       =[dl_manager.iter_files(snake_case) for file in files]
             splits.append(datasets.SplitGenerator(name=snake_case    ,				gen_kwargs={'files': files}))
          return splits






       def      lowerCAmelCase						(     self    ,				snake_case)		->    pa.Table:




          '''simple docstring'''
          if self.config.features is not None:
             _UpperCAmelCase  :  List[str]				       =self.config.features.arrow_schema
             if all(not require_storage_cast(snake_case) for feature in self.config.features.values()):
                # cheaper cast
                _UpperCAmelCase  :  Any				       =pa.Table.from_arrays([pa_table[field.name] for field in schema]    ,				schema=snake_case)
             else:
                # more expensive cast; allows str <-> int/float or str to Audio for example
                _UpperCAmelCase  :  Dict				       =table_cast(snake_case    ,				snake_case)
          return pa_table






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[Any]				       =self.config.features.arrow_schema if self.config.features else None
          # dtype allows reading an int column as str
          _UpperCAmelCase  :  Union[str, Any]				       =(
              {
                  name: dtype.to_pandas_dtype() if not require_storage_cast(snake_case) else object
                  for name, dtype, feature in zip(schema.names    ,				schema.types    ,				self.config.features.values())
              }
              if schema is not None
              else None
          )
          for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case)):
             _UpperCAmelCase  :  Tuple				       =pd.read_csv(snake_case    ,				iterator=snake_case    ,				dtype=snake_case    ,				**self.config.pd_read_csv_kwargs)
             try:
                for batch_idx, df in enumerate(snake_case):
                   _UpperCAmelCase  :  Tuple				       =pa.Table.from_pandas(snake_case)
                   # Uncomment for debugging (will print the Arrow table size and elements)
                   # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                   # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                   yield (file_idx, batch_idx), self._cast_table(snake_case)
             except ValueError as e:
                logger.error(f"Failed to read file '{file}' with error {type(snake_case)}: {e}")
                raise





'''simple docstring'''






from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


lowercase				=logging.get_logger(__name__)






class    __magic_name__				(						lowerCAmelCase							):
       UpperCAmelCase		 =["input_features", "attention_mask"]






       def __init__(     self    ,				snake_case=8_0    ,				snake_case=1_6_0_0_0    ,				snake_case=8_0    ,				snake_case=0.0    ,				snake_case=True    ,				snake_case=True    ,				snake_case=True    ,				**snake_case    ,				)		->    str:




          '''simple docstring'''
          super().__init__(feature_size=snake_case    ,				sampling_rate=snake_case    ,				padding_value=snake_case    ,				**snake_case)
          _UpperCAmelCase  :  Optional[Any]				       =num_mel_bins
          _UpperCAmelCase  :  Optional[int]				       =do_ceptral_normalize
          _UpperCAmelCase  :  Optional[Any]				       =normalize_means
          _UpperCAmelCase  :  Optional[Any]				       =normalize_vars
          _UpperCAmelCase  :  Tuple				       =True






       def      lowerCAmelCase						(     self    ,				snake_case    ,				)		->    np.ndarray:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =waveform * (2**1_5)  # Kaldi compliance: 16-bit signed integers
          _UpperCAmelCase  :  Dict				       =torch.from_numpy(snake_case).unsqueeze(0)
          _UpperCAmelCase  :  Union[str, Any]				       =ta_kaldi.fbank(snake_case    ,				num_mel_bins=self.num_mel_bins    ,				sample_frequency=self.sampling_rate)
          return features.numpy()






       @staticmethod
       def      lowerCAmelCase						(     snake_case    ,				snake_case    ,				snake_case = True    ,				snake_case = True    ,				snake_case = 0.0    ,				)		->    np.ndarray:




          '''simple docstring'''
          # make sure we normalize float32 arrays
          if normalize_means:
             _UpperCAmelCase  :  int				       =x[:input_length].mean(axis=0)
             _UpperCAmelCase  :  List[Any]				       =np.subtract(snake_case    ,				snake_case)
          if normalize_vars:
             _UpperCAmelCase  :  List[Any]				       =x[:input_length].std(axis=0)
             _UpperCAmelCase  :  Optional[int]				       =np.divide(snake_case    ,				snake_case)

          if input_length < x.shape[0]:
             _UpperCAmelCase  :  Dict				       =padding_value

          # make sure array is in float32
          _UpperCAmelCase  :  str				       =x.astype(np.floataa)

          return x






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    List[np.ndarray]:




          '''simple docstring'''
          _UpperCAmelCase  :  str				       =attention_mask.sum(-1) if attention_mask is not None else [x.shape[0] for x in input_features]
          return [
              self.utterance_cmvn(snake_case    ,				snake_case    ,				self.normalize_means    ,				self.normalize_vars    ,				self.padding_value)
              for x, n in zip(snake_case    ,				snake_case)
          ]






       def __call__(     self    ,				snake_case    ,				snake_case = False    ,				snake_case = None    ,				snake_case = False    ,				snake_case = None    ,				snake_case = None    ,				snake_case = None    ,				snake_case = None    ,				**snake_case    ,				)		->    BatchFeature:




          '''simple docstring'''

          if sampling_rate is not None:
             if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                    f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                    f" {self.sampling_rate} and not {sampling_rate}.")
          else:
             logger.warning(
                 'It is strongly recommended to pass the `sampling_rate` argument to this function. '
                 'Failing to do so can result in silent errors that might be hard to debug.')

          _UpperCAmelCase  :  Optional[int]				       =isinstance(snake_case    ,				np.ndarray) and len(raw_speech.shape) > 1
          if is_batched_numpy and len(raw_speech.shape) > 2:
             raise ValueError(f"Only mono-channel audio is supported for input to {self}")
          _UpperCAmelCase  :  List[Any]				       =is_batched_numpy or (
              isinstance(snake_case    ,				(list, tuple)) and (isinstance(raw_speech[0]    ,				(np.ndarray, tuple, list)))
          )

          if is_batched:
             _UpperCAmelCase  :  int				       =[np.asarray(snake_case    ,				dtype=np.floataa) for speech in raw_speech]
          elif not is_batched and not isinstance(snake_case    ,				np.ndarray):
             _UpperCAmelCase  :  Tuple				       =np.asarray(snake_case    ,				dtype=np.floataa)
          elif isinstance(snake_case    ,				np.ndarray) and raw_speech.dtype is np.dtype(np.floataa):
             _UpperCAmelCase  :  int				       =raw_speech.astype(np.floataa)

          # always return batch
          if not is_batched:
             _UpperCAmelCase  :  Dict				       =[raw_speech]

          # extract fbank features
          _UpperCAmelCase  :  Optional[Any]				       =[self._extract_fbank_features(snake_case) for waveform in raw_speech]

          # convert into correct format for padding
          _UpperCAmelCase  :  List[str]				       =BatchFeature({'input_features': features})

          _UpperCAmelCase  :  Any				       =self.pad(
              snake_case    ,				padding=snake_case    ,				max_length=snake_case    ,				truncation=snake_case    ,				pad_to_multiple_of=snake_case    ,				return_attention_mask=snake_case    ,				**snake_case    ,				)

          # make sure list is in array format
          _UpperCAmelCase  :  Dict				       =padded_inputs.get('input_features')
          if isinstance(input_features[0]    ,				snake_case):
             _UpperCAmelCase  :  Any				       =[np.asarray(snake_case    ,				dtype=np.floataa) for feature in input_features]

          _UpperCAmelCase  :  int				       =padded_inputs.get('attention_mask')
          if attention_mask is not None:
             _UpperCAmelCase  :  Tuple				       =[np.asarray(snake_case    ,				dtype=np.intaa) for array in attention_mask]

          # Utterance-level cepstral mean and variance normalization
          if self.do_ceptral_normalize:
             _UpperCAmelCase  :  Optional[Any]				       =(
                 np.array(snake_case    ,				dtype=np.intaa)
                 if self._get_padding_strategies(snake_case    ,				max_length=snake_case) is not PaddingStrategy.DO_NOT_PAD
                 else None
             )
             _UpperCAmelCase  :  Optional[int]				       =self.normalize(
                 padded_inputs['input_features']    ,				attention_mask=snake_case)

          if return_tensors is not None:
             _UpperCAmelCase  :  List[Any]				       =padded_inputs.convert_to_tensors(snake_case)

          return padded_inputs

