



'''simple docstring'''

import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


a							   =				logging.get_logger(__name__)






class 						a_		(     snake_case    ):
							UpperCAmelCase			:		Any        =				["""input_values""", """attention_mask"""]


							def __init__(				self   :						int      ,						a_   :						int = 1      ,						a_   :						int = 1_6_0_0_0      ,						a_   :						float = 0.0      ,						a_   :						bool = False      ,						a_   :						int = 8_0      ,						a_   :						int = 1_6      ,						a_   :						int = 6_4      ,						a_   :						str = "hann_window"      ,						a_   :						float = 1.0      ,						a_   :						float = 8_0      ,						a_   :						float = 7_6_0_0      ,						a_   :						float = 1E-1_0      ,						a_   :						int = 2      ,						a_   :						bool = True      ,						**a_   :						Optional[Any]      ,						)			->					Dict:
										super().__init__(feature_size=a_      ,						sampling_rate=a_      ,						padding_value=a_      ,						**a_			)
										snake_case:    Optional[int]      							=do_normalize
										snake_case:    List[Any]      							=return_attention_mask

										snake_case:    Union[str, Any]      							=num_mel_bins
										snake_case:    int      							=hop_length
										snake_case:    Union[str, Any]      							=win_length
										snake_case:    List[str]      							=win_function
										snake_case:    List[str]      							=frame_signal_scale
										snake_case:    Optional[Any]      							=fmin
										snake_case:    str      							=fmax
										snake_case:    Optional[int]      							=mel_floor
										snake_case:    str      							=reduction_factor

										snake_case:    Any      							=win_length * sampling_rate // 1_0_0_0
										snake_case:    Union[str, Any]      							=hop_length * sampling_rate // 1_0_0_0
										snake_case:    Any      							=optimal_fft_length(self.sample_size			)
										snake_case:    str      							=(self.n_fft // 2) + 1

										snake_case:    List[Any]      							=window_function(window_length=self.sample_size      ,						name=self.win_function      ,						periodic=a_			)

										snake_case:    Any      							=mel_filter_bank(
										    num_frequency_bins=self.n_freqs      ,						num_mel_filters=self.num_mel_bins      ,						min_frequency=self.fmin      ,						max_frequency=self.fmax      ,						sampling_rate=self.sampling_rate      ,						norm='slaney'      ,						mel_scale='slaney'      ,						)

										if frame_signal_scale != 1.0:
													warnings.warn(
													    'The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers'      ,						a_      ,						)
										if reduction_factor != 2.0:
													warnings.warn(
													    'The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers'      ,						a_      ,						)


							@staticmethod
							# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
							def    UpperCamelCase     (				a_   :						List[np.ndarray]      ,						a_   :						List[np.ndarray]      ,						a_   :						float = 0.0			)			->					List[np.ndarray]:
										if attention_mask is not None:
													snake_case:    List[str]      							=np.array(a_      ,						np.intaa			)
													snake_case:    str      							=[]

													for vector, length in zip(a_      ,						attention_mask.sum(-1			)			):
																snake_case:    Optional[Any]      							=(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7			)
																if length < normed_slice.shape[0]:
																			snake_case:    str      							=padding_value

																normed_input_values.append(a_			)
										else:
													snake_case:    Any      							=[(x - x.mean()) / np.sqrt(x.var() + 1E-7			) for x in input_values]

										return normed_input_values


							def    UpperCamelCase     (				self   :						Optional[int]      ,						a_   :						np.ndarray      ,						)			->					np.ndarray:
										snake_case:    str      							=spectrogram(
										    a_      ,						window=self.window      ,						frame_length=self.sample_size      ,						hop_length=self.sample_stride      ,						fft_length=self.n_fft      ,						mel_filters=self.mel_filters      ,						mel_floor=self.mel_floor      ,						log_mel='log10'      ,						)
										return log_mel_spec.T


							def __call__(				self   :						Dict      ,						a_   :						Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None      ,						a_   :						Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None      ,						a_   :						Union[bool, str, PaddingStrategy] = False      ,						a_   :						Optional[int] = None      ,						a_   :						bool = False      ,						a_   :						Optional[int] = None      ,						a_   :						Optional[bool] = None      ,						a_   :						Optional[Union[str, TensorType]] = None      ,						a_   :						Optional[int] = None      ,						**a_   :						List[Any]      ,						)			->					BatchFeature:
										if audio is None and audio_target is None:
													raise ValueError('You must provide either `audio` or `audio_target` values.'			)

										if sampling_rate is not None:
													if sampling_rate != self.sampling_rate:
																raise ValueError(
																    F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
																    F''' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'''
																    F''' {self.sampling_rate} and not {sampling_rate}.'''			)
										else:
													logger.warning(
													    'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
													    'Failing to do so can result in silent errors that might be hard to debug.'			)

										if audio is not None:
													snake_case:    Union[str, Any]      							=self._process_audio(
													    a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						**a_      ,						)
										else:
													snake_case:    Tuple      							=None

										if audio_target is not None:
													snake_case:    int      							=self._process_audio(
													    a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						a_      ,						**a_      ,						)

													if inputs is None:
																return inputs_target
													else:
																snake_case:    Dict      							=inputs_target['input_values']
																snake_case:    Optional[Any]      							=inputs_target.get('attention_mask'			)
																if decoder_attention_mask is not None:
																			snake_case:    Optional[int]      							=decoder_attention_mask

										return inputs


							def    UpperCamelCase     (				self   :						Tuple      ,						a_   :						Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]      ,						a_   :						bool = False      ,						a_   :						Union[bool, str, PaddingStrategy] = False      ,						a_   :						Optional[int] = None      ,						a_   :						bool = False      ,						a_   :						Optional[int] = None      ,						a_   :						Optional[bool] = None      ,						a_   :						Optional[Union[str, TensorType]] = None      ,						**a_   :						Optional[Any]      ,						)			->					BatchFeature:
										snake_case:    List[str]      							=isinstance(a_      ,						np.ndarray			) and len(speech.shape			) > 1
										if is_batched_numpy and len(speech.shape			) > 2:
													raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''			)
										snake_case:    List[str]      							=is_batched_numpy or (
										    isinstance(a_      ,						(list, tuple)			) and (isinstance(speech[0]      ,						(np.ndarray, tuple, list)			))
										)

										if is_batched:
													snake_case:    List[Any]      							=[np.asarray(a_      ,						dtype=np.floataa			) for speech in speech]
										elif not is_batched and not isinstance(a_      ,						np.ndarray			):
													snake_case:    str      							=np.asarray(a_      ,						dtype=np.floataa			)
										elif isinstance(a_      ,						np.ndarray			) and speech.dtype is np.dtype(np.floataa			):
													snake_case:    Optional[Any]      							=speech.astype(np.floataa			)

										# always return batch
										if not is_batched:
													snake_case:    List[Any]      							=[speech]

										# needed to make pad() work on spectrogram inputs
										snake_case:    List[Any]      							=self.feature_size

										# convert into correct format for padding
										if is_target:
													snake_case:    Union[str, Any]      							=[self._extract_mel_features(a_			) for waveform in speech]
													snake_case:    Tuple      							=BatchFeature({'input_values': features}			)
													snake_case:    Union[str, Any]      							=self.num_mel_bins
										else:
													snake_case:    Dict      							=BatchFeature({'input_values': speech}			)

										snake_case:    Optional[Any]      							=self.pad(
										    a_      ,						padding=a_      ,						max_length=a_      ,						truncation=a_      ,						pad_to_multiple_of=a_      ,						return_attention_mask=a_      ,						**a_      ,						)

										snake_case:    Union[str, Any]      							=feature_size_hack

										# convert input values to correct format
										snake_case:    List[Any]      							=padded_inputs['input_values']
										if not isinstance(input_values[0]      ,						np.ndarray			):
													snake_case:    int      							=[np.asarray(a_      ,						dtype=np.floataa			) for array in input_values]
										elif (
										    not isinstance(a_      ,						np.ndarray			)
										    and isinstance(input_values[0]      ,						np.ndarray			)
										    and input_values[0].dtype is np.dtype(np.floataa			)
										):
													snake_case:    Optional[Any]      							=[array.astype(np.floataa			) for array in input_values]
										elif isinstance(a_      ,						np.ndarray			) and input_values.dtype is np.dtype(np.floataa			):
													snake_case:    Any      							=input_values.astype(np.floataa			)

										# convert attention_mask to correct format
										snake_case:    Dict      							=padded_inputs.get('attention_mask'			)
										if attention_mask is not None:
													snake_case:    List[str]      							=[np.asarray(a_      ,						dtype=np.intaa			) for array in attention_mask]

										# zero-mean and unit-variance normalization
										if not is_target and self.do_normalize:
													snake_case:    int      							=(
													    attention_mask
													    if self._get_padding_strategies(a_      ,						max_length=a_			) is not PaddingStrategy.DO_NOT_PAD
													    else None
													)
													snake_case:    Optional[int]      							=self.zero_mean_unit_var_norm(
													    padded_inputs['input_values']      ,						attention_mask=a_      ,						padding_value=self.padding_value			)

										if return_tensors is not None:
													snake_case:    List[Any]      							=padded_inputs.convert_to_tensors(a_			)

										return padded_inputs




							def    UpperCamelCase     (				self   :						List[Any]			)			->					Dict[str, Any]:
										snake_case:    Tuple      							=super().to_dict()

										# Don't serialize these as they are derived from the other properties.
										snake_case:    Union[str, Any]      							=['window', 'mel_filters', 'sample_size', 'sample_stride', 'n_fft', 'n_freqs']
										for name in names:
													if name in output:
																del output[name]

										return output









'''simple docstring'''

import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						a_		(     snake_case    ,   unittest.TestCase    ):
							UpperCAmelCase			:		int        =				FunnelTokenizer
							UpperCAmelCase			:		Tuple        =				FunnelTokenizerFast
							UpperCAmelCase			:		List[str]        =				True
							UpperCAmelCase			:		Tuple        =				True


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					Optional[Any]:
										super().setUp()

										snake_case:    Optional[Any]      							=[
										    '<unk>',
										    '<cls>',
										    '<sep>',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										snake_case:    Any      							=os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES['vocab_file']			)
										with open(self.vocab_file      ,						'w'      ,						encoding='utf-8'			) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]			)			)


							def    UpperCamelCase     (				self   :						Tuple      ,						**a_   :						str			)			->					Any:
										return FunnelTokenizer.from_pretrained(self.tmpdirname      ,						**a_			)


							def    UpperCamelCase     (				self   :						List[Any]      ,						**a_   :						int			)			->					str:
										return FunnelTokenizerFast.from_pretrained(self.tmpdirname      ,						**a_			)


							def    UpperCamelCase     (				self   :						Any      ,						a_   :						Union[str, Any]			)			->					Union[str, Any]:
										snake_case:    List[str]      							='UNwant\u00E9d,running'
										snake_case:    Dict      							='unwanted, running'
										return input_text, output_text


							def    UpperCamelCase     (				self   :						Optional[int]			)			->					List[Any]:
										snake_case:    Union[str, Any]      							=self.tokenizer_class(self.vocab_file			)

										snake_case:    Dict      							=tokenizer.tokenize('UNwant\u00E9d,running'			)
										self.assertListEqual(a_      ,						['un', '##want', '##ed', ',', 'runn', '##ing']			)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)      ,						[7, 4, 5, 1_0, 8, 9]			)




							def    UpperCamelCase     (				self   :						List[str]			)			->					Dict:
										snake_case:    Optional[Any]      							=self.get_tokenizers(do_lower_case=a_			)
										for tokenizer in tokenizers:
													snake_case:    str      							=tokenizer('UNwant\u00E9d,running'			)
													snake_case:    Union[str, Any]      							=len(inputs['input_ids']			) - 1
													self.assertListEqual(inputs['token_type_ids']      ,						[2] + [0] * sentence_len			)

													snake_case:    List[str]      							=tokenizer('UNwant\u00E9d,running'      ,						'UNwant\u00E9d,running'			)
													self.assertListEqual(inputs['token_type_ids']      ,						[2] + [0] * sentence_len + [1] * sentence_len			)




