


'''simple docstring'''
import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


UpperCAmelCase_ :     Optional[int]								=      logging.get_logger(__name__)

class 						_lowerCamelCase					(    snake_case_				):




 '''simple docstring'''
 __lowercase				:		List[Any]    							=						['''input_values''', '''attention_mask''']


 def __init__(      self      ,      __lowercase = 1      ,      __lowercase = 16_000      ,      __lowercase = 0.0      ,      __lowercase = False      ,      __lowercase = 80      ,      __lowercase = 16      ,      __lowercase = 64      ,      __lowercase = "hann_window"      ,      __lowercase = 1.0      ,      __lowercase = 80      ,      __lowercase = 7_600      ,      __lowercase = 1E-10      ,      __lowercase = 2      ,      __lowercase = True      ,      **__lowercase      ,      ):

  """simple docstring"""







  super().__init__(feature_size=__lowercase      ,      sampling_rate=__lowercase      ,      padding_value=__lowercase      ,      **__lowercase						)
  __A  :		int       =			do_normalize
  __A  :		Any       =			return_attention_mask

  __A  :		Optional[int]       =			num_mel_bins
  __A  :		Optional[int]       =			hop_length
  __A  :		Union[str, Any]       =			win_length
  __A  :		List[Any]       =			win_function
  __A  :		List[str]       =			frame_signal_scale
  __A  :		List[str]       =			fmin
  __A  :		List[Any]       =			fmax
  __A  :		List[Any]       =			mel_floor
  __A  :		str       =			reduction_factor

  __A  :		List[Any]       =			win_length * sampling_rate // 1_000
  __A  :		List[Any]       =			hop_length * sampling_rate // 1_000
  __A  :		Optional[Any]       =			optimal_fft_length(self.sample_size						)
  __A  :		Union[str, Any]       =			(self.n_fft // 2) + 1

  __A  :		Optional[Any]       =			window_function(window_length=self.sample_size      ,      name=self.win_function      ,      periodic=__lowercase						)

  __A  :		List[str]       =			mel_filter_bank(
      num_frequency_bins=self.n_freqs      ,      num_mel_filters=self.num_mel_bins      ,      min_frequency=self.fmin      ,      max_frequency=self.fmax      ,      sampling_rate=self.sampling_rate      ,      norm='slaney'      ,      mel_scale='slaney'      ,      )

  if frame_signal_scale != 1.0:
   warnings.warn(
       'The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers'      ,      __lowercase      ,      )
  if reduction_factor != 2.0:
   warnings.warn(
       'The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers'      ,      __lowercase      ,      )


 @staticmethod
 # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
 def  snake_case__   (      __lowercase      ,      __lowercase      ,      __lowercase = 0.0						):

  """simple docstring"""







  if attention_mask is not None:
   __A  :		Dict       =			np.array(__lowercase      ,      np.intaa						)
   __A  :		Dict       =			[]

   for vector, length in zip(__lowercase      ,      attention_mask.sum(-1						)						):
    __A  :		List[Any]       =			(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7						)
    if length < normed_slice.shape[0]:
     __A  :		List[str]       =			padding_value

    normed_input_values.append(__lowercase						)
  else:
   __A  :		Dict       =			[(x - x.mean()) / np.sqrt(x.var() + 1E-7						) for x in input_values]

  return normed_input_values


 def  snake_case__   (      self      ,      __lowercase      ,      ):

  """simple docstring"""







  __A  :		Optional[int]       =			spectrogram(
      __lowercase      ,      window=self.window      ,      frame_length=self.sample_size      ,      hop_length=self.sample_stride      ,      fft_length=self.n_fft      ,      mel_filters=self.mel_filters      ,      mel_floor=self.mel_floor      ,      log_mel='log10'      ,      )
  return log_mel_spec.T


 def __call__(      self      ,      __lowercase = None      ,      __lowercase = None      ,      __lowercase = False      ,      __lowercase = None      ,      __lowercase = False      ,      __lowercase = None      ,      __lowercase = None      ,      __lowercase = None      ,      __lowercase = None      ,      **__lowercase      ,      ):

  """simple docstring"""







  if audio is None and audio_target is None:
   raise ValueError('You must provide either `audio` or `audio_target` values.'						)

  if sampling_rate is not None:
   if sampling_rate != self.sampling_rate:
    raise ValueError(
        F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
        F""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
        F""" {self.sampling_rate} and not {sampling_rate}."""						)
  else:
   logger.warning(
       'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
       'Failing to do so can result in silent errors that might be hard to debug.'						)

  if audio is not None:
   __A  :		int       =			self._process_audio(
       __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      **__lowercase      ,      )
  else:
   __A  :		Optional[int]       =			None

  if audio_target is not None:
   __A  :		Dict       =			self._process_audio(
       __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      **__lowercase      ,      )

   if inputs is None:
    return inputs_target
   else:
    __A  :		Any       =			inputs_target['input_values']
    __A  :		List[str]       =			inputs_target.get('attention_mask'						)
    if decoder_attention_mask is not None:
     __A  :		Union[str, Any]       =			decoder_attention_mask

  return inputs


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase = False      ,      __lowercase = False      ,      __lowercase = None      ,      __lowercase = False      ,      __lowercase = None      ,      __lowercase = None      ,      __lowercase = None      ,      **__lowercase      ,      ):

  """simple docstring"""







  __A  :		int       =			isinstance(__lowercase      ,      np.ndarray						) and len(speech.shape						) > 1
  if is_batched_numpy and len(speech.shape						) > 2:
   raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""						)
  __A  :		List[str]       =			is_batched_numpy or (
      isinstance(__lowercase      ,      (list, tuple)						) and (isinstance(speech[0]      ,      (np.ndarray, tuple, list)						))
  )

  if is_batched:
   __A  :		Optional[Any]       =			[np.asarray(__lowercase      ,      dtype=np.floataa						) for speech in speech]
  elif not is_batched and not isinstance(__lowercase      ,      np.ndarray						):
   __A  :		List[str]       =			np.asarray(__lowercase      ,      dtype=np.floataa						)
  elif isinstance(__lowercase      ,      np.ndarray						) and speech.dtype is np.dtype(np.floataa						):
   __A  :		str       =			speech.astype(np.floataa						)

  # always return batch
  if not is_batched:
   __A  :		Union[str, Any]       =			[speech]

  # needed to make pad() work on spectrogram inputs
  __A  :		Dict       =			self.feature_size

  # convert into correct format for padding
  if is_target:
   __A  :		Tuple       =			[self._extract_mel_features(__lowercase						) for waveform in speech]
   __A  :		Tuple       =			BatchFeature({'input_values': features}						)
   __A  :		Tuple       =			self.num_mel_bins
  else:
   __A  :		Dict       =			BatchFeature({'input_values': speech}						)

  __A  :		Dict       =			self.pad(
      __lowercase      ,      padding=__lowercase      ,      max_length=__lowercase      ,      truncation=__lowercase      ,      pad_to_multiple_of=__lowercase      ,      return_attention_mask=__lowercase      ,      **__lowercase      ,      )

  __A  :		str       =			feature_size_hack

  # convert input values to correct format
  __A  :		Any       =			padded_inputs['input_values']
  if not isinstance(input_values[0]      ,      np.ndarray						):
   __A  :		Optional[Any]       =			[np.asarray(__lowercase      ,      dtype=np.floataa						) for array in input_values]
  elif (
      not isinstance(__lowercase      ,      np.ndarray						)
      and isinstance(input_values[0]      ,      np.ndarray						)
      and input_values[0].dtype is np.dtype(np.floataa						)
  ):
   __A  :		Dict       =			[array.astype(np.floataa						) for array in input_values]
  elif isinstance(__lowercase      ,      np.ndarray						) and input_values.dtype is np.dtype(np.floataa						):
   __A  :		Dict       =			input_values.astype(np.floataa						)

  # convert attention_mask to correct format
  __A  :		Any       =			padded_inputs.get('attention_mask'						)
  if attention_mask is not None:
   __A  :		List[Any]       =			[np.asarray(__lowercase      ,      dtype=np.intaa						) for array in attention_mask]

  # zero-mean and unit-variance normalization
  if not is_target and self.do_normalize:
   __A  :		int       =			(
       attention_mask
       if self._get_padding_strategies(__lowercase      ,      max_length=__lowercase						) is not PaddingStrategy.DO_NOT_PAD
       else None
   )
   __A  :		List[Any]       =			self.zero_mean_unit_var_norm(
       padded_inputs['input_values']      ,      attention_mask=__lowercase      ,      padding_value=self.padding_value						)

  if return_tensors is not None:
   __A  :		Tuple       =			padded_inputs.convert_to_tensors(__lowercase						)

  return padded_inputs







 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		str       =			super().to_dict()

  # Don't serialize these as they are derived from the other properties.
  __A  :		List[str]       =			['window', 'mel_filters', 'sample_size', 'sample_stride', 'n_fft', 'n_freqs']
  for name in names:
   if name in output:
    del output[name]

  return output










'''simple docstring'''
import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						_lowerCamelCase					(    snake_case_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		Dict    							=						XLMTokenizer
 __lowercase				:		Any    							=						False


 def  snake_case__   (      self						):

  """simple docstring"""







  super().setUp()

  # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  __A  :		Optional[int]       =			[
      'l',
      'o',
      'w',
      'e',
      'r',
      's',
      't',
      'i',
      'd',
      'n',
      'w</w>',
      'r</w>',
      't</w>',
      'lo',
      'low',
      'er</w>',
      'low</w>',
      'lowest</w>',
      'newer</w>',
      'wider</w>',
      '<unk>',
  ]
  __A  :		int       =			dict(zip(__lowercase      ,      range(len(__lowercase						)						)						)						)
  __A  :		str       =			['l o 123', 'lo w 1456', 'e r</w> 1789', '']

  __A  :		Dict       =			os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['vocab_file']						)
  __A  :		List[Any]       =			os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['merges_file']						)
  with open(self.vocab_file      ,      'w'						) as fp:
   fp.write(json.dumps(__lowercase						)						)
  with open(self.merges_file      ,      'w'						) as fp:
   fp.write('\n'.join(__lowercase						)						)


 def  snake_case__   (      self      ,      __lowercase						):

  """simple docstring"""







  __A  :		Any       =			'lower newer'
  __A  :		int       =			'lower newer'
  return input_text, output_text


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Any       =			XLMTokenizer(self.vocab_file      ,      self.merges_file						)

  __A  :		Optional[Any]       =			'lower'
  __A  :		Union[str, Any]       =			['low', 'er</w>']
  __A  :		Any       =			tokenizer.tokenize(__lowercase						)
  self.assertListEqual(__lowercase      ,      __lowercase						)

  __A  :		Optional[int]       =			tokens + ['<unk>']
  __A  :		Any       =			[14, 15, 20]
  self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						)      ,      __lowercase						)







 @slow
 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		List[Any]       =			XLMTokenizer.from_pretrained('xlm-mlm-en-2048'						)

  __A  :		Tuple       =			tokenizer.encode('sequence builders'      ,      add_special_tokens=__lowercase						)
  __A  :		Union[str, Any]       =			tokenizer.encode('multi-sequence build'      ,      add_special_tokens=__lowercase						)

  __A  :		int       =			tokenizer.build_inputs_with_special_tokens(__lowercase						)
  __A  :		Tuple       =			tokenizer.build_inputs_with_special_tokens(__lowercase      ,      __lowercase						)

  assert encoded_sentence == [0] + text + [1]
  assert encoded_pair == [0] + text + [1] + text_a + [1]






