



'''simple docstring'''





import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


A__    :   Tuple           =					logging.get_logger(__name__)



class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 A__				= ['''input_features''']
 def __init__(   self						:							List[str]      ,     __a						:							List[str]=80      ,     __a						:							Dict=16000      ,     __a						:							int=160      ,     __a						:							List[Any]=30      ,     __a						:							Dict=400      ,     __a						:							Dict=0.0      ,     __a						:							Dict=False      ,     **__a						:							Optional[Any]      ,     )      ->   List[Any]:







       '''simple docstring'''
       super().__init__(
           feature_size=__a      ,     sampling_rate=__a      ,     padding_value=__a      ,     return_attention_mask=__a      ,     **__a      ,     )
       __snake_case :   Optional[Any]													=    n_fft
       __snake_case :   List[str]													=    hop_length
       __snake_case :   Optional[Any]													=    chunk_length
       __snake_case :   Tuple													=    chunk_length * sampling_rate
       __snake_case :   Tuple													=    self.n_samples // hop_length
       __snake_case :   Tuple													=    sampling_rate
       __snake_case :   List[Any]													=    mel_filter_bank(
           num_frequency_bins=1 + n_fft // 2      ,     num_mel_filters=__a      ,     min_frequency=0.0      ,     max_frequency=8_0_0_0.0      ,     sampling_rate=__a      ,     norm='slaney'      ,     mel_scale='slaney'      ,     )
 def A_   (   self						:							Dict      ,     __a						:							np.array  )      ->   np.ndarray:







       '''simple docstring'''
       __snake_case :   List[str]													=    spectrogram(
           __a      ,     window_function(self.n_fft      ,     'hann'  )      ,     frame_length=self.n_fft      ,     hop_length=self.hop_length      ,     power=2.0      ,     mel_filters=self.mel_filters      ,     log_mel='log10'      ,     )
       __snake_case :   List[str]													=    log_spec[:, :-1]
       __snake_case :   int													=    np.maximum(__a      ,     log_spec.max() - 8.0  )
       __snake_case :   Tuple													=    (log_spec + 4.0) / 4.0
       return log_spec
 @staticmethod
 # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
 def A_   (   __a						:							List[np.ndarray]      ,     __a						:							List[np.ndarray]      ,     __a						:							float = 0.0  )      ->   List[np.ndarray]:







       '''simple docstring'''
       if attention_mask is not None:
             __snake_case :   Optional[Any]													=    np.array(__a      ,     np.intaa  )
             __snake_case :   Tuple													=    []

             for vector, length in zip(__a      ,     attention_mask.sum(-1  )  ):
                   __snake_case :   str													=    (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7  )
                   if length < normed_slice.shape[0]:
                         __snake_case :   List[Any]													=    padding_value

                   normed_input_values.append(__a  )
       else:
             __snake_case :   List[str]													=    [(x - x.mean()) / np.sqrt(x.var() + 1e-7  ) for x in input_values]

       return normed_input_values
 def __call__(   self						:							Dict      ,     __a						:							Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]      ,     __a						:							bool = True      ,     __a						:							Optional[int] = None      ,     __a						:							Optional[Union[str, TensorType]] = None      ,     __a						:							Optional[bool] = None      ,     __a						:							Optional[str] = "max_length"      ,     __a						:							Optional[int] = None      ,     __a						:							Optional[int] = None      ,     __a						:							Optional[bool] = None      ,     **__a						:							List[Any]      ,     )      ->   BatchFeature:







       '''simple docstring'''

       if sampling_rate is not None:
             if sampling_rate != self.sampling_rate:
                   raise ValueError(
                       f'''The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a'''
                       f''' sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input'''
                       f''' was sampled with {self.sampling_rate} and not {sampling_rate}.'''  )
       else:
             logger.warning(
                 'It is strongly recommended to pass the `sampling_rate` argument to this function. '
                 'Failing to do so can result in silent errors that might be hard to debug.'  )

       __snake_case :   int													=    isinstance(__a      ,     np.ndarray  ) and len(raw_speech.shape  ) > 1
       if is_batched_numpy and len(raw_speech.shape  ) > 2:
             raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''  )
       __snake_case :   str													=    is_batched_numpy or (
           isinstance(__a      ,     (list, tuple)  ) and (isinstance(raw_speech[0]      ,     (np.ndarray, tuple, list)  ))
       )

       if is_batched:
             __snake_case :   List[Any]													=    [np.asarray([speech]      ,     dtype=np.floataa  ).T for speech in raw_speech]
       elif not is_batched and not isinstance(__a      ,     np.ndarray  ):
             __snake_case :   Dict													=    np.asarray(__a      ,     dtype=np.floataa  )
       elif isinstance(__a      ,     np.ndarray  ) and raw_speech.dtype is np.dtype(np.floataa  ):
             __snake_case :   Any													=    raw_speech.astype(np.floataa  )

       # always return batch
       if not is_batched:
             __snake_case :   str													=    [np.asarray([raw_speech]  ).T]

       __snake_case :   Dict													=    BatchFeature({'input_features': raw_speech}  )

       # convert into correct format for padding

       __snake_case :   Optional[Any]													=    self.pad(
           __a      ,     padding=__a      ,     max_length=max_length if max_length else self.n_samples      ,     truncation=__a      ,     pad_to_multiple_of=__a      ,     return_attention_mask=return_attention_mask or do_normalize      ,     )

       # zero-mean and unit-variance normalization
       if do_normalize:
             __snake_case :   Any													=    self.zero_mean_unit_var_norm(
                 padded_inputs['input_features']      ,     attention_mask=padded_inputs['attention_mask']      ,     padding_value=self.padding_value      ,     )
             __snake_case :   Union[str, Any]													=    np.stack(padded_inputs['input_features']      ,     axis=0  )

       # make sure list is in array format
       __snake_case :   int													=    padded_inputs.get('input_features'  ).transpose(2      ,     0      ,     1  )

       __snake_case :   Optional[int]													=    [self._np_extract_fbank_features(__a  ) for waveform in input_features[0]]

       if isinstance(input_features[0]      ,     __a  ):
             __snake_case :   Any													=    [np.asarray(__a      ,     dtype=np.floataa  ) for feature in input_features]
       else:
             __snake_case :   str													=    input_features

       if return_attention_mask:
             # rescale from sample (48000) to feature (3000)
             __snake_case :   Dict													=    padded_inputs['attention_mask'][:, :: self.hop_length]

       if return_tensors is not None:
             __snake_case :   Dict													=    padded_inputs.convert_to_tensors(__a  )

       return padded_inputs





 def A_   (   self						:							Optional[int]  )      ->   Dict[str, Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    copy.deepcopy(self.__dict__  )
       __snake_case :   List[str]													=    self.__class__.__name__
       if "mel_filters" in output:
             del output["mel_filters"]
       return output








'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BertTokenizer
 A__				= BertTokenizerFast
 A__				= True
 A__				= True
 A__				= filter_non_english
 def A_   (   self						:							Optional[Any]  )      ->   Dict:







       '''simple docstring'''
       super().setUp()

       __snake_case :   List[Any]													=    [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       __snake_case :   Optional[int]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
 def A_   (   self						:							Dict      ,     __a						:							Optional[int]  )      ->   int:







       '''simple docstring'''
       __snake_case :   List[Any]													=    'UNwant\u00E9d,running'
       __snake_case :   Tuple													=    'unwanted, running'
       return input_text, output_text
 def A_   (   self						:							Tuple  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   Any													=    self.tokenizer_class(self.vocab_file  )

       __snake_case :   Tuple													=    tokenizer.tokenize('UNwant\u00E9d,running'  )
       self.assertListEqual(__a      ,     ['un', '##want', '##ed', ',', 'runn', '##ing']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     [9, 6, 7, 12, 10, 11]  )
 def A_   (   self						:							Optional[Any]  )      ->   Tuple:







       '''simple docstring'''
       if not self.test_rust_tokenizer:
             return

       __snake_case :   int													=    self.get_tokenizer()
       __snake_case :   Optional[Any]													=    self.get_rust_tokenizer()

       __snake_case :   int													=    'UNwant\u00E9d,running'

       __snake_case :   Any													=    tokenizer.tokenize(__a  )
       __snake_case :   Dict													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Optional[int]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Optional[int]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    self.get_rust_tokenizer()
       __snake_case :   Tuple													=    tokenizer.encode(__a  )
       __snake_case :   Tuple													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )

       # With lower casing
       __snake_case :   Union[str, Any]													=    self.get_tokenizer(do_lower_case=__a  )
       __snake_case :   Any													=    self.get_rust_tokenizer(do_lower_case=__a  )

       __snake_case :   Optional[int]													=    'UNwant\u00E9d,running'

       __snake_case :   List[Any]													=    tokenizer.tokenize(__a  )
       __snake_case :   List[str]													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Union[str, Any]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Union[str, Any]													=    self.get_rust_tokenizer()
       __snake_case :   Dict													=    tokenizer.encode(__a  )
       __snake_case :   str													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )
 def A_   (   self						:							Tuple  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )      ,     ['ah', '\u535A', '\u63A8', 'zz']  )
 def A_   (   self						:							List[str]  )      ->   Any:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['hello', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   int:







       '''simple docstring'''
       __snake_case :   Dict													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hällo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['h\u00E9llo']  )
 def A_   (   self						:							List[str]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   str:







       '''simple docstring'''
       __snake_case :   Tuple													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Any													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   List[Any]													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Optional[Any]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    BasicTokenizer(do_lower_case=__a      ,     never_split=['[UNK]']  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer()
       __snake_case :   List[str]													=    'a\n\'ll !!to?\'d of, can\'t.'
       __snake_case :   str													=    ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
       self.assertListEqual(tokenizer.tokenize(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       __snake_case :   Dict													=    {}
       for i, token in enumerate(__a  ):
             __snake_case :   str													=    i
       __snake_case :   List[str]													=    WordpieceTokenizer(vocab=__a      ,     unk_token='[UNK]'  )

       self.assertListEqual(tokenizer.tokenize(''  )      ,     []  )

       self.assertListEqual(tokenizer.tokenize('unwanted running'  )      ,     ['un', '##want', '##ed', 'runn', '##ing']  )

       self.assertListEqual(tokenizer.tokenize('unwantedX running'  )      ,     ['[UNK]', 'runn', '##ing']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       self.assertTrue(_is_whitespace(' '  )  )
       self.assertTrue(_is_whitespace('\t'  )  )
       self.assertTrue(_is_whitespace('\r'  )  )
       self.assertTrue(_is_whitespace('\n'  )  )
       self.assertTrue(_is_whitespace('\u00A0'  )  )

       self.assertFalse(_is_whitespace('A'  )  )
       self.assertFalse(_is_whitespace('-'  )  )
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       self.assertTrue(_is_control('\u0005'  )  )

       self.assertFalse(_is_control('A'  )  )
       self.assertFalse(_is_control(' '  )  )
       self.assertFalse(_is_control('\t'  )  )
       self.assertFalse(_is_control('\r'  )  )
 def A_   (   self						:							Any  )      ->   Union[str, Any]:







       '''simple docstring'''
       self.assertTrue(_is_punctuation('-'  )  )
       self.assertTrue(_is_punctuation('$'  )  )
       self.assertTrue(_is_punctuation('`'  )  )
       self.assertTrue(_is_punctuation('.'  )  )

       self.assertFalse(_is_punctuation('A'  )  )
       self.assertFalse(_is_punctuation(' '  )  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   int													=    self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )

       self.assertListEqual(
           [rust_tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )
 @slow
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Dict													=    self.tokenizer_class.from_pretrained('bert-base-uncased'  )

       __snake_case :   Dict													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   int													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   Optional[int]													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Optional[Any]													=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   __snake_case :   Tuple													=    tokenizer_r.encode_plus(
                       __a      ,     return_attention_mask=__a      ,     return_token_type_ids=__a      ,     return_offsets_mapping=__a      ,     add_special_tokens=__a      ,     )

                   __snake_case :   Optional[Any]													=    tokenizer_r.do_lower_case if hasattr(__a      ,     'do_lower_case'  ) else False
                   __snake_case :   Any													=    (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]      ,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
                   self.assertEqual([e[0] for e in expected_results]      ,     tokens['offset_mapping']  )





 def A_   (   self						:							Any  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Any													=    ['的', '人', '有']
       __snake_case :   List[Any]													=    ''.join(__a  )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   List[Any]													=    True
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Union[str, Any]													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Dict													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   str													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )

                   __snake_case :   Any													=    False
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   int													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   str													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Optional[Any]													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   __snake_case :   Dict													=    [
                       f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a  )
                   ]
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )



