"""simple docstring"""







import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin



class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):
				_snake_case     =  ProphetNetTokenizer
				_snake_case     =  False







				def 							__UpperCAmelCase					( self    :       Tuple						):

					"""simple docstring"""






					super().setUp()

					_lowercase :    List[str]							=							[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    '[PAD]',
					    '[MASK]',
					    'want',
					    '##want',
					    '##ed',
					    'wa',
					    'un',
					    'runn',
					    '##ing',
					    ',',
					    'low',
					    'lowest',
					]
					_lowercase :    Union[str, Any]							=							os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file			,					'w'			,					encoding='utf-8'						) as vocab_writer:
						vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       Dict						):

					"""simple docstring"""






					_lowercase :    Any							=							'UNwant\u00E9d,running'
					_lowercase :    Tuple							=							'unwanted, running'
					return input_text, output_text







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							self.tokenizer_class(self.vocab_file						)

					_lowercase :    Tuple							=							tokenizer.tokenize('UNwant\u00E9d,running'						)
					self.assertListEqual(lowerCamelCase_			,					['un', '##want', '##ed', ',', 'runn', '##ing']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_						)			,					[9, 6, 7, 1_2, 1_0, 1_1]						)







				def 							__UpperCAmelCase					( self    :       Optional[int]						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)			,					['ah', '\u535A', '\u63A8', 'zz']						)







				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					_lowercase :    List[str]							=							BasicTokenizer(do_lower_case=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)			,					['hello', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							BasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hällo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['h\u00E9llo']						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							BasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hallo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					_lowercase :    str							=							BasicTokenizer(do_lower_case=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hallo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							BasicTokenizer(do_lower_case=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)







				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							BasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)







				def 							__UpperCAmelCase					( self    :       List[Any]						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							BasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)







				def 							__UpperCAmelCase					( self    :       Optional[int]						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							BasicTokenizer(do_lower_case=lowerCamelCase_			,					never_split=['[UNK]']						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)







				def 							__UpperCAmelCase					( self    :       Optional[int]						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

					_lowercase :    List[Any]							=							{}
					for i, token in enumerate(lowerCamelCase_						):
						_lowercase :    Any							=							i
					_lowercase :    Any							=							WordpieceTokenizer(vocab=lowerCamelCase_			,					unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)			,					[]						)

					self.assertListEqual(tokenizer.tokenize('unwanted running'						)			,					['un', '##want', '##ed', 'runn', '##ing']						)

					self.assertListEqual(tokenizer.tokenize('unwantedX running'						)			,					['[UNK]', 'runn', '##ing']						)







				@require_torch
				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

					_lowercase :    Union[str, Any]							=							['A long paragraph for summarization.', 'Another paragraph for summarization.']
					_lowercase :    int							=							[1_0_3_7, 2_1_4_6, 2_0_4_2_3, 2_0_0_5, 7_6_8_0, 7_8_4_9, 3_9_8_9, 1_0_1_2, 1_0_2]
					_lowercase :    List[str]							=							tokenizer(lowerCamelCase_			,					padding=lowerCamelCase_			,					return_tensors='pt'						)
					self.assertIsInstance(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Dict							=							list(batch.input_ids.numpy()[0]						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					self.assertEqual((2, 9)			,					batch.input_ids.shape						)
					self.assertEqual((2, 9)			,					batch.attention_mask.shape						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					self.assertTrue(_is_whitespace(' '						)						)
					self.assertTrue(_is_whitespace('\t'						)						)
					self.assertTrue(_is_whitespace('\r'						)						)
					self.assertTrue(_is_whitespace('\n'						)						)
					self.assertTrue(_is_whitespace('\u00A0'						)						)

					self.assertFalse(_is_whitespace('A'						)						)
					self.assertFalse(_is_whitespace('-'						)						)







				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					self.assertTrue(_is_control('\u0005'						)						)

					self.assertFalse(_is_control('A'						)						)
					self.assertFalse(_is_control(' '						)						)
					self.assertFalse(_is_control('\t'						)						)
					self.assertFalse(_is_control('\r'						)						)







				def 							__UpperCAmelCase					( self    :       Optional[int]						):

					"""simple docstring"""






					self.assertTrue(_is_punctuation('-'						)						)
					self.assertTrue(_is_punctuation('$'						)						)
					self.assertTrue(_is_punctuation('`'						)						)
					self.assertTrue(_is_punctuation('.'						)						)

					self.assertFalse(_is_punctuation('A'						)						)
					self.assertFalse(_is_punctuation(' '						)						)







				@slow
				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					_lowercase :    Tuple							=							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

					_lowercase :    List[Any]							=							tokenizer.encode('sequence builders'			,					add_special_tokens=lowerCamelCase_						)
					_lowercase :    Union[str, Any]							=							tokenizer.encode('multi-sequence build'			,					add_special_tokens=lowerCamelCase_						)

					_lowercase :    List[Any]							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_						)
					_lowercase :    int							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_			,					lowerCamelCase_						)

					assert encoded_sentence == text + [1_0_2]
					assert encoded_pair == text + [1_0_2] + text_a + [1_0_2]




"""simple docstring"""







from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


SCREAMING_SNAKE_CASE									=						logging.get_logger(__name__)



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  ["input_features", "attention_mask"]







				def __init__( self    :       int			,					lowerCamelCase_    :       List[str]=8_0			,					lowerCamelCase_    :       Tuple=1_6_0_0_0			,					lowerCamelCase_    :       Union[str, Any]=0.0			,					lowerCamelCase_    :       List[Any]=1_0			,					lowerCamelCase_    :       List[str]=2_5			,					lowerCamelCase_    :       List[Any]="hamming_window"			,					lowerCamelCase_    :       Tuple=3_2768.0			,					lowerCamelCase_    :       int=0.97			,					lowerCamelCase_    :       Optional[int]=1.0			,					lowerCamelCase_    :       Optional[Any]=True			,					lowerCamelCase_    :       Union[str, Any]=True			,					lowerCamelCase_    :       Optional[int]=False			,					**lowerCamelCase_    :       Optional[Any]			,					):

					"""simple docstring"""






					super().__init__(feature_size=lowerCamelCase_			,					sampling_rate=lowerCamelCase_			,					padding_value=lowerCamelCase_			,					**lowerCamelCase_						)

					_lowercase :    Dict							=							feature_size
					_lowercase :    Dict							=							sampling_rate
					_lowercase :    Tuple							=							padding_value
					_lowercase :    int							=							hop_length
					_lowercase :    Any							=							win_length
					_lowercase :    Union[str, Any]							=							frame_signal_scale
					_lowercase :    Tuple							=							preemphasis_coeff
					_lowercase :    Tuple							=							mel_floor
					_lowercase :    Tuple							=							normalize_means
					_lowercase :    List[Any]							=							normalize_vars
					_lowercase :    List[str]							=							win_function
					_lowercase :    int							=							return_attention_mask

					_lowercase :    Optional[Any]							=							win_length * sampling_rate // 1_0_0_0
					_lowercase :    Tuple							=							hop_length * sampling_rate // 1_0_0_0

					_lowercase :    str							=							optimal_fft_length(self.sample_size						)
					_lowercase :    Dict							=							(self.n_fft // 2) + 1







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       np.array						):

					"""simple docstring"""






					if self.win_function == "hamming_window":
						_lowercase :    List[Any]							=							window_function(window_length=self.sample_size			,					name=self.win_function			,					periodic=lowerCamelCase_						)
					else:
						_lowercase :    Union[str, Any]							=							window_function(window_length=self.sample_size			,					name=self.win_function						)

					_lowercase :    Tuple							=							mel_filter_bank(
					    num_frequency_bins=self.n_freqs			,					num_mel_filters=self.feature_size			,					min_frequency=0.0			,					max_frequency=self.sampling_rate / 2.0			,					sampling_rate=self.sampling_rate			,					)

					_lowercase :    Tuple							=							spectrogram(
					    one_waveform * self.frame_signal_scale			,					window=lowerCamelCase_			,					frame_length=self.sample_size			,					hop_length=self.sample_stride			,					fft_length=self.n_fft			,					center=lowerCamelCase_			,					preemphasis=self.preemphasis_coeff			,					mel_filters=lowerCamelCase_			,					mel_floor=self.mel_floor			,					log_mel='log'			,					)
					return msfc_features.T







				def 							__UpperCAmelCase					( self    :       Optional[Any]			,					lowerCamelCase_    :       Dict			,					lowerCamelCase_    :       Dict			,					lowerCamelCase_    :       Tuple						):

					"""simple docstring"""






					if self.normalize_means:
						_lowercase :    Optional[int]							=							x[:input_length].mean(axis=0						)
						_lowercase :    int							=							np.subtract(lowerCamelCase_			,					lowerCamelCase_						)
					if self.normalize_vars:
						_lowercase :    int							=							x[:input_length].std(axis=0						)
						_lowercase :    Optional[Any]							=							np.divide(lowerCamelCase_			,					lowerCamelCase_						)

					if input_length < x.shape[0]:
						_lowercase :    Dict							=							padding_value

					# make sure array is in float32
					_lowercase :    Tuple							=							x.astype(np.floataa						)

					return x







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       List[np.ndarray]			,					lowerCamelCase_    :       Optional[np.ndarray] = None						):

					"""simple docstring"""






					_lowercase :    Dict							=							attention_mask.sum(-1						) if attention_mask is not None else [x.shape[0] for x in input_features]
					return [self._normalize_one(lowerCamelCase_			,					lowerCamelCase_			,					self.padding_value						) for x, n in zip(lowerCamelCase_			,					lowerCamelCase_						)]







				def __call__( self    :       Dict			,					lowerCamelCase_    :       Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]			,					lowerCamelCase_    :       Union[bool, str, PaddingStrategy] = False			,					lowerCamelCase_    :       Optional[int] = None			,					lowerCamelCase_    :       bool = False			,					lowerCamelCase_    :       Optional[int] = None			,					lowerCamelCase_    :       Optional[bool] = None			,					lowerCamelCase_    :       Optional[Union[str, TensorType]] = None			,					lowerCamelCase_    :       Optional[int] = None			,					**lowerCamelCase_    :       Optional[int]			,					):

					"""simple docstring"""






					if sampling_rate is not None:
						if sampling_rate != self.sampling_rate:
							raise ValueError(
							    F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
							    F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
							    F''' {self.sampling_rate} and not {sampling_rate}.'''						)
					else:
						logger.warning(
						    'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
						    'Failing to do so can result in silent errors that might be hard to debug.'						)

					_lowercase :    Optional[Any]							=							isinstance(lowerCamelCase_			,					np.ndarray						) and len(raw_speech.shape						) > 1
					if is_batched_numpy and len(raw_speech.shape						) > 2:
						raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''						)
					_lowercase :    Optional[int]							=							is_batched_numpy or (
					    isinstance(lowerCamelCase_			,					(list, tuple)						) and (isinstance(raw_speech[0]			,					(np.ndarray, tuple, list)						))
					)

					if is_batched:
						_lowercase :    str							=							[np.asarray(lowerCamelCase_			,					dtype=np.floataa						) for speech in raw_speech]
					elif not is_batched and not isinstance(lowerCamelCase_			,					np.ndarray						):
						_lowercase :    Tuple							=							np.asarray(lowerCamelCase_			,					dtype=np.floataa						)
					elif isinstance(lowerCamelCase_			,					np.ndarray						) and raw_speech.dtype is np.dtype(np.floataa						):
						_lowercase :    Optional[int]							=							raw_speech.astype(np.floataa						)

					# always return batch
					if not is_batched:
						_lowercase :    str							=							[raw_speech]

					# extract fbank features
					_lowercase :    Optional[Any]							=							[self._extract_mfsc_features(lowerCamelCase_						) for one_waveform in raw_speech]

					# convert into correct format for padding
					_lowercase :    Optional[int]							=							BatchFeature({'input_features': features}						)

					_lowercase :    Tuple							=							self.pad(
					    lowerCamelCase_			,					padding=lowerCamelCase_			,					max_length=lowerCamelCase_			,					truncation=lowerCamelCase_			,					pad_to_multiple_of=lowerCamelCase_			,					return_attention_mask=lowerCamelCase_			,					**lowerCamelCase_			,					)
					# make sure list is in array format
					_lowercase :    Dict							=							padded_inputs.get('input_features'						)
					if isinstance(input_features[0]			,					lowerCamelCase_						):
						_lowercase :    List[str]							=							[np.asarray(lowerCamelCase_			,					dtype=np.floataa						) for feature in input_features]

					_lowercase :    List[Any]							=							padded_inputs.get('attention_mask'						)
					if attention_mask is not None:
						_lowercase :    Union[str, Any]							=							[np.asarray(lowerCamelCase_			,					dtype=np.intaa						) for array in attention_mask]

					if self.normalize_means or self.normalize_vars:
						_lowercase :    int							=							(
						    np.array(lowerCamelCase_			,					dtype=np.intaa						)
						    if self._get_padding_strategies(lowerCamelCase_			,					max_length=lowerCamelCase_						) is not PaddingStrategy.DO_NOT_PAD
						    and padding
						    else None
						)
						_lowercase :    List[Any]							=							self.normalize(
						    padded_inputs['input_features']			,					attention_mask=lowerCamelCase_						)

					if return_tensors is not None:
						_lowercase :    Union[str, Any]							=							padded_inputs.convert_to_tensors(lowerCamelCase_						)

					return padded_inputs



