


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__lowerCAmelCase			=							logging.get_logger(__name__)

__lowerCAmelCase			=							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase			=							{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCAmelCase			=							{
    '''roberta-base''': 5_12,
    '''roberta-large''': 5_12,
    '''roberta-large-mnli''': 5_12,
    '''distilroberta-base''': 5_12,
    '''roberta-base-openai-detector''': 5_12,
    '''roberta-large-openai-detector''': 5_12,
}





class   __a      (   __UpperCamelCase				):
 __lowercase			:							Any       					=			VOCAB_FILES_NAMES
 __lowercase			:							List[str]       					=			PRETRAINED_VOCAB_FILES_MAP
 __lowercase			:							Tuple       					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase			:							Optional[int]       					=			['input_ids', 'attention_mask']
 __lowercase			:							Tuple       					=			RobertaTokenizer
 def __init__(       self    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__="replace"    ,			lowerCAmelCase__="<s>"    ,			lowerCAmelCase__="</s>"    ,			lowerCAmelCase__="</s>"    ,			lowerCAmelCase__="<s>"    ,			lowerCAmelCase__="<unk>"    ,			lowerCAmelCase__="<pad>"    ,			lowerCAmelCase__="<mask>"    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=True    ,			**lowerCAmelCase__    ,			) ->    List[Any]:




        '''simple docstring'''
        super().__init__(
            lowerCAmelCase__    ,			lowerCAmelCase__    ,			tokenizer_file=lowerCAmelCase__    ,			errors=lowerCAmelCase__    ,			bos_token=lowerCAmelCase__    ,			eos_token=lowerCAmelCase__    ,			sep_token=lowerCAmelCase__    ,			cls_token=lowerCAmelCase__    ,			unk_token=lowerCAmelCase__    ,			pad_token=lowerCAmelCase__    ,			mask_token=lowerCAmelCase__    ,			add_prefix_space=lowerCAmelCase__    ,			trim_offsets=lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)

        lowercase__:       str        =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
        if pre_tok_state.get('add_prefix_space'    ,			lowerCAmelCase__			) != add_prefix_space:
               lowercase__:       str        =       getattr(lowerCAmelCase__    ,			pre_tok_state.pop('type'			)			)
               lowercase__:       int        =       add_prefix_space
               lowercase__:       Union[str, Any]        =       pre_tok_class(**lowerCAmelCase__			)

        lowercase__:       Dict        =       add_prefix_space

        lowercase__:       Any        =       'post_processor'
        lowercase__:       int        =       getattr(self.backend_tokenizer    ,			lowerCAmelCase__    ,			lowerCAmelCase__			)
        if tokenizer_component_instance:
               lowercase__:       List[str]        =       json.loads(tokenizer_component_instance.__getstate__()			)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                      lowercase__:       List[str]        =       tuple(state['sep']			)
               if "cls" in state:
                      lowercase__:       Optional[Any]        =       tuple(state['cls']			)

               lowercase__:       List[str]        =       False

               if state.get('add_prefix_space'    ,			lowerCAmelCase__			) != add_prefix_space:
                      lowercase__:       List[Any]        =       add_prefix_space
                      lowercase__:       Tuple        =       True

               if state.get('trim_offsets'    ,			lowerCAmelCase__			) != trim_offsets:
                      lowercase__:       Union[str, Any]        =       trim_offsets
                      lowercase__:       Dict        =       True

               if changes_to_apply:
                      lowercase__:       Union[str, Any]        =       getattr(lowerCAmelCase__    ,			state.pop('type'			)			)
                      lowercase__:       Optional[int]        =       component_class(**lowerCAmelCase__			)
                      setattr(self.backend_tokenizer    ,			lowerCAmelCase__    ,			lowerCAmelCase__			)
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    str:




        '''simple docstring'''
        if self._mask_token is None:
               if self.verbose:
                      logger.error('Using mask_token, but it is not set yet.'			)
               return None
        return str(self._mask_token			)
 @mask_token.setter
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    List[str]:




        '''simple docstring'''
        lowercase__:       Optional[int]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else value
        lowercase__:       Dict        =       value
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			) ->    BatchEncoding:




        '''simple docstring'''
        lowercase__:       Optional[Any]        =       kwargs.get('is_split_into_words'    ,			lowerCAmelCase__			)
        assert self.add_prefix_space or not is_split_into_words, (
            F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*lowerCAmelCase__    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			) ->    BatchEncoding:




        '''simple docstring'''
        lowercase__:       Tuple        =       kwargs.get('is_split_into_words'    ,			lowerCAmelCase__			)

        assert self.add_prefix_space or not is_split_into_words, (
            F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*lowerCAmelCase__    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    Tuple[str]:




        '''simple docstring'''
        lowercase__:       str        =       self._tokenizer.model.save(lowerCAmelCase__    ,			name=lowerCAmelCase__			)
        return tuple(lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__=None			) ->    Union[str, Any]:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
               return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    List[int]:




        '''simple docstring'''
        lowercase__:       Any        =       [self.sep_token_id]
        lowercase__:       int        =       [self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep			) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


__lowerCAmelCase			=							logging.get_logger(__name__)





class   __a      (   __UpperCamelCase				):
 __lowercase			:							Tuple       					=			['input_values', 'attention_mask']
 def __init__(       self    ,			lowerCAmelCase__ = 1    ,			lowerCAmelCase__ = 16_000    ,			lowerCAmelCase__ = 0.0    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = 80    ,			lowerCAmelCase__ = 16    ,			lowerCAmelCase__ = 64    ,			lowerCAmelCase__ = "hann_window"    ,			lowerCAmelCase__ = 1.0    ,			lowerCAmelCase__ = 80    ,			lowerCAmelCase__ = 7_600    ,			lowerCAmelCase__ = 1E-10    ,			lowerCAmelCase__ = 2    ,			lowerCAmelCase__ = True    ,			**lowerCAmelCase__    ,			) ->    List[Any]:




        '''simple docstring'''
        super().__init__(feature_size=lowerCAmelCase__    ,			sampling_rate=lowerCAmelCase__    ,			padding_value=lowerCAmelCase__    ,			**lowerCAmelCase__			)
        lowercase__:       Dict        =       do_normalize
        lowercase__:       Optional[Any]        =       return_attention_mask

        lowercase__:       str        =       num_mel_bins
        lowercase__:       Dict        =       hop_length
        lowercase__:       Dict        =       win_length
        lowercase__:       Optional[int]        =       win_function
        lowercase__:       Any        =       frame_signal_scale
        lowercase__:       Tuple        =       fmin
        lowercase__:       Tuple        =       fmax
        lowercase__:       Dict        =       mel_floor
        lowercase__:       int        =       reduction_factor

        lowercase__:       List[Any]        =       win_length * sampling_rate // 1_000
        lowercase__:       Optional[Any]        =       hop_length * sampling_rate // 1_000
        lowercase__:       Optional[int]        =       optimal_fft_length(self.sample_size			)
        lowercase__:       Optional[Any]        =       (self.n_fft // 2) + 1

        lowercase__:       str        =       window_function(window_length=self.sample_size    ,			name=self.win_function    ,			periodic=lowerCAmelCase__			)

        lowercase__:       Optional[Any]        =       mel_filter_bank(
            num_frequency_bins=self.n_freqs    ,			num_mel_filters=self.num_mel_bins    ,			min_frequency=self.fmin    ,			max_frequency=self.fmax    ,			sampling_rate=self.sampling_rate    ,			norm='slaney'    ,			mel_scale='slaney'    ,			)

        if frame_signal_scale != 1.0:
               warnings.warn(
                   'The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers'    ,			lowerCAmelCase__    ,			)
        if reduction_factor != 2.0:
               warnings.warn(
                   'The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers'    ,			lowerCAmelCase__    ,			)
 @staticmethod
 # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
 def 		SCREAMING_SNAKE_CASE__       (       lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = 0.0			) ->    List[np.ndarray]:




        '''simple docstring'''
        if attention_mask is not None:
               lowercase__:       List[str]        =       np.array(lowerCAmelCase__    ,			np.intaa			)
               lowercase__:       Tuple        =       []

               for vector, length in zip(lowerCAmelCase__    ,			attention_mask.sum(-1			)			):
                      lowercase__:       int        =       (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7			)
                      if length < normed_slice.shape[0]:
                             lowercase__:       Tuple        =       padding_value

                      normed_input_values.append(lowerCAmelCase__			)
        else:
               lowercase__:       Union[str, Any]        =       [(x - x.mean()) / np.sqrt(x.var() + 1E-7			) for x in input_values]

        return normed_input_values
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			) ->    np.ndarray:




        '''simple docstring'''
        lowercase__:       List[str]        =       spectrogram(
            lowerCAmelCase__    ,			window=self.window    ,			frame_length=self.sample_size    ,			hop_length=self.sample_stride    ,			fft_length=self.n_fft    ,			mel_filters=self.mel_filters    ,			mel_floor=self.mel_floor    ,			log_mel='log10'    ,			)
        return log_mel_spec.T
 def __call__(       self    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			**lowerCAmelCase__    ,			) ->    BatchFeature:




        '''simple docstring'''
        if audio is None and audio_target is None:
               raise ValueError('You must provide either `audio` or `audio_target` values.'			)

        if sampling_rate is not None:
               if sampling_rate != self.sampling_rate:
                      raise ValueError(
                          F'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
                          F' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'
                          F' {self.sampling_rate} and not {sampling_rate}.'			)
        else:
               logger.warning(
                   'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
                   'Failing to do so can result in silent errors that might be hard to debug.'			)

        if audio is not None:
               lowercase__:       Dict        =       self._process_audio(
                   lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)
        else:
               lowercase__:       str        =       None

        if audio_target is not None:
               lowercase__:       List[str]        =       self._process_audio(
                   lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)

               if inputs is None:
                      return inputs_target
               else:
                      lowercase__:       int        =       inputs_target['input_values']
                      lowercase__:       List[str]        =       inputs_target.get('attention_mask'			)
                      if decoder_attention_mask is not None:
                             lowercase__:       Optional[int]        =       decoder_attention_mask

        return inputs
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = False    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None    ,			**lowerCAmelCase__    ,			) ->    BatchFeature:




        '''simple docstring'''
        lowercase__:       int        =       isinstance(lowerCAmelCase__    ,			np.ndarray			) and len(speech.shape			) > 1
        if is_batched_numpy and len(speech.shape			) > 2:
               raise ValueError(F'Only mono-channel audio is supported for input to {self}'			)
        lowercase__:       Tuple        =       is_batched_numpy or (
            isinstance(lowerCAmelCase__    ,			(list, tuple)			) and (isinstance(speech[0]    ,			(np.ndarray, tuple, list)			))
        )

        if is_batched:
               lowercase__:       Dict        =       [np.asarray(lowerCAmelCase__    ,			dtype=np.floataa			) for speech in speech]
        elif not is_batched and not isinstance(lowerCAmelCase__    ,			np.ndarray			):
               lowercase__:       Optional[Any]        =       np.asarray(lowerCAmelCase__    ,			dtype=np.floataa			)
        elif isinstance(lowerCAmelCase__    ,			np.ndarray			) and speech.dtype is np.dtype(np.floataa			):
               lowercase__:       Optional[Any]        =       speech.astype(np.floataa			)

        # always return batch
        if not is_batched:
               lowercase__:       Optional[int]        =       [speech]

        # needed to make pad() work on spectrogram inputs
        lowercase__:       str        =       self.feature_size

        # convert into correct format for padding
        if is_target:
               lowercase__:       int        =       [self._extract_mel_features(lowerCAmelCase__			) for waveform in speech]
               lowercase__:       Dict        =       BatchFeature({'input_values': features}			)
               lowercase__:       Union[str, Any]        =       self.num_mel_bins
        else:
               lowercase__:       Union[str, Any]        =       BatchFeature({'input_values': speech}			)

        lowercase__:       Dict        =       self.pad(
            lowerCAmelCase__    ,			padding=lowerCAmelCase__    ,			max_length=lowerCAmelCase__    ,			truncation=lowerCAmelCase__    ,			pad_to_multiple_of=lowerCAmelCase__    ,			return_attention_mask=lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)

        lowercase__:       List[str]        =       feature_size_hack

        # convert input values to correct format
        lowercase__:       Union[str, Any]        =       padded_inputs['input_values']
        if not isinstance(input_values[0]    ,			np.ndarray			):
               lowercase__:       List[Any]        =       [np.asarray(lowerCAmelCase__    ,			dtype=np.floataa			) for array in input_values]
        elif (
            not isinstance(lowerCAmelCase__    ,			np.ndarray			)
            and isinstance(input_values[0]    ,			np.ndarray			)
            and input_values[0].dtype is np.dtype(np.floataa			)
        ):
               lowercase__:       Dict        =       [array.astype(np.floataa			) for array in input_values]
        elif isinstance(lowerCAmelCase__    ,			np.ndarray			) and input_values.dtype is np.dtype(np.floataa			):
               lowercase__:       Tuple        =       input_values.astype(np.floataa			)

        # convert attention_mask to correct format
        lowercase__:       Tuple        =       padded_inputs.get('attention_mask'			)
        if attention_mask is not None:
               lowercase__:       str        =       [np.asarray(lowerCAmelCase__    ,			dtype=np.intaa			) for array in attention_mask]

        # zero-mean and unit-variance normalization
        if not is_target and self.do_normalize:
               lowercase__:       Tuple        =       (
                   attention_mask
                   if self._get_padding_strategies(lowerCAmelCase__    ,			max_length=lowerCAmelCase__			) is not PaddingStrategy.DO_NOT_PAD
                   else None
               )
               lowercase__:       str        =       self.zero_mean_unit_var_norm(
                   padded_inputs['input_values']    ,			attention_mask=lowerCAmelCase__    ,			padding_value=self.padding_value			)

        if return_tensors is not None:
               lowercase__:       Union[str, Any]        =       padded_inputs.convert_to_tensors(lowerCAmelCase__			)

        return padded_inputs
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Dict[str, Any]:




        '''simple docstring'''
        lowercase__:       int        =       super().to_dict()

        # Don't serialize these as they are derived from the other properties.
        lowercase__:       str        =       ['window', 'mel_filters', 'sample_size', 'sample_stride', 'n_fft', 'n_freqs']
        for name in names:
               if name in output:
                      del output[name]

        return output


