




"""simple docstring"""

import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a							      =     logging.get_logger(__name__)


a							      =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

a							      =     {
    '''vocab_file''': {
        '''facebook/blenderbot_small-90M''': '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''facebook/blenderbot_small-90M''': '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'''
    },
    '''tokenizer_config_file''': {
        '''facebook/blenderbot_small-90M''': (
            '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'''
        )
    },
}

a							      =     {'''facebook/blenderbot_small-90M''': 512}






def 		_snake_case	(   _snake_case				:   List[str]				)       ->   Union[str, Any]:






      '''simple docstring'''


      _A     =  set()
      _A     =  word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            _A     =  char

      _A     =  set(_snake_case				)
      return pairs







class 							lowercase_			(	__lowerCAmelCase				):



 '''simple docstring'''
 UpperCAmelCase       :	List[str]     =      VOCAB_FILES_NAMES
 UpperCAmelCase       :	Optional[int]     =      PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase       :	int     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase       :	List[Any]     =      ['''input_ids''', '''attention_mask''']

 def __init__(					self			:			int       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Tuple="__start__"       ,				_UpperCAmelCase			:			int="__end__"       ,				_UpperCAmelCase			:			Optional[Any]="__unk__"       ,				_UpperCAmelCase			:			List[str]="__null__"       ,				**_UpperCAmelCase			:			str       ,				):
       super().__init__(unk_token=_UpperCAmelCase       ,				bos_token=_UpperCAmelCase       ,				eos_token=_UpperCAmelCase       ,				pad_token=_UpperCAmelCase       ,				**_UpperCAmelCase						)

       with open(_UpperCAmelCase       ,				encoding='utf-8'						) as vocab_handle:
             _A     =  json.load(_UpperCAmelCase						)
       _A     =  {v: k for k, v in self.encoder.items()}
       with open(_UpperCAmelCase       ,				encoding='utf-8'						) as merges_handle:
             _A     =  merges_handle.read().split('\n'						)[1:-1]
       _A     =  [tuple(merge.split()						) for merge in merges]
       _A     =  dict(zip(_UpperCAmelCase       ,				range(len(_UpperCAmelCase						)						)						)						)
       _A     =  {}

 @property
 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       return len(self.encoder						)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]						):
       return dict(self.encoder       ,				**self.added_tokens_encoder						)

 def      lowerCAmelCase_				(					self			:			Optional[int]       ,				_UpperCAmelCase			:			str						):
       if token in self.cache:
             return self.cache[token]
       _A     =  re.sub('([.,!?()])'       ,				r' \1'       ,				_UpperCAmelCase						)
       _A     =  re.sub('(\')'       ,				r' \1 '       ,				_UpperCAmelCase						)
       _A     =  re.sub(r'\s{2,}'       ,				' '       ,				_UpperCAmelCase						)
       if "\n" in token:
             _A     =  token.replace('\n'       ,				' __newln__'						)

       _A     =  token.split(' '						)
       _A     =  []
       for token in tokens:
             if not len(_UpperCAmelCase						):
                   continue

             _A     =  token.lower()
             _A     =  tuple(_UpperCAmelCase						)
             _A     =  tuple(list(word[:-1]						) + [word[-1] + '</w>']						)
             _A     =  get_pairs(_UpperCAmelCase						)

             if not pairs:
                   words.append(_UpperCAmelCase						)
                   continue

             while True:
                   _A     =  min(_UpperCAmelCase       ,				key=lambda _UpperCAmelCase						: self.bpe_ranks.get(_UpperCAmelCase       ,				float('inf'						)						)						)
                   if bigram not in self.bpe_ranks:
                         break
                   _A  ,       _A     =  bigram
                   _A     =  []
                   _A     =  0

                   while i < len(_UpperCAmelCase						):
                         try:
                               _A     =  word.index(_UpperCAmelCase       ,				_UpperCAmelCase						)
                               new_word.extend(word[i:j]						)
                               _A     =  j
                         except ValueError:
                               new_word.extend(word[i:]						)
                               break

                         if word[i] == first and i < len(_UpperCAmelCase						) - 1 and word[i + 1] == second:
                               new_word.append(first + second						)
                               i += 2
                         else:
                               new_word.append(word[i]						)
                               i += 1
                   _A     =  tuple(_UpperCAmelCase						)
                   _A     =  new_word
                   if len(_UpperCAmelCase						) == 1:
                         break
                   else:
                         _A     =  get_pairs(_UpperCAmelCase						)
             _A     =  '@@ '.join(_UpperCAmelCase						)
             _A     =  word[:-4]

             _A     =  word
             words.append(_UpperCAmelCase						)
       return " ".join(_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Optional[int]       ,				_UpperCAmelCase			:			str						):
       _A     =  []

       _A     =  re.findall(r'\S+\n?'       ,				_UpperCAmelCase						)

       for token in words:
             split_tokens.extend(list(self.bpe(_UpperCAmelCase						).split(' '						)						)						)
       return split_tokens

 def      lowerCAmelCase_				(					self			:			List[str]       ,				_UpperCAmelCase			:			str						):
       _A     =  token.lower()
       return self.encoder.get(_UpperCAmelCase       ,				self.encoder.get(self.unk_token						)						)

 def      lowerCAmelCase_				(					self			:			Tuple       ,				_UpperCAmelCase			:			int						):
       return self.decoder.get(_UpperCAmelCase       ,				self.unk_token						)

 def      lowerCAmelCase_				(					self			:			Dict       ,				_UpperCAmelCase			:			List[str]						):
       _A     =  ' '.join(_UpperCAmelCase						).replace('@@ '       ,				''						).strip()
       return out_string



 def      lowerCAmelCase_				(					self			:			Optional[int]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[str] = None						):
       if not os.path.isdir(_UpperCAmelCase						):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
             return
       _A     =  os.path.join(
           _UpperCAmelCase       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
       _A     =  os.path.join(
           _UpperCAmelCase       ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

       with open(_UpperCAmelCase       ,				'w'       ,				encoding='utf-8'						) as f:
             f.write(json.dumps(self.encoder       ,				indent=2       ,				sort_keys=_UpperCAmelCase       ,				ensure_ascii=_UpperCAmelCase						) + '\n'						)

       _A     =  0
       with open(_UpperCAmelCase       ,				'w'       ,				encoding='utf-8'						) as writer:
             writer.write('#version: 0.2\n'						)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,				key=lambda _UpperCAmelCase						: kv[1]						):
                   if index != token_index:
                         logger.warning(
                             F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                             ' Please check that the tokenizer is not corrupted!'						)
                         _A     =  token_index
                   writer.write(' '.join(_UpperCAmelCase						) + '\n'						)
                   index += 1

       return vocab_file, merge_file



'''simple docstring'''





import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    int				=							["input_values", "attention_mask"]







 def __init__(					self		:  Any			,				snake_case__		:  int = 1			,				snake_case__		:  int = 1_60_00			,				snake_case__		:  float = 0.0			,				snake_case__		:  bool = False			,				snake_case__		:  int = 80			,				snake_case__		:  int = 16			,				snake_case__		:  int = 64			,				snake_case__		:  str = "hann_window"			,				snake_case__		:  float = 1.0			,				snake_case__		:  float = 80			,				snake_case__		:  float = 76_00			,				snake_case__		:  float = 1e-10			,				snake_case__		:  int = 2			,				snake_case__		:  bool = True			,				**snake_case__		:  List[Any]			,				):



   '''simple docstring'''




   super().__init__(feature_size=snake_case__			,				sampling_rate=snake_case__			,				padding_value=snake_case__			,				**snake_case__ )
   UpperCAmelCase__      :					int				       =							do_normalize
   UpperCAmelCase__      :					Tuple				       =							return_attention_mask

   UpperCAmelCase__      :					Union[str, Any]				       =							num_mel_bins
   UpperCAmelCase__      :					List[str]				       =							hop_length
   UpperCAmelCase__      :					List[str]				       =							win_length
   UpperCAmelCase__      :					Union[str, Any]				       =							win_function
   UpperCAmelCase__      :					str				       =							frame_signal_scale
   UpperCAmelCase__      :					int				       =							fmin
   UpperCAmelCase__      :					Union[str, Any]				       =							fmax
   UpperCAmelCase__      :					Optional[Any]				       =							mel_floor
   UpperCAmelCase__      :					List[str]				       =							reduction_factor

   UpperCAmelCase__      :					str				       =							win_length * sampling_rate // 10_00
   UpperCAmelCase__      :					Union[str, Any]				       =							hop_length * sampling_rate // 10_00
   UpperCAmelCase__      :					Optional[Any]				       =							optimal_fft_length(self.sample_size )
   UpperCAmelCase__      :					str				       =							(self.n_fft // 2) + 1

   UpperCAmelCase__      :					Dict				       =							window_function(window_length=self.sample_size			,				name=self.win_function			,				periodic=snake_case__ )

   UpperCAmelCase__      :					List[str]				       =							mel_filter_bank(
       num_frequency_bins=self.n_freqs			,				num_mel_filters=self.num_mel_bins			,				min_frequency=self.fmin			,				max_frequency=self.fmax			,				sampling_rate=self.sampling_rate			,				norm="slaney"			,				mel_scale="slaney"			,				)

   if frame_signal_scale != 1.0:
     warnings.warn(
         "The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers"			,				snake_case__			,				)
   if reduction_factor != 2.0:
     warnings.warn(
         "The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers"			,				snake_case__			,				)







 @staticmethod
 # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
 def 							UpperCamelCase  (					snake_case__		:  List[np.ndarray]			,				snake_case__		:  List[np.ndarray]			,				snake_case__		:  float = 0.0 ):



   '''simple docstring'''




   if attention_mask is not None:
     UpperCAmelCase__      :					Tuple				       =							np.array(snake_case__			,				np.intaa )
     UpperCAmelCase__      :					Tuple				       =							[]

     for vector, length in zip(snake_case__			,				attention_mask.sum(-1 ) ):
       UpperCAmelCase__      :					List[str]				       =							(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7 )
       if length < normed_slice.shape[0]:
         UpperCAmelCase__      :					Optional[int]				       =							padding_value

       normed_input_values.append(snake_case__ )
   else:
     UpperCAmelCase__      :					Optional[int]				       =							[(x - x.mean()) / np.sqrt(x.var() + 1e-7 ) for x in input_values]

   return normed_input_values







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  np.ndarray			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							spectrogram(
       snake_case__			,				window=self.window			,				frame_length=self.sample_size			,				hop_length=self.sample_stride			,				fft_length=self.n_fft			,				mel_filters=self.mel_filters			,				mel_floor=self.mel_floor			,				log_mel="log10"			,				)
   return log_mel_spec.T







 def __call__(					self		:  List[Any]			,				snake_case__		:  Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,				snake_case__		:  Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,				snake_case__		:  Union[bool, str, PaddingStrategy] = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  bool = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[Union[str, TensorType]] = None			,				snake_case__		:  Optional[int] = None			,				**snake_case__		:  List[Any]			,				):



   '''simple docstring'''




   if audio is None and audio_target is None:
     raise ValueError("You must provide either `audio` or `audio_target` values." )

   if sampling_rate is not None:
     if sampling_rate != self.sampling_rate:
       raise ValueError(
           F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
           F""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
           F""" {self.sampling_rate} and not {sampling_rate}.""" )
   else:
     logger.warning(
         "It is strongly recommended to pass the ``sampling_rate`` argument to this function. "
         "Failing to do so can result in silent errors that might be hard to debug." )

   if audio is not None:
     UpperCAmelCase__      :					Union[str, Any]				       =							self._process_audio(
         snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				**snake_case__			,				)
   else:
     UpperCAmelCase__      :					str				       =							None

   if audio_target is not None:
     UpperCAmelCase__      :					str				       =							self._process_audio(
         snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				snake_case__			,				**snake_case__			,				)

     if inputs is None:
       return inputs_target
     else:
       UpperCAmelCase__      :					Union[str, Any]				       =							inputs_target["input_values"]
       UpperCAmelCase__      :					Any				       =							inputs_target.get("attention_mask" )
       if decoder_attention_mask is not None:
         UpperCAmelCase__      :					Dict				       =							decoder_attention_mask

   return inputs







 def 							UpperCamelCase  (					self		:  List[Any]			,				snake_case__		:  Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]			,				snake_case__		:  bool = False			,				snake_case__		:  Union[bool, str, PaddingStrategy] = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  bool = False			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[Union[str, TensorType]] = None			,				**snake_case__		:  int			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							isinstance(snake_case__			,				np.ndarray ) and len(speech.shape ) > 1
   if is_batched_numpy and len(speech.shape ) > 2:
     raise ValueError(F"""Only mono-channel audio is supported for input to {self}""" )
   UpperCAmelCase__      :					Union[str, Any]				       =							is_batched_numpy or (
       isinstance(snake_case__			,				(list, tuple) ) and (isinstance(speech[0]			,				(np.ndarray, tuple, list) ))
   )

   if is_batched:
     UpperCAmelCase__      :					List[Any]				       =							[np.asarray(snake_case__			,				dtype=np.floataa ) for speech in speech]
   elif not is_batched and not isinstance(snake_case__			,				np.ndarray ):
     UpperCAmelCase__      :					int				       =							np.asarray(snake_case__			,				dtype=np.floataa )
   elif isinstance(snake_case__			,				np.ndarray ) and speech.dtype is np.dtype(np.floataa ):
     UpperCAmelCase__      :					List[Any]				       =							speech.astype(np.floataa )

   # always return batch
   if not is_batched:
     UpperCAmelCase__      :					str				       =							[speech]

   # needed to make pad() work on spectrogram inputs
   UpperCAmelCase__      :					str				       =							self.feature_size

   # convert into correct format for padding
   if is_target:
     UpperCAmelCase__      :					List[Any]				       =							[self._extract_mel_features(snake_case__ ) for waveform in speech]
     UpperCAmelCase__      :					List[Any]				       =							BatchFeature({"input_values": features} )
     UpperCAmelCase__      :					List[str]				       =							self.num_mel_bins
   else:
     UpperCAmelCase__      :					List[str]				       =							BatchFeature({"input_values": speech} )

   UpperCAmelCase__      :					Optional[Any]				       =							self.pad(
       snake_case__			,				padding=snake_case__			,				max_length=snake_case__			,				truncation=snake_case__			,				pad_to_multiple_of=snake_case__			,				return_attention_mask=snake_case__			,				**snake_case__			,				)

   UpperCAmelCase__      :					List[str]				       =							feature_size_hack

   # convert input values to correct format
   UpperCAmelCase__      :					Tuple				       =							padded_inputs["input_values"]
   if not isinstance(input_values[0]			,				np.ndarray ):
     UpperCAmelCase__      :					List[Any]				       =							[np.asarray(snake_case__			,				dtype=np.floataa ) for array in input_values]
   elif (
       not isinstance(snake_case__			,				np.ndarray )
       and isinstance(input_values[0]			,				np.ndarray )
       and input_values[0].dtype is np.dtype(np.floataa )
   ):
     UpperCAmelCase__      :					str				       =							[array.astype(np.floataa ) for array in input_values]
   elif isinstance(snake_case__			,				np.ndarray ) and input_values.dtype is np.dtype(np.floataa ):
     UpperCAmelCase__      :					str				       =							input_values.astype(np.floataa )

   # convert attention_mask to correct format
   UpperCAmelCase__      :					Dict				       =							padded_inputs.get("attention_mask" )
   if attention_mask is not None:
     UpperCAmelCase__      :					int				       =							[np.asarray(snake_case__			,				dtype=np.intaa ) for array in attention_mask]

   # zero-mean and unit-variance normalization
   if not is_target and self.do_normalize:
     UpperCAmelCase__      :					Any				       =							(
         attention_mask
         if self._get_padding_strategies(snake_case__			,				max_length=snake_case__ ) is not PaddingStrategy.DO_NOT_PAD
         else None
     )
     UpperCAmelCase__      :					int				       =							self.zero_mean_unit_var_norm(
         padded_inputs["input_values"]			,				attention_mask=snake_case__			,				padding_value=self.padding_value )

   if return_tensors is not None:
     UpperCAmelCase__      :					Union[str, Any]				       =							padded_inputs.convert_to_tensors(snake_case__ )

   return padded_inputs







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							super().to_dict()

   # Don't serialize these as they are derived from the other properties.
   UpperCAmelCase__      :					Dict				       =							["window", "mel_filters", "sample_size", "sample_stride", "n_fft", "n_freqs"]
   for name in names:
     if name in output:
       del output[name]

   return output



