




from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


_lowerCAmelCase						:   Optional[Any]		=   logging.get_logger(__name__)
class 			_UpperCamelCase      (				lowerCAmelCase		):
      UpperCAmelCase_        =		["""input_features""", """attention_mask"""]
      def __init__(			self   :Dict ,      lowerCamelCase   :Optional[Any]=80 ,      lowerCamelCase   :List[str]=1_6000 ,      lowerCamelCase   :int=0.0 ,      lowerCamelCase   :str=10 ,      lowerCamelCase   :str=25 ,      lowerCamelCase   :Dict="hamming_window" ,      lowerCamelCase   :int=3_27_68.0 ,      lowerCamelCase   :str=0.97 ,      lowerCamelCase   :Tuple=1.0 ,      lowerCamelCase   :Tuple=True ,      lowerCamelCase   :List[str]=True ,      lowerCamelCase   :int=False ,      **lowerCamelCase   :str ,      )     ->			Tuple:
       super().__init__(feature_size=lowerCamelCase ,      sampling_rate=lowerCamelCase ,      padding_value=lowerCamelCase ,      **lowerCamelCase      )

       UpperCAmelCase__							 =						feature_size
       UpperCAmelCase__							 =						sampling_rate
       UpperCAmelCase__							 =						padding_value
       UpperCAmelCase__							 =						hop_length
       UpperCAmelCase__							 =						win_length
       UpperCAmelCase__							 =						frame_signal_scale
       UpperCAmelCase__							 =						preemphasis_coeff
       UpperCAmelCase__							 =						mel_floor
       UpperCAmelCase__							 =						normalize_means
       UpperCAmelCase__							 =						normalize_vars
       UpperCAmelCase__							 =						win_function
       UpperCAmelCase__							 =						return_attention_mask

       UpperCAmelCase__							 =						win_length * sampling_rate // 1000
       UpperCAmelCase__							 =						hop_length * sampling_rate // 1000

       UpperCAmelCase__							 =						optimal_fft_length(self.sample_size      )
       UpperCAmelCase__							 =						(self.n_fft // 2) + 1
      def 	UpperCAmelCase_						(			self   :str ,      lowerCamelCase   :np.array      )     ->			np.ndarray:
       if self.win_function == "hamming_window":
        UpperCAmelCase__							 =						window_function(window_length=self.sample_size ,      name=self.win_function ,      periodic=lowerCamelCase      )
       else:
        UpperCAmelCase__							 =						window_function(window_length=self.sample_size ,      name=self.win_function      )

       UpperCAmelCase__							 =						mel_filter_bank(
           num_frequency_bins=self.n_freqs ,      num_mel_filters=self.feature_size ,      min_frequency=0.0 ,      max_frequency=self.sampling_rate / 2.0 ,      sampling_rate=self.sampling_rate ,      )

       UpperCAmelCase__							 =						spectrogram(
           one_waveform * self.frame_signal_scale ,      window=lowerCamelCase ,      frame_length=self.sample_size ,      hop_length=self.sample_stride ,      fft_length=self.n_fft ,      center=lowerCamelCase ,      preemphasis=self.preemphasis_coeff ,      mel_filters=lowerCamelCase ,      mel_floor=self.mel_floor ,      log_mel="log" ,      )
       return msfc_features.T
      def 	UpperCAmelCase_						(			self   :Optional[Any] ,      lowerCamelCase   :Optional[int] ,      lowerCamelCase   :Optional[int] ,      lowerCamelCase   :int      )     ->			str:
       # make sure we normalize float32 arrays
       if self.normalize_means:
        UpperCAmelCase__							 =						x[:input_length].mean(axis=0      )
        UpperCAmelCase__							 =						np.subtract(lowerCamelCase ,      lowerCamelCase      )
       if self.normalize_vars:
        UpperCAmelCase__							 =						x[:input_length].std(axis=0      )
        UpperCAmelCase__							 =						np.divide(lowerCamelCase ,      lowerCamelCase      )

       if input_length < x.shape[0]:
        UpperCAmelCase__							 =						padding_value

       # make sure array is in float32
       UpperCAmelCase__							 =						x.astype(np.floataa      )

       return x
      def 	UpperCAmelCase_						(			self   :str ,      lowerCamelCase   :List[np.ndarray] ,      lowerCamelCase   :Optional[np.ndarray] = None      )     ->			List[np.ndarray]:
       UpperCAmelCase__							 =						attention_mask.sum(-1      ) if attention_mask is not None else [x.shape[0] for x in input_features]
       return [self._normalize_one(lowerCamelCase ,      lowerCamelCase ,      self.padding_value      ) for x, n in zip(lowerCamelCase ,      lowerCamelCase      )]







      def __call__(			self   :Optional[Any] ,      lowerCamelCase   :Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]] ,      lowerCamelCase   :Union[bool, str, PaddingStrategy] = False ,      lowerCamelCase   :Optional[int] = None ,      lowerCamelCase   :bool = False ,      lowerCamelCase   :Optional[int] = None ,      lowerCamelCase   :Optional[bool] = None ,      lowerCamelCase   :Optional[Union[str, TensorType]] = None ,      lowerCamelCase   :Optional[int] = None ,      **lowerCamelCase   :Optional[int] ,      )     ->			BatchFeature:

       if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
         raise ValueError(
             f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
             f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
             f''' {self.sampling_rate} and not {sampling_rate}.'''      )
       else:
        logger.warning(
            "It is strongly recommended to pass the ``sampling_rate`` argument to this function. "
            "Failing to do so can result in silent errors that might be hard to debug."      )

       UpperCAmelCase__							 =						isinstance(lowerCamelCase ,      np.ndarray      ) and len(raw_speech.shape      ) > 1
       if is_batched_numpy and len(raw_speech.shape      ) > 2:
        raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''      )
       UpperCAmelCase__							 =						is_batched_numpy or (
           isinstance(lowerCamelCase ,      (list, tuple)      ) and (isinstance(raw_speech[0] ,      (np.ndarray, tuple, list)      ))
       )

       if is_batched:
        UpperCAmelCase__							 =						[np.asarray(lowerCamelCase ,      dtype=np.floataa      ) for speech in raw_speech]
       elif not is_batched and not isinstance(lowerCamelCase ,      np.ndarray      ):
        UpperCAmelCase__							 =						np.asarray(lowerCamelCase ,      dtype=np.floataa      )
       elif isinstance(lowerCamelCase ,      np.ndarray      ) and raw_speech.dtype is np.dtype(np.floataa      ):
        UpperCAmelCase__							 =						raw_speech.astype(np.floataa      )

       # always return batch
       if not is_batched:
        UpperCAmelCase__							 =						[raw_speech]

       # extract fbank features
       UpperCAmelCase__							 =						[self._extract_mfsc_features(lowerCamelCase      ) for one_waveform in raw_speech]

       # convert into correct format for padding
       UpperCAmelCase__							 =						BatchFeature({"input_features": features}      )

       UpperCAmelCase__							 =						self.pad(
           lowerCamelCase ,      padding=lowerCamelCase ,      max_length=lowerCamelCase ,      truncation=lowerCamelCase ,      pad_to_multiple_of=lowerCamelCase ,      return_attention_mask=lowerCamelCase ,      **lowerCamelCase ,      )
       # make sure list is in array format
       UpperCAmelCase__							 =						padded_inputs.get("input_features"      )
       if isinstance(input_features[0] ,      lowerCamelCase      ):
        UpperCAmelCase__							 =						[np.asarray(lowerCamelCase ,      dtype=np.floataa      ) for feature in input_features]

       UpperCAmelCase__							 =						padded_inputs.get("attention_mask"      )
       if attention_mask is not None:
        UpperCAmelCase__							 =						[np.asarray(lowerCamelCase ,      dtype=np.intaa      ) for array in attention_mask]

       if self.normalize_means or self.normalize_vars:
        UpperCAmelCase__							 =						(
            np.array(lowerCamelCase ,      dtype=np.intaa      )
            if self._get_padding_strategies(lowerCamelCase ,      max_length=lowerCamelCase      ) is not PaddingStrategy.DO_NOT_PAD
            and padding
            else None
        )
        UpperCAmelCase__							 =						self.normalize(
            padded_inputs["input_features"] ,      attention_mask=lowerCamelCase      )

       if return_tensors is not None:
        UpperCAmelCase__							 =						padded_inputs.convert_to_tensors(lowerCamelCase      )

       return padded_inputs











import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


_lowerCAmelCase						:   str		=   logging.get_logger(__name__)

_lowerCAmelCase						:   List[Any]		=   {"tokenizer_file": "tokenizer.json"}

_lowerCAmelCase						:   Tuple		=   {
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}
class 			_UpperCamelCase      (				lowerCAmelCase		):
      UpperCAmelCase_        =		VOCAB_FILES_NAMES
      UpperCAmelCase_        =		PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_        =		["""input_ids""", """attention_mask"""]
      UpperCAmelCase_        =		None
      def __init__(			self   :List[Any] ,      lowerCamelCase   :Union[str, Any]=None ,      lowerCamelCase   :Optional[Any]=None ,      lowerCamelCase   :List[Any]=None ,      lowerCamelCase   :Union[str, Any]="<unk>" ,      lowerCamelCase   :Union[str, Any]="<s>" ,      lowerCamelCase   :Union[str, Any]="</s>" ,      lowerCamelCase   :str="<pad>" ,      lowerCamelCase   :Any=False ,      lowerCamelCase   :Optional[Any]=False ,      **lowerCamelCase   :Optional[Any] ,      )     ->			Any:
       super().__init__(
           lowerCamelCase ,      lowerCamelCase ,      tokenizer_file=lowerCamelCase ,      unk_token=lowerCamelCase ,      bos_token=lowerCamelCase ,      eos_token=lowerCamelCase ,      pad_token=lowerCamelCase ,      add_prefix_space=lowerCamelCase ,      clean_up_tokenization_spaces=lowerCamelCase ,      **lowerCamelCase ,      )
       UpperCAmelCase__							 =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
       if pre_tok_state.get("add_prefix_space" ,      lowerCamelCase      ) != add_prefix_space:
        UpperCAmelCase__							 =						getattr(lowerCamelCase ,      pre_tok_state.pop("type"      )      )
        UpperCAmelCase__							 =						add_prefix_space
        UpperCAmelCase__							 =						pre_tok_class(**lowerCamelCase      )

       UpperCAmelCase__							 =						add_prefix_space
      def 	UpperCAmelCase_						(			self   :int ,      *lowerCamelCase   :Any ,      **lowerCamelCase   :int      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )
       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._batch_encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Any ,      *lowerCamelCase   :List[Any] ,      **lowerCamelCase   :Union[str, Any]      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )

       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Tuple ,      lowerCamelCase   :str ,      lowerCamelCase   :Optional[str] = None      )     ->			Tuple[str]:
       UpperCAmelCase__							 =						self._tokenizer.model.save(lowerCamelCase ,      name=lowerCamelCase      )
       return tuple(lowerCamelCase      )







      def 	UpperCAmelCase_						(			self   :List[str] ,      lowerCamelCase   :"Conversation"      )     ->			List[int]:
       UpperCAmelCase__							 =						[]
       for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      ) + [self.eos_token_id]      )

       if len(lowerCamelCase      ) > self.model_max_length:
        UpperCAmelCase__							 =						input_ids[-self.model_max_length :]
       return input_ids





