



'''simple docstring'''





import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


SCREAMING_SNAKE_CASE__      					=     logging.get_logger(__name__)


class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Optional[int]						   =					["input_features"]



       def __init__(				self     ,UpperCAmelCase_=80     ,UpperCAmelCase_=16_000     ,UpperCAmelCase_=160     ,UpperCAmelCase_=30     ,UpperCAmelCase_=400     ,UpperCAmelCase_=0.0     ,UpperCAmelCase_=False     ,**UpperCAmelCase_     ,)  ->							List[str]:
           super().__init__(
               feature_size=UpperCAmelCase_     ,sampling_rate=UpperCAmelCase_     ,padding_value=UpperCAmelCase_     ,return_attention_mask=UpperCAmelCase_     ,**UpperCAmelCase_     ,)
           lowercase__		       =						n_fft
           lowercase__		       =						hop_length
           lowercase__		       =						chunk_length
           lowercase__		       =						chunk_length * sampling_rate
           lowercase__		       =						self.n_samples // hop_length
           lowercase__		       =						sampling_rate
           lowercase__		       =						mel_filter_bank(
               num_frequency_bins=1 + n_fft // 2     ,num_mel_filters=UpperCAmelCase_     ,min_frequency=0.0     ,max_frequency=8_000.0     ,sampling_rate=UpperCAmelCase_     ,norm="slaney"     ,mel_scale="slaney"     ,)



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							np.ndarray:
           lowercase__		       =						spectrogram(
               UpperCAmelCase_     ,window_function(self.n_fft     ,"hann"      )     ,frame_length=self.n_fft     ,hop_length=self.hop_length     ,power=2.0     ,mel_filters=self.mel_filters     ,log_mel="log10"     ,)
           lowercase__		       =						log_spec[:, :-1]
           lowercase__		       =						np.maximum(UpperCAmelCase_     ,log_spec.max() - 8.0      )
           lowercase__		       =						(log_spec + 4.0) / 4.0
           return log_spec



       @staticmethod
       # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
       def 							_a      (				UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_ = 0.0      )  ->							List[np.ndarray]:
           if attention_mask is not None:
               lowercase__		       =						np.array(UpperCAmelCase_     ,np.intaa      )
               lowercase__		       =						[]

               for vector, length in zip(UpperCAmelCase_     ,attention_mask.sum(-1      )      ):
                   lowercase__		       =						(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7      )
                   if length < normed_slice.shape[0]:
                       lowercase__		       =						padding_value

                   normed_input_values.append(UpperCAmelCase_      )
           else:
               lowercase__		       =						[(x - x.mean()) / np.sqrt(x.var() + 1E-7      ) for x in input_values]

           return normed_input_values



       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = True     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = "max_length"     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = None     ,**UpperCAmelCase_     ,)  ->							BatchFeature:

           if sampling_rate is not None:
               if sampling_rate != self.sampling_rate:
                   raise ValueError(
                       F'''The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a'''
                       F''' sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input'''
                       F''' was sampled with {self.sampling_rate} and not {sampling_rate}.'''      )
           else:
               logger.warning(
                   "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                   "Failing to do so can result in silent errors that might be hard to debug."      )

           lowercase__		       =						isinstance(UpperCAmelCase_     ,np.ndarray      ) and len(raw_speech.shape      ) > 1
           if is_batched_numpy and len(raw_speech.shape      ) > 2:
               raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''      )
           lowercase__		       =						is_batched_numpy or (
               isinstance(UpperCAmelCase_     ,(list, tuple)      ) and (isinstance(raw_speech[0]     ,(np.ndarray, tuple, list)      ))
           )

           if is_batched:
               lowercase__		       =						[np.asarray([speech]     ,dtype=np.floataa      ).T for speech in raw_speech]
           elif not is_batched and not isinstance(UpperCAmelCase_     ,np.ndarray      ):
               lowercase__		       =						np.asarray(UpperCAmelCase_     ,dtype=np.floataa      )
           elif isinstance(UpperCAmelCase_     ,np.ndarray      ) and raw_speech.dtype is np.dtype(np.floataa      ):
               lowercase__		       =						raw_speech.astype(np.floataa      )

           # always return batch
           if not is_batched:
               lowercase__		       =						[np.asarray([raw_speech]      ).T]

           lowercase__		       =						BatchFeature({"input_features": raw_speech}      )

           # convert into correct format for padding

           lowercase__		       =						self.pad(
               UpperCAmelCase_     ,padding=UpperCAmelCase_     ,max_length=max_length if max_length else self.n_samples     ,truncation=UpperCAmelCase_     ,pad_to_multiple_of=UpperCAmelCase_     ,return_attention_mask=return_attention_mask or do_normalize     ,)

           # zero-mean and unit-variance normalization
           if do_normalize:
               lowercase__		       =						self.zero_mean_unit_var_norm(
                   padded_inputs["input_features"]     ,attention_mask=padded_inputs["attention_mask"]     ,padding_value=self.padding_value     ,)
               lowercase__		       =						np.stack(padded_inputs["input_features"]     ,axis=0      )

           # make sure list is in array format
           lowercase__		       =						padded_inputs.get("input_features"      ).transpose(2     ,0     ,1      )

           lowercase__		       =						[self._np_extract_fbank_features(UpperCAmelCase_      ) for waveform in input_features[0]]

           if isinstance(input_features[0]     ,UpperCAmelCase_      ):
               lowercase__		       =						[np.asarray(UpperCAmelCase_     ,dtype=np.floataa      ) for feature in input_features]
           else:
               lowercase__		       =						input_features

           if return_attention_mask:
               # rescale from sample (48000) to feature (3000)
               lowercase__		       =						padded_inputs["attention_mask"][:, :: self.hop_length]

           if return_tensors is not None:
               lowercase__		       =						padded_inputs.convert_to_tensors(UpperCAmelCase_      )

           return padded_inputs







       def 							_a      (				self      )  ->							Dict[str, Any]:
           lowercase__		       =						copy.deepcopy(self.__dict__      )
           lowercase__		       =						self.__class__.__name__
           if "mel_filters" in output:
               del output["mel_filters"]
           return output









'''simple docstring'''





import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer


class 				snake_case (tf.keras.layers.Layer   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = None      )  ->							List[Any]:
           super().__init__()
           lowercase__		       =						pad_token_id
           lowercase__		       =						max_length
           lowercase__		       =						vocab
           lowercase__		       =						merges
           lowercase__		       =						BytePairTokenizer(UpperCAmelCase_     ,UpperCAmelCase_     ,sequence_length=UpperCAmelCase_      )



       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							List[str]:
           lowercase__		       =						[" ".join(UpperCAmelCase_      ) for m in tokenizer.bpe_ranks.keys()]
           lowercase__		       =						tokenizer.get_vocab()
           return cls(UpperCAmelCase_     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )



       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							Optional[Any]:
           lowercase__		       =						GPTaTokenizer.from_pretrained(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
           return cls.from_tokenizer(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )



       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_      )  ->							Optional[Any]:
           return cls(**UpperCAmelCase_      )



       def 							_a      (				self      )  ->							Tuple:
           return {
               "vocab": self.vocab,
               "merges": self.merges,
               "max_length": self.max_length,
               "pad_token_id": self.pad_token_id,
           }







       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None      )  ->							Tuple:
           lowercase__		       =						self.tf_tokenizer(UpperCAmelCase_      )
           lowercase__		       =						tf.ones_like(UpperCAmelCase_      )

           if self.pad_token_id is not None:
               # pad the tokens up to max length
               lowercase__		       =						max_length if max_length is not None else self.max_length

               if max_length is not None:
                   lowercase__       ,       lowercase__		       =						pad_model_inputs(
                       UpperCAmelCase_     ,max_seq_length=UpperCAmelCase_     ,pad_value=self.pad_token_id      )

           return {"attention_mask": attention_mask, "input_ids": input_ids}




