



'''simple docstring'''





from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


SCREAMING_SNAKE_CASE__          =  logging.get_logger(__name__)


class 					snake_case			(UpperCamelCase ):
       lowerCAmelCase__	:str     =       ["input_features", "attention_mask"]



       def __init__(					self  ,UpperCAmelCase_=80  ,UpperCAmelCase_=16_000  ,UpperCAmelCase_=80  ,UpperCAmelCase_=0.0  ,UpperCAmelCase_=True  ,UpperCAmelCase_=True  ,UpperCAmelCase_=True  ,**UpperCAmelCase_  ,)	->				Optional[Any]:
           super().__init__(feature_size=UpperCAmelCase_  ,sampling_rate=UpperCAmelCase_  ,padding_value=UpperCAmelCase_  ,**UpperCAmelCase_   )
           lowercase__									=    num_mel_bins
           lowercase__									=    do_ceptral_normalize
           lowercase__									=    normalize_means
           lowercase__									=    normalize_vars
           lowercase__									=    True



       def 				_a   (					self  ,UpperCAmelCase_  ,)	->				np.ndarray:
           lowercase__									=    waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
           lowercase__									=    torch.from_numpy(UpperCAmelCase_   ).unsqueeze(0   )
           lowercase__									=    ta_kaldi.fbank(UpperCAmelCase_  ,num_mel_bins=self.num_mel_bins  ,sample_frequency=self.sampling_rate   )
           return features.numpy()



       @staticmethod
       def 				_a   (					UpperCAmelCase_  ,UpperCAmelCase_  ,UpperCAmelCase_ = True  ,UpperCAmelCase_ = True  ,UpperCAmelCase_ = 0.0  ,)	->				np.ndarray:
           # make sure we normalize float32 arrays
           if normalize_means:
               lowercase__									=    x[:input_length].mean(axis=0   )
               lowercase__									=    np.subtract(UpperCAmelCase_  ,UpperCAmelCase_   )
           if normalize_vars:
               lowercase__									=    x[:input_length].std(axis=0   )
               lowercase__									=    np.divide(UpperCAmelCase_  ,UpperCAmelCase_   )

           if input_length < x.shape[0]:
               lowercase__									=    padding_value

           # make sure array is in float32
           lowercase__									=    x.astype(np.floataa   )

           return x



       def 				_a   (					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = None   )	->				List[np.ndarray]:
           lowercase__									=    attention_mask.sum(-1   ) if attention_mask is not None else [x.shape[0] for x in input_features]
           return [
               self.utterance_cmvn(UpperCAmelCase_  ,UpperCAmelCase_  ,self.normalize_means  ,self.normalize_vars  ,self.padding_value   )
               for x, n in zip(UpperCAmelCase_  ,UpperCAmelCase_   )
           ]







       def __call__(					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = False  ,UpperCAmelCase_ = None  ,UpperCAmelCase_ = False  ,UpperCAmelCase_ = None  ,UpperCAmelCase_ = None  ,UpperCAmelCase_ = None  ,UpperCAmelCase_ = None  ,**UpperCAmelCase_  ,)	->				BatchFeature:

           if sampling_rate is not None:
               if sampling_rate != self.sampling_rate:
                   raise ValueError(
                       F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
                       F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
                       F''' {self.sampling_rate} and not {sampling_rate}.'''   )
           else:
               logger.warning(
                   "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                   "Failing to do so can result in silent errors that might be hard to debug."   )

           lowercase__									=    isinstance(UpperCAmelCase_  ,np.ndarray   ) and len(raw_speech.shape   ) > 1
           if is_batched_numpy and len(raw_speech.shape   ) > 2:
               raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''   )
           lowercase__									=    is_batched_numpy or (
               isinstance(UpperCAmelCase_  ,(list, tuple)   ) and (isinstance(raw_speech[0]  ,(np.ndarray, tuple, list)   ))
           )

           if is_batched:
               lowercase__									=    [np.asarray(UpperCAmelCase_  ,dtype=np.floataa   ) for speech in raw_speech]
           elif not is_batched and not isinstance(UpperCAmelCase_  ,np.ndarray   ):
               lowercase__									=    np.asarray(UpperCAmelCase_  ,dtype=np.floataa   )
           elif isinstance(UpperCAmelCase_  ,np.ndarray   ) and raw_speech.dtype is np.dtype(np.floataa   ):
               lowercase__									=    raw_speech.astype(np.floataa   )

           # always return batch
           if not is_batched:
               lowercase__									=    [raw_speech]

           # extract fbank features
           lowercase__									=    [self._extract_fbank_features(UpperCAmelCase_   ) for waveform in raw_speech]

           # convert into correct format for padding
           lowercase__									=    BatchFeature({"input_features": features}   )

           lowercase__									=    self.pad(
               UpperCAmelCase_  ,padding=UpperCAmelCase_  ,max_length=UpperCAmelCase_  ,truncation=UpperCAmelCase_  ,pad_to_multiple_of=UpperCAmelCase_  ,return_attention_mask=UpperCAmelCase_  ,**UpperCAmelCase_  ,)

           # make sure list is in array format
           lowercase__									=    padded_inputs.get("input_features"   )
           if isinstance(input_features[0]  ,UpperCAmelCase_   ):
               lowercase__									=    [np.asarray(UpperCAmelCase_  ,dtype=np.floataa   ) for feature in input_features]

           lowercase__									=    padded_inputs.get("attention_mask"   )
           if attention_mask is not None:
               lowercase__									=    [np.asarray(UpperCAmelCase_  ,dtype=np.intaa   ) for array in attention_mask]

           # Utterance-level cepstral mean and variance normalization
           if self.do_ceptral_normalize:
               lowercase__									=    (
                   np.array(UpperCAmelCase_  ,dtype=np.intaa   )
                   if self._get_padding_strategies(UpperCAmelCase_  ,max_length=UpperCAmelCase_   ) is not PaddingStrategy.DO_NOT_PAD
                   else None
               )
               lowercase__									=    self.normalize(
                   padded_inputs["input_features"]  ,attention_mask=UpperCAmelCase_   )

           if return_tensors is not None:
               lowercase__									=    padded_inputs.convert_to_tensors(UpperCAmelCase_   )

           return padded_inputs









'''simple docstring'''





import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 					snake_case			(UpperCamelCase ):



       def __init__(					self  ,UpperCAmelCase_ = "▁"  ,UpperCAmelCase_ = True  ,UpperCAmelCase_ = "<unk>"  ,UpperCAmelCase_ = "</s>"  ,UpperCAmelCase_ = "<pad>"  ,)	->				Optional[int]:
           lowercase__									=    {
               "pad": {"id": 0, "token": pad_token},
               "eos": {"id": 1, "token": eos_token},
               "unk": {"id": 2, "token": unk_token},
           }

           lowercase__									=    [None] * len(self.special_tokens   )
           for token_dict in self.special_tokens.values():
               lowercase__									=    token_dict["token"]

           lowercase__									=    Tokenizer(Unigram()   )

           lowercase__									=    normalizers.Sequence(
               [
                   normalizers.Nmt(),
                   normalizers.NFKC(),
                   normalizers.Replace(Regex(" {2,}"   )  ," "   ),
                   normalizers.Lowercase(),
               ]   )
           lowercase__									=    pre_tokenizers.Sequence(
               [
                   pre_tokenizers.Metaspace(replacement=UpperCAmelCase_  ,add_prefix_space=UpperCAmelCase_   ),
                   pre_tokenizers.Digits(individual_digits=UpperCAmelCase_   ),
                   pre_tokenizers.Punctuation(),
               ]   )
           lowercase__									=    decoders.Metaspace(replacement=UpperCAmelCase_  ,add_prefix_space=UpperCAmelCase_   )

           lowercase__									=    TemplateProcessing(
               single=F'''$A {self.special_tokens['eos']['token']}'''  ,special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])]  ,)

           lowercase__									=    {
               "model": "SentencePieceUnigram",
               "replacement": replacement,
               "add_prefix_space": add_prefix_space,
           }

           super().__init__(UpperCAmelCase_  ,UpperCAmelCase_   )



       def 				_a   (					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = 8_000  ,UpperCAmelCase_ = True  ,)	->				List[str]:
           lowercase__									=    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase_  ,special_tokens=self.special_tokens_list  ,show_progress=UpperCAmelCase_  ,)

           if isinstance(UpperCAmelCase_  ,UpperCAmelCase_   ):
               lowercase__									=    [files]
           self._tokenizer.train(UpperCAmelCase_  ,trainer=UpperCAmelCase_   )

           self.add_unk_id()



       def 				_a   (					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = 8_000  ,UpperCAmelCase_ = True  ,)	->				Union[str, Any]:
           lowercase__									=    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase_  ,special_tokens=self.special_tokens_list  ,show_progress=UpperCAmelCase_  ,)

           self._tokenizer.train_from_iterator(UpperCAmelCase_  ,trainer=UpperCAmelCase_   )

           self.add_unk_id()







       def 				_a   (					self   )	->				str:
           lowercase__									=    json.loads(self._tokenizer.to_str()   )

           lowercase__									=    self.special_tokens["unk"]["id"]

           lowercase__									=    Tokenizer.from_str(json.dumps(UpperCAmelCase_   )   )




