"""simple docstring"""



from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


_UpperCamelCase								=		logging.get_logger(__name__)
class 						__UpperCAmelCase    (a_     ):







			'''simple docstring'''
			_UpperCamelCase  :						int					     =				['input_features', 'attention_mask']





			def __init__(			self   ,      snake_case_=80   ,      snake_case_=16_000   ,      snake_case_=80   ,      snake_case_=0.0   ,      snake_case_=True   ,      snake_case_=True   ,      snake_case_=True   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(feature_size=lowercase_   ,      sampling_rate=lowercase_   ,      padding_value=lowercase_   ,      **lowercase_							)
										A__       :     Optional[Any]								=			num_mel_bins
										A__       :     str								=			do_ceptral_normalize
										A__       :     List[Any]								=			normalize_means
										A__       :     Optional[Any]								=			normalize_vars
										A__       :     int								=			True





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      ):







										'''simple docstring'''


										A__       :     Union[str, Any]								=			waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
										A__       :     Union[str, Any]								=			torch.from_numpy(lowercase_							).unsqueeze(0							)
										A__       :     List[str]								=			ta_kaldi.fbank(lowercase_   ,      num_mel_bins=self.num_mel_bins   ,      sample_frequency=self.sampling_rate							)
										return features.numpy()





			@staticmethod
			def 		lowerCamelCase					(			snake_case_   ,      snake_case_   ,      snake_case_ = True   ,      snake_case_ = True   ,      snake_case_ = 0.0   ,      ):







										'''simple docstring'''


										if normalize_means:
																	A__       :     Union[str, Any]								=			x[:input_length].mean(axis=0							)
																	A__       :     int								=			np.subtract(lowercase_   ,      lowercase_							)
										if normalize_vars:
																	A__       :     str								=			x[:input_length].std(axis=0							)
																	A__       :     Optional[int]								=			np.divide(lowercase_   ,      lowercase_							)

										if input_length < x.shape[0]:
																	A__       :     Optional[int]								=			padding_value

										# make sure array is in float32
										A__       :     int								=			x.astype(np.floataa							)

										return x





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     List[Any]								=			attention_mask.sum(-1							) if attention_mask is not None else [x.shape[0] for x in input_features]
										return [
										    self.utterance_cmvn(lowercase_   ,      lowercase_   ,      self.normalize_means   ,      self.normalize_vars   ,      self.padding_value							)
										    for x, n in zip(lowercase_   ,      lowercase_							)
										]





			def __call__(			self   ,      snake_case_   ,      snake_case_ = False   ,      snake_case_ = None   ,      snake_case_ = False   ,      snake_case_ = None   ,      snake_case_ = None   ,      snake_case_ = None   ,      snake_case_ = None   ,      **snake_case_   ,      ):







										'''simple docstring'''


										if sampling_rate is not None:
																	if sampling_rate != self.sampling_rate:
																								raise ValueError(
																								    F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
																								    F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
																								    F''' {self.sampling_rate} and not {sampling_rate}.'''							)
										else:
																	logger.warning(
																	    """It is strongly recommended to pass the `sampling_rate` argument to this function. """
																	    """Failing to do so can result in silent errors that might be hard to debug."""							)

										A__       :     Union[str, Any]								=			isinstance(lowercase_   ,      np.ndarray							) and len(raw_speech.shape							) > 1
										if is_batched_numpy and len(raw_speech.shape							) > 2:
																	raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''							)
										A__       :     Optional[int]								=			is_batched_numpy or (
										    isinstance(lowercase_   ,      (list, tuple)							) and (isinstance(raw_speech[0]   ,      (np.ndarray, tuple, list)							))
										)

										if is_batched:
																	A__       :     Dict								=			[np.asarray(lowercase_   ,      dtype=np.floataa							) for speech in raw_speech]
										elif not is_batched and not isinstance(lowercase_   ,      np.ndarray							):
																	A__       :     Optional[int]								=			np.asarray(lowercase_   ,      dtype=np.floataa							)
										elif isinstance(lowercase_   ,      np.ndarray							) and raw_speech.dtype is np.dtype(np.floataa							):
																	A__       :     Any								=			raw_speech.astype(np.floataa							)

										# always return batch
										if not is_batched:
																	A__       :     str								=			[raw_speech]

										# extract fbank features
										A__       :     List[str]								=			[self._extract_fbank_features(lowercase_							) for waveform in raw_speech]

										# convert into correct format for padding
										A__       :     List[str]								=			BatchFeature({"""input_features""": features}							)

										A__       :     List[Any]								=			self.pad(
										    lowercase_   ,      padding=lowercase_   ,      max_length=lowercase_   ,      truncation=lowercase_   ,      pad_to_multiple_of=lowercase_   ,      return_attention_mask=lowercase_   ,      **lowercase_   ,      )

										# make sure list is in array format
										A__       :     Optional[int]								=			padded_inputs.get("""input_features"""							)
										if isinstance(input_features[0]   ,      lowercase_							):
																	A__       :     List[str]								=			[np.asarray(lowercase_   ,      dtype=np.floataa							) for feature in input_features]

										A__       :     Dict								=			padded_inputs.get("""attention_mask"""							)
										if attention_mask is not None:
																	A__       :     Optional[int]								=			[np.asarray(lowercase_   ,      dtype=np.intaa							) for array in attention_mask]

										# Utterance-level cepstral mean and variance normalization
										if self.do_ceptral_normalize:
																	A__       :     Optional[Any]								=			(
																	    np.array(lowercase_   ,      dtype=np.intaa							)
																	    if self._get_padding_strategies(lowercase_   ,      max_length=lowercase_							) is not PaddingStrategy.DO_NOT_PAD
																	    else None
																	)
																	A__       :     List[str]								=			self.normalize(
																	    padded_inputs["""input_features"""]   ,      attention_mask=lowercase_							)

										if return_tensors is not None:
																	A__       :     Dict								=			padded_inputs.convert_to_tensors(lowercase_							)

										return padded_inputs








"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_albert import AlbertTokenizer
else:
							__UpperCamelCase    : Any				 =						None

__UpperCamelCase    : Optional[int]				 =						logging.get_logger(__name__)
__UpperCamelCase    : List[Any]				 =						{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase    : str				 =						{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

__UpperCamelCase    : Any				 =						{
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

__UpperCamelCase    : Union[str, Any]				 =						'''▁'''







class 							SCREAMING_SNAKE_CASE   (						a_			):






					"""simple docstring"""



					lowercase__												=			VOCAB_FILES_NAMES
					lowercase__												=			PRETRAINED_VOCAB_FILES_MAP
					lowercase__												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__												=			AlbertTokenizer







					def __init__(			self     :							Union[str, Any]  ,lowercase_     :							List[str]=None  ,lowercase_     :							Union[str, Any]=None  ,lowercase_     :							str=True  ,lowercase_     :							Optional[Any]=True  ,lowercase_     :							str=False  ,lowercase_     :							Tuple="[CLS]"  ,lowercase_     :							Optional[int]="[SEP]"  ,lowercase_     :							Optional[Any]="<unk>"  ,lowercase_     :							List[Any]="[SEP]"  ,lowercase_     :							Optional[int]="<pad>"  ,lowercase_     :							List[Any]="[CLS]"  ,lowercase_     :							Optional[int]="[MASK]"  ,**lowercase_     :							Optional[int]  ,):
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						lowerCAmelCase__      :		Optional[Any]    							=      (
						    AddedToken(lowercase_  ,lstrip=lowercase_  ,rstrip=lowercase_  ,normalized=lowercase_						)
						    if isinstance(lowercase_  ,lowercase_						)
						    else mask_token
						)

						super().__init__(
						    lowercase_  ,tokenizer_file=lowercase_  ,do_lower_case=lowercase_  ,remove_space=lowercase_  ,keep_accents=lowercase_  ,bos_token=lowercase_  ,eos_token=lowercase_  ,unk_token=lowercase_  ,sep_token=lowercase_  ,pad_token=lowercase_  ,cls_token=lowercase_  ,mask_token=lowercase_  ,**lowercase_  ,)

						lowerCAmelCase__      :		List[Any]    							=      do_lower_case
						lowerCAmelCase__      :		str    							=      remove_space
						lowerCAmelCase__      :		Optional[int]    							=      keep_accents
						lowerCAmelCase__      :		Dict    							=      vocab_file
						lowerCAmelCase__      :		Optional[int]    							=      False if not self.vocab_file else True







					def 			__lowerCAmelCase					(			self     :							Dict  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		List[Any]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Dict    							=      [self.cls_token_id]
						if token_ids_a is None:
							return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		List[str]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		List[Any]    							=      [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
						return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







					def 			__lowerCAmelCase					(			self     :							Optional[int]  ,lowercase_     :							str  ,lowercase_     :							Optional[str] = None						):
						if not self.can_save_slow_tokenizer:
							raise ValueError(
							    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
							    '''tokenizer.'''						)

						if not os.path.isdir(lowercase_						):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
							return
						lowerCAmelCase__      :		Union[str, Any]    							=      os.path.join(
						    lowercase_  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

						if os.path.abspath(self.vocab_file						) != os.path.abspath(lowercase_						):
							copyfile(self.vocab_file  ,lowercase_						)

						return (out_vocab_file,)




