


"""simple docstring"""


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__snake_case    					=					logging.get_logger(__name__)

__snake_case    					=					{'vocab_file': 'spiece.model'}

__snake_case    					=					{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

__snake_case    					=					{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
__snake_case    					=					0
__snake_case    					=					1
__snake_case    					=					2
__snake_case    					=					3
__snake_case    					=					4

class 				_SCREAMING_SNAKE_CASE				(							__UpperCAmelCase  ):
   """simple docstring"""



   _a   :       List[Any]									=    VOCAB_FILES_NAMES
   _a   :       List[str]									=    PRETRAINED_VOCAB_FILES_MAP
   _a   :       Dict									=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a   :       Any									=    '''left'''


   def __init__(						self ,     lowerCamelCase__ ,     lowerCamelCase__=False ,     lowerCamelCase__=True ,     lowerCamelCase__=False ,     lowerCamelCase__="<s>" ,     lowerCamelCase__="</s>" ,     lowerCamelCase__="<unk>" ,     lowerCamelCase__="<sep>" ,     lowerCamelCase__="<pad>" ,     lowerCamelCase__="<cls>" ,     lowerCamelCase__="<mask>" ,     lowerCamelCase__=["<eop>", "<eod>"] ,     lowerCamelCase__ = None ,     **lowerCamelCase__ ,     )      ->							None:
    # Mask token behave like a normal word, i.e. include the space before it
    lowercase__       :       Optional[int]				=						AddedToken(lowerCamelCase__ ,     lstrip=lowerCamelCase__ ,     rstrip=lowerCamelCase__						) if isinstance(lowerCamelCase__ ,     lowerCamelCase__						) else mask_token

    lowercase__       :       Dict				=						{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=lowerCamelCase__ ,     remove_space=lowerCamelCase__ ,     keep_accents=lowerCamelCase__ ,     bos_token=lowerCamelCase__ ,     eos_token=lowerCamelCase__ ,     unk_token=lowerCamelCase__ ,     sep_token=lowerCamelCase__ ,     pad_token=lowerCamelCase__ ,     cls_token=lowerCamelCase__ ,     mask_token=lowerCamelCase__ ,     additional_special_tokens=lowerCamelCase__ ,     sp_model_kwargs=self.sp_model_kwargs ,     **lowerCamelCase__ ,     )

    lowercase__       :       Optional[int]				=						3

    lowercase__       :       List[Any]				=						do_lower_case
    lowercase__       :       Dict				=						remove_space
    lowercase__       :       Optional[int]				=						keep_accents
    lowercase__       :       List[Any]				=						vocab_file

    lowercase__       :       Tuple				=						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
    self.sp_model.Load(lowerCamelCase__						)


   @property
   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    return len(self.sp_model						)


   def 					UpperCAmelCase__(						self						)      ->							int:
    lowercase__       :       Optional[Any]				=						{self.convert_ids_to_tokens(lowerCamelCase__						): i for i in range(self.vocab_size						)}
    vocab.update(self.added_tokens_encoder						)
    return vocab


   def __getstate__(						self						)      ->							Tuple:
    lowercase__       :       Optional[Any]				=						self.__dict__.copy()
    lowercase__       :       int				=						None
    return state


   def __setstate__(						self ,     lowerCamelCase__						)      ->							Optional[Any]:
    lowercase__       :       str				=						d

    # for backward compatibility
    if not hasattr(self ,     """sp_model_kwargs"""						):
     lowercase__       :       Optional[Any]				=						{}

    lowercase__       :       List[str]				=						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
    self.sp_model.Load(self.vocab_file						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							str:
    if self.remove_space:
     lowercase__       :       Any				=						""" """.join(inputs.strip().split()						)
    else:
     lowercase__       :       str				=						inputs
    lowercase__       :       List[Any]				=						outputs.replace("""``""" ,     """\""""						).replace("""''""" ,     """\""""						)

    if not self.keep_accents:
     lowercase__       :       Optional[Any]				=						unicodedata.normalize("""NFKD""" ,     lowerCamelCase__						)
     lowercase__       :       Any				=						"""""".join([c for c in outputs if not unicodedata.combining(lowerCamelCase__						)]						)
    if self.do_lower_case:
     lowercase__       :       Tuple				=						outputs.lower()

    return outputs


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							List[str]:
    lowercase__       :       str				=						self.preprocess_text(lowerCamelCase__						)
    lowercase__       :       str				=						self.sp_model.encode(lowerCamelCase__ ,     out_type=lowerCamelCase__						)
    lowercase__       :       Optional[int]				=						[]
    for piece in pieces:
     if len(lowerCamelCase__						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
      lowercase__       :       str				=						self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__ ,     """"""						)						)
      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
       if len(cur_pieces[0]						) == 1:
        lowercase__       :       Tuple				=						cur_pieces[1:]
       else:
        lowercase__       :       Optional[int]				=						cur_pieces[0][1:]
      cur_pieces.append(piece[-1]						)
      new_pieces.extend(lowerCamelCase__						)
     else:
      new_pieces.append(lowerCamelCase__						)

    return new_pieces


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							Union[str, Any]:
    return self.sp_model.PieceToId(lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							Union[str, Any]:
    return self.sp_model.IdToPiece(lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							List[Any]:
    lowercase__       :       Tuple				=						"""""".join(lowerCamelCase__						).replace(lowerCamelCase__ ,     """ """						).strip()
    return out_string


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = False ,     lowerCamelCase__ = None ,     lowerCamelCase__ = True ,     **lowerCamelCase__ ,     )      ->							str:
    lowercase__       :       Dict				=						kwargs.pop("""use_source_tokenizer""" ,     lowerCamelCase__						)

    lowercase__       :       Dict				=						self.convert_ids_to_tokens(lowerCamelCase__ ,     skip_special_tokens=lowerCamelCase__						)

    # To avoid mixing byte-level and unicode for byte-level BPT
    # we need to build string separately for added tokens and byte-level tokens
    # cf. https://github.com/huggingface/transformers/issues/1133
    lowercase__       :       List[Any]				=						[]
    lowercase__       :       str				=						[]
    for token in filtered_tokens:
     if skip_special_tokens and token in self.all_special_ids:
      continue
     if token in self.added_tokens_encoder:
      if current_sub_text:
       sub_texts.append(self.convert_tokens_to_string(lowerCamelCase__						)						)
       lowercase__       :       Optional[Any]				=						[]
      sub_texts.append(lowerCamelCase__						)
     else:
      current_sub_text.append(lowerCamelCase__						)
    if current_sub_text:
     sub_texts.append(self.convert_tokens_to_string(lowerCamelCase__						)						)

    # Mimic the behavior of the Rust tokenizer:
    # By default, there are no spaces between special tokens
    lowercase__       :       Optional[int]				=						"""""".join(lowerCamelCase__						)

    lowercase__       :       List[str]				=						(
        clean_up_tokenization_spaces
        if clean_up_tokenization_spaces is not None
        else self.clean_up_tokenization_spaces
    )
    if clean_up_tokenization_spaces:
     lowercase__       :       Any				=						self.clean_up_tokenization(lowerCamelCase__						)
     return clean_text
    else:
     return text


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = None						)      ->							List[int]:
    lowercase__       :       Optional[Any]				=						[self.sep_token_id]
    lowercase__       :       Dict				=						[self.cls_token_id]
    if token_ids_a is None:
     return token_ids_a + sep + cls
    return token_ids_a + sep + token_ids_a + sep + cls


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = None ,     lowerCamelCase__ = False						)      ->							List[int]:

    if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowerCamelCase__ ,     token_ids_a=lowerCamelCase__ ,     already_has_special_tokens=lowerCamelCase__						)

    if token_ids_a is not None:
     return ([0] * len(lowerCamelCase__						)) + [1] + ([0] * len(lowerCamelCase__						)) + [1, 1]
    return ([0] * len(lowerCamelCase__						)) + [1, 1]


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = None						)      ->							List[int]:
    lowercase__       :       List[str]				=						[self.sep_token_id]
    lowercase__       :       List[Any]				=						[2]

    if token_ids_a is None:
     return len(token_ids_a + sep						) * [0] + cls_segment_id
    return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = None						)      ->							Tuple[str]:
    if not os.path.isdir(lowerCamelCase__						):
     logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
     return
    lowercase__       :       Union[str, Any]				=						os.path.join(
        lowerCamelCase__ ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

    if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCamelCase__						) and os.path.isfile(self.vocab_file						):
     copyfile(self.vocab_file ,     lowerCamelCase__						)
    elif not os.path.isfile(self.vocab_file						):
     with open(lowerCamelCase__ ,     """wb"""						) as fi:
      lowercase__       :       Any				=						self.sp_model.serialized_model_proto()
      fi.write(lowerCamelCase__						)

    return (out_vocab_file,)



"""simple docstring"""


from math import cos, sin, sqrt, tau

from audio_filters.iir_filter import IIRFilter


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )  ):
 lowercase__       :       str				=						tau * frequency / samplerate
 lowercase__       :       Union[str, Any]				=						sin(lowerCamelCase__  )
 lowercase__       :       int				=						cos(lowerCamelCase__  )
 lowercase__       :       Union[str, Any]				=						_sin / (2 * q_factor)

 lowercase__       :       str				=						(1 - _cos) / 2
 lowercase__       :       Optional[Any]				=						1 - _cos

 lowercase__       :       List[Any]				=						1 + alpha
 lowercase__       :       Dict				=						-2 * _cos
 lowercase__       :       List[str]				=						1 - alpha

 lowercase__       :       int				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )  ):
 lowercase__       :       Optional[int]				=						tau * frequency / samplerate
 lowercase__       :       Any				=						sin(lowerCamelCase__  )
 lowercase__       :       Any				=						cos(lowerCamelCase__  )
 lowercase__       :       Tuple				=						_sin / (2 * q_factor)

 lowercase__       :       Tuple				=						(1 + _cos) / 2
 lowercase__       :       str				=						-1 - _cos

 lowercase__       :       Dict				=						1 + alpha
 lowercase__       :       Dict				=						-2 * _cos
 lowercase__       :       int				=						1 - alpha

 lowercase__       :       str				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )  ):
 lowercase__       :       Optional[int]				=						tau * frequency / samplerate
 lowercase__       :       str				=						sin(lowerCamelCase__  )
 lowercase__       :       int				=						cos(lowerCamelCase__  )
 lowercase__       :       Dict				=						_sin / (2 * q_factor)

 lowercase__       :       Dict				=						_sin / 2
 lowercase__       :       Union[str, Any]				=						0
 lowercase__       :       int				=						-ba

 lowercase__       :       List[Any]				=						1 + alpha
 lowercase__       :       Tuple				=						-2 * _cos
 lowercase__       :       List[str]				=						1 - alpha

 lowercase__       :       Dict				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )  ):
 lowercase__       :       str				=						tau * frequency / samplerate
 lowercase__       :       Dict				=						sin(lowerCamelCase__  )
 lowercase__       :       Union[str, Any]				=						cos(lowerCamelCase__  )
 lowercase__       :       Tuple				=						_sin / (2 * q_factor)

 lowercase__       :       List[Any]				=						1 - alpha
 lowercase__       :       Any				=						-2 * _cos
 lowercase__       :       Optional[Any]				=						1 + alpha

 lowercase__       :       List[Any]				=						IIRFilter(2  )
 filt.set_coefficients([ba, ba, ba]       ,		[ba, ba, ba]  )
 return filt


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )       ,		):
 lowercase__       :       Tuple				=						tau * frequency / samplerate
 lowercase__       :       str				=						sin(lowerCamelCase__  )
 lowercase__       :       List[str]				=						cos(lowerCamelCase__  )
 lowercase__       :       Optional[int]				=						_sin / (2 * q_factor)
 lowercase__       :       Optional[Any]				=						10 ** (gain_db / 40)

 lowercase__       :       Optional[Any]				=						1 + alpha * big_a
 lowercase__       :       Dict				=						-2 * _cos
 lowercase__       :       Tuple				=						1 - alpha * big_a
 lowercase__       :       List[str]				=						1 + alpha / big_a
 lowercase__       :       List[Any]				=						-2 * _cos
 lowercase__       :       Optional[int]				=						1 - alpha / big_a

 lowercase__       :       Union[str, Any]				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )       ,		):
 lowercase__       :       Optional[int]				=						tau * frequency / samplerate
 lowercase__       :       Dict				=						sin(lowerCamelCase__  )
 lowercase__       :       Union[str, Any]				=						cos(lowerCamelCase__  )
 lowercase__       :       int				=						_sin / (2 * q_factor)
 lowercase__       :       Union[str, Any]				=						10 ** (gain_db / 40)
 lowercase__       :       Dict				=						(big_a + 1) - (big_a - 1) * _cos
 lowercase__       :       Union[str, Any]				=						(big_a + 1) + (big_a - 1) * _cos
 lowercase__       :       List[Any]				=						(big_a - 1) - (big_a + 1) * _cos
 lowercase__       :       Dict				=						(big_a - 1) + (big_a + 1) * _cos
 lowercase__       :       Optional[int]				=						2 * sqrt(lowerCamelCase__  ) * alpha

 lowercase__       :       str				=						big_a * (pmc + aaa)
 lowercase__       :       Optional[Any]				=						2 * big_a * mpc
 lowercase__       :       str				=						big_a * (pmc - aaa)
 lowercase__       :       Dict				=						ppmc + aaa
 lowercase__       :       List[Any]				=						-2 * pmpc
 lowercase__       :       List[Any]				=						ppmc - aaa

 lowercase__       :       Optional[int]				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt






def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							int       ,		lowerCamelCase__	:							float       ,		lowerCamelCase__	:							float = 1 / sqrt(2  )       ,		):
 lowercase__       :       int				=						tau * frequency / samplerate
 lowercase__       :       List[Any]				=						sin(lowerCamelCase__  )
 lowercase__       :       List[Any]				=						cos(lowerCamelCase__  )
 lowercase__       :       List[str]				=						_sin / (2 * q_factor)
 lowercase__       :       int				=						10 ** (gain_db / 40)
 lowercase__       :       str				=						(big_a + 1) - (big_a - 1) * _cos
 lowercase__       :       Optional[Any]				=						(big_a + 1) + (big_a - 1) * _cos
 lowercase__       :       Optional[int]				=						(big_a - 1) - (big_a + 1) * _cos
 lowercase__       :       int				=						(big_a - 1) + (big_a + 1) * _cos
 lowercase__       :       Optional[Any]				=						2 * sqrt(lowerCamelCase__  ) * alpha

 lowercase__       :       str				=						big_a * (ppmc + aaa)
 lowercase__       :       Union[str, Any]				=						-2 * big_a * pmpc
 lowercase__       :       List[Any]				=						big_a * (ppmc - aaa)
 lowercase__       :       str				=						pmc + aaa
 lowercase__       :       int				=						2 * mpc
 lowercase__       :       Any				=						pmc - aaa

 lowercase__       :       int				=						IIRFilter(2  )
 filt.set_coefficients([aa, aa, aa]       ,		[ba, ba, ba]  )
 return filt