






"""simple docstring"""


from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase				:    List[Any]     =						logging.get_logger(__name__)






class 						snake_case  (  UpperCAmelCase       ):
    __magic_name__         =     ['''input_ids''', '''attention_mask''']





    def __init__(							self	:					Union[str, Any]				,		A	:					Any="[PAD]"				,		A	:					int="[BOS]"				,		A	:					List[str]="[EOS]"				,		A	:					str="[MASK]"				,		A	:					List[Any]="[CLS]"				,		A	:					int="[SEP]"				,		A	:					Any=2_0_4_8				,		**A	:					Dict				,		):


      '''simple docstring'''




      a     :			Dict				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else pad_token
      a     :			int				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else bos_token
      a     :			str				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else eos_token
      a     :			Tuple				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else mask_token
      a     :			List[Any]				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else cls_token
      a     :			List[str]				=   AddedToken(A				,		lstrip=A				,		rstrip=A			) if isinstance(A				,		A			) else sep_token

      super().__init__(
          pad_token=A				,		bos_token=A				,		eos_token=A				,		mask_token=A				,		cls_token=A				,		sep_token=A				,		model_max_length=A				,		**A				,		)

      a     :			List[Any]				=   2**8  # utf is 8 bits

      # define special tokens dict
      a     :			Dict[str, int]				=   {
          self.pad_token: 0,
          self.bos_token: 1,
          self.eos_token: 2,
          self.mask_token: 3,
          self.cls_token: 4,
          self.sep_token: 5,
      }
      a     :			Tuple				=   len(self.special_tokens_encoder			)
      a     :			Dict[int, str]				=   {v: k for k, v in self.special_tokens_encoder.items()}





    def 				lowerCamelCase__      (							self	:					List[str]			):


      '''simple docstring'''




      a     :			Dict				=   self.special_tokens_encoder.copy()
      vocab.update(self.added_tokens_encoder			)
      for i in range(self._utf_vocab_size			):
        a     :			int				=   chr(A			)
        a     :			str				=   i + len(self.special_tokens_encoder			)
      return vocab





    @property
    def 				lowerCamelCase__      (							self	:					Dict			):


      '''simple docstring'''




      return self._utf_vocab_size + self._num_special_tokens





    def 				lowerCamelCase__      (							self	:					Tuple				,		A	:					List[int]				,		A	:					Optional[List[int]] = None				,		A	:					bool = False			):


      '''simple docstring'''




      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=A				,		token_ids_a=A				,		already_has_special_tokens=A			)

      # normal case: some special tokens
      if token_ids_a is None:
        return [1] + [0] * len(A			) + [1]
      return [1] + ([0] * len(A			)) + [1] + ([0] * len(A			)) + [1]





    def 				lowerCamelCase__      (							self	:					Dict				,		A	:					List[int]				,		A	:					Optional[List[int]] = None			):


      '''simple docstring'''




      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      else:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]





    def 				lowerCamelCase__      (							self	:					List[Any]				,		A	:					str			):


      '''simple docstring'''




      a     :			List[str]				=   [chr(A			) for i in text.encode('utf-8'			)]
      return tokens





    def 				lowerCamelCase__      (							self	:					Dict				,		A	:					Tuple			):


      '''simple docstring'''




      if token in self.special_tokens_encoder:
        a     :			List[str]				=   self.special_tokens_encoder[token]
      elif token in self.added_tokens_encoder:
        a     :			Dict				=   self.added_tokens_encoder[token]
      elif len(A			) != 1:
        a     :			Optional[Any]				=   self.unk_token_id
      else:
        a     :			List[str]				=   ord(A			) + self._num_special_tokens
      return token_id





    def 				lowerCamelCase__      (							self	:					Tuple				,		A	:					Dict			):


      '''simple docstring'''




      if index in self.special_tokens_decoder:
        a     :			Union[str, Any]				=   self.special_tokens_decoder[index]
      elif index in self.added_tokens_decoder:
        a     :			Optional[Any]				=   self.added_tokens_decoder[index]
      else:
        a     :			Union[str, Any]				=   chr(index - self._num_special_tokens			)
      return token





    def 				lowerCamelCase__      (							self	:					Optional[Any]				,		A	:					Any			):


      '''simple docstring'''




      a     :			Optional[Any]				=   B''
      for token in tokens:
        if token in self.special_tokens_decoder:
          a     :			Any				=   self.special_tokens_decoder[token].encode('utf-8'			)
        elif token in self.added_tokens_decoder:
          a     :			Optional[int]				=   self.special_tokens_decoder[token].encode('utf-8'			)
        elif token in self.special_tokens_encoder:
          a     :			List[str]				=   token.encode('utf-8'			)
        elif token in self.added_tokens_encoder:
          a     :			List[Any]				=   token.encode('utf-8'			)
        else:
          a     :			Optional[Any]				=   bytes([ord(A			)]			)
        bstring += tok_string
      a     :			Union[str, Any]				=   bstring.decode('utf-8'				,		errors='replace'			)
      return string





    def 				lowerCamelCase__      (							self	:					Optional[int]				,		A	:					str				,		A	:					Optional[str] = None			):


      '''simple docstring'''




      return ()











"""simple docstring"""
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin







class 	a				(					a__   ):
 snake_case__    	=    '''AutoFeatureExtractor'''
 snake_case__    	=    '''Speech2Text2Tokenizer'''



 def __init__(			self							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  super().__init__(_snake_case							,    _snake_case			)
  lowerCAmelCase  							=      self.feature_extractor
  lowerCAmelCase  							=      False



 def __call__(			self							,    *_snake_case							,    **_snake_case			):







  """simple docstring"""






  if self._in_target_context_manager:
   return self.current_processor(*_snake_case							,    **_snake_case			)

  if "raw_speech" in kwargs:
   warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'			)
   lowerCAmelCase  							=      kwargs.pop('raw_speech'			)
  else:
   lowerCAmelCase  							=      kwargs.pop('audio'							,    _snake_case			)
  lowerCAmelCase  							=      kwargs.pop('sampling_rate'							,    _snake_case			)
  lowerCAmelCase  							=      kwargs.pop('text'							,    _snake_case			)
  if len(_snake_case			) > 0:
   lowerCAmelCase  							=      args[0]
   lowerCAmelCase  							=      args[1:]

  if audio is None and text is None:
   raise ValueError('You need to specify either an `audio` or `text` input to process.'			)

  if audio is not None:
   lowerCAmelCase  							=      self.feature_extractor(_snake_case							,    *_snake_case							,    sampling_rate=_snake_case							,    **_snake_case			)
  if text is not None:
   lowerCAmelCase  							=      self.tokenizer(_snake_case							,    **_snake_case			)

  if text is None:
   return inputs
  elif audio is None:
   return encodings
  else:
   lowerCAmelCase  							=      encodings['input_ids']
   return inputs



 def 					UpperCamelCase__   (			self							,    *_snake_case							,    **_snake_case			):







  """simple docstring"""






  return self.tokenizer.batch_decode(*_snake_case							,    **_snake_case			)



 def 					UpperCamelCase__   (			self							,    *_snake_case							,    **_snake_case			):







  """simple docstring"""






  return self.tokenizer.decode(*_snake_case							,    **_snake_case			)



 @contextmanager
 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  warnings.warn(
      '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
      'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
      'your audio inputs, or in a separate call.'			)
  lowerCAmelCase  							=      True
  lowerCAmelCase  							=      self.tokenizer
  yield
  lowerCAmelCase  							=      self.feature_extractor
  lowerCAmelCase  							=      False





