


"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_xlnet import XLNetTokenizer
else:
  __snake_case       =       None


__snake_case       =       logging.get_logger(__name__)

__snake_case       =       {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__snake_case       =       {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

__snake_case       =       {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

__snake_case       =       """▁"""

# Segments (not really needed)
__snake_case       =       0
__snake_case       =       1
__snake_case       =       2
__snake_case       =       3
__snake_case       =       4







class _lowerCAmelCase							( snake_case_      ):
   __UpperCAmelCase					:			List[str]				    =    VOCAB_FILES_NAMES
   __UpperCAmelCase					:			List[str]				    =    PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase					:			Any				    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase					:			Tuple				    =    '''left'''
   __UpperCAmelCase					:			str				    =    XLNetTokenizer







   def __init__(      self		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      UpperCamelCase__=False		,      UpperCamelCase__=True		,      UpperCamelCase__=False		,      UpperCamelCase__="<s>"		,      UpperCamelCase__="</s>"		,      UpperCamelCase__="<unk>"		,      UpperCamelCase__="<sep>"		,      UpperCamelCase__="<pad>"		,      UpperCamelCase__="<cls>"		,      UpperCamelCase__="<mask>"		,      UpperCamelCase__=["<eop>", "<eod>"]		,      **UpperCamelCase__		,      )      ->						Optional[Any]:

       '''simple docstring'''




       snake_case       : int    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else mask_token

       super().__init__(
           vocab_file=UpperCamelCase__		,      tokenizer_file=UpperCamelCase__		,      do_lower_case=UpperCamelCase__		,      remove_space=UpperCamelCase__		,      keep_accents=UpperCamelCase__		,      bos_token=UpperCamelCase__		,      eos_token=UpperCamelCase__		,      unk_token=UpperCamelCase__		,      sep_token=UpperCamelCase__		,      pad_token=UpperCamelCase__		,      cls_token=UpperCamelCase__		,      mask_token=UpperCamelCase__		,      additional_special_tokens=UpperCamelCase__		,      **UpperCamelCase__		,      )

       snake_case       : List[Any]    =       3
       snake_case       : Optional[int]    =       do_lower_case
       snake_case       : Tuple    =       remove_space
       snake_case       : str    =       keep_accents
       snake_case       : Union[str, Any]    =       vocab_file
       snake_case       : Optional[Any]    =       False if not self.vocab_file else True







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						List[int]:

       '''simple docstring'''




       snake_case       : str    =       [self.sep_token_id]
       snake_case       : List[Any]    =       [self.cls_token_id]
       if token_ids_a is None:
           return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						List[int]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       [self.sep_token_id]
       snake_case       : List[Any]    =       [2]

       if token_ids_a is None:
           return len(token_ids_a + sep		) * [0] + cls_segment_id
       return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						Tuple[str]:

       '''simple docstring'''




       if not self.can_save_slow_tokenizer:
           raise ValueError(
               "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
               "tokenizer."		)

       if not os.path.isdir(UpperCamelCase__		):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
           return
       snake_case       : int    =       os.path.join(
           UpperCamelCase__		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

       if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCamelCase__		):
           copyfile(self.vocab_file		,      UpperCamelCase__		)

       return (out_vocab_file,)









"""simple docstring"""



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool







class _lowerCAmelCase							( snake_case_      ):
   __UpperCAmelCase					:			List[str]				    =    '''openai/whisper-base'''
   __UpperCAmelCase					:			Any				    =    (
       '''This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the '''
       '''transcribed text.'''
   )
   __UpperCAmelCase					:			Tuple				    =    '''transcriber'''
   __UpperCAmelCase					:			Union[str, Any]				    =    WhisperProcessor
   __UpperCAmelCase					:			str				    =    WhisperForConditionalGeneration

   __UpperCAmelCase					:			int				    =    ['''audio''']
   __UpperCAmelCase					:			Optional[int]				    =    ['''text''']







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Tuple:

       '''simple docstring'''




       return self.pre_processor(UpperCamelCase__		,      return_tensors="pt"		).input_features







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						int:

       '''simple docstring'''




       return self.model.generate(inputs=UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Optional[Any]:

       '''simple docstring'''




       return self.pre_processor.batch_decode(UpperCamelCase__		,      skip_special_tokens=UpperCamelCase__		)[0]





