


import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A		:				Optional[Any]									=       logging.get_logger(__name__)

_A		:				Dict									=       '''▁'''

_A		:				List[Any]									=       {'''vocab_file''': '''prophetnet.tokenizer'''}

_A		:				Optional[Any]									=       {
    '''vocab_file''': {
        '''microsoft/xprophetnet-large-wiki100-cased''': (
            '''https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'''
        ),
    }
}

_A		:				Dict									=       {
    '''microsoft/xprophetnet-large-wiki100-cased''': {'''do_lower_case''': False},
}

_A		:				Any									=       {
    '''microsoft/xprophetnet-large-wiki100-cased''': 512,
}



def     __lowerCAmelCase						(							snake_case	:    Dict )       ->							Tuple:
   __lowerCamelCase:				List[Any]     =	collections.OrderedDict()
   with open(snake_case						,      """r"""						,      encoding="""utf-8""" ) as reader:
      __lowerCamelCase:				Optional[int]     =	reader.readlines()
   for index, token in enumerate(snake_case ):
      __lowerCamelCase:				Dict     =	token.rstrip("""\n""" )
      __lowerCamelCase:				Any     =	index
   return vocab







class        a					(			lowercase_   ):
     UpperCAmelCase__ :  List[Any]						      =	VOCAB_FILES_NAMES
     UpperCAmelCase__ :  List[Any]						      =	PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__ :  int						      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__ :  str						      =	['''input_ids''', '''attention_mask''']





     def __init__(    self					:				Union[str, Any]    ,	SCREAMING_SNAKE_CASE_					:				List[Any]    ,	SCREAMING_SNAKE_CASE_					:				Optional[Any]="[SEP]"    ,	SCREAMING_SNAKE_CASE_					:				List[str]="[SEP]"    ,	SCREAMING_SNAKE_CASE_					:				Optional[Any]="[SEP]"    ,	SCREAMING_SNAKE_CASE_					:				int="[UNK]"    ,	SCREAMING_SNAKE_CASE_					:				int="[PAD]"    ,	SCREAMING_SNAKE_CASE_					:				Dict="[CLS]"    ,	SCREAMING_SNAKE_CASE_					:				Tuple="[MASK]"    ,	SCREAMING_SNAKE_CASE_					:				Optional[Dict[str, Any]] = None    ,	**SCREAMING_SNAKE_CASE_					:				Any    ,	):
        __lowerCamelCase:				Optional[int]     =	{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=UpperCamelCase__    ,	eos_token=UpperCamelCase__    ,	sep_token=UpperCamelCase__    ,	unk_token=UpperCamelCase__    ,	pad_token=UpperCamelCase__    ,	cls_token=UpperCamelCase__    ,	mask_token=UpperCamelCase__    ,	sp_model_kwargs=self.sp_model_kwargs    ,	**UpperCamelCase__    ,	)

        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""							)
           raise

        __lowerCamelCase:				Optional[int]     =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
        self.sp_model.Load(str(UpperCamelCase__							)							)
        __lowerCamelCase:				List[str]     =	vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # put special tokens and [unused] tokens into the vocab
        __lowerCamelCase:				int     =	{"""[PAD]""": 0, """[CLS]""": 1, """[SEP]""": 2, """[UNK]""": 3, """[MASK]""": 4}

        for i in range(10							):
           __lowerCamelCase:				str     =	F'[unused{i}]'
           __lowerCamelCase:				Any     =	5 + i

        # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
        __lowerCamelCase:				Optional[int]     =	12
        __lowerCamelCase:				Optional[Any]     =	{v: k for k, v in self.fairseq_tokens_to_ids.items()}
        for k in self.fairseq_tokens_to_ids.keys():
           self.unique_no_split_tokens.append(UpperCamelCase__							)





     def __getstate__(    self					:				Dict							):
        __lowerCamelCase:				Dict     =	self.__dict__.copy()
        __lowerCamelCase:				Any     =	None
        return state





     def __setstate__(    self					:				Dict    ,	SCREAMING_SNAKE_CASE_					:				Optional[Any]							):
        __lowerCamelCase:				Optional[int]     =	d
        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""							)
           raise

        # for backward compatibility
        if not hasattr(self    ,	"""sp_model_kwargs"""							):
           __lowerCamelCase:				Any     =	{}

        __lowerCamelCase:				Dict     =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
        self.sp_model.Load(self.vocab_file							)





     def  SCREAMING_SNAKE_CASE__       (    self					:				List[Any]    ,	SCREAMING_SNAKE_CASE_					:				List[int]    ,	SCREAMING_SNAKE_CASE_					:				Optional[List[int]] = None    ,	SCREAMING_SNAKE_CASE_					:				bool = False							):







        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=UpperCamelCase__    ,	token_ids_a=UpperCamelCase__    ,	already_has_special_tokens=UpperCamelCase__							)

        if token_ids_a is None:
           return ([0] * len(UpperCamelCase__							)) + [1]
        return ([0] * len(UpperCamelCase__							)) + [1] + ([0] * len(UpperCamelCase__							)) + [1]





     def  SCREAMING_SNAKE_CASE__       (    self					:				Tuple    ,	SCREAMING_SNAKE_CASE_					:				List[int]    ,	SCREAMING_SNAKE_CASE_					:				Optional[List[int]] = None							):
        __lowerCamelCase:				str     =	[self.sep_token_id]

        if token_ids_a is None:
           return len(token_ids_a + sep							) * [0]
        return len(token_ids_a + sep + sep + token_ids_a + sep							) * [0]





     @property
     def  SCREAMING_SNAKE_CASE__       (    self					:				List[str]							):
        return len(self.sp_model							) + self.fairseq_offset





     def  SCREAMING_SNAKE_CASE__       (    self					:				Optional[int]							):
        __lowerCamelCase:				Any     =	{self.convert_ids_to_tokens(UpperCamelCase__							): i for i in range(self.vocab_size							)}
        vocab.update(self.added_tokens_encoder							)
        return vocab





     def  SCREAMING_SNAKE_CASE__       (    self					:				List[str]    ,	SCREAMING_SNAKE_CASE_					:				str							):
        return self.sp_model.encode(UpperCamelCase__    ,	out_type=UpperCamelCase__							)





     def  SCREAMING_SNAKE_CASE__       (    self					:				List[Any]    ,	SCREAMING_SNAKE_CASE_					:				Tuple							):







        if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
        __lowerCamelCase:				List[Any]     =	self.sp_model.PieceToId(UpperCamelCase__							)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





     def  SCREAMING_SNAKE_CASE__       (    self					:				int    ,	SCREAMING_SNAKE_CASE_					:				Tuple							):







        if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset							)





     def  SCREAMING_SNAKE_CASE__       (    self					:				Optional[Any]    ,	SCREAMING_SNAKE_CASE_					:				List[Any]							):
        __lowerCamelCase:				int     =	"""""".join(UpperCamelCase__							).replace(UpperCamelCase__    ,	""" """							).strip()
        return out_string





     def  SCREAMING_SNAKE_CASE__       (    self					:				str    ,	SCREAMING_SNAKE_CASE_					:				str    ,	SCREAMING_SNAKE_CASE_					:				Optional[str] = None							):







        if not os.path.isdir(UpperCamelCase__							):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
           return
        __lowerCamelCase:				Union[str, Any]     =	os.path.join(
            UpperCamelCase__    ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

        if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase__							) and os.path.isfile(self.vocab_file							):
           copyfile(self.vocab_file    ,	UpperCamelCase__							)
        elif not os.path.isfile(self.vocab_file							):
           with open(UpperCamelCase__    ,	"""wb"""							) as fi:
              __lowerCamelCase:				Optional[Any]     =	self.sp_model.serialized_model_proto()
              fi.write(UpperCamelCase__							)

        return (out_vocab_file,)







     def  SCREAMING_SNAKE_CASE__       (    self					:				List[str]    ,	SCREAMING_SNAKE_CASE_					:				List[int]    ,	SCREAMING_SNAKE_CASE_					:				Optional[List[int]] = None							):







        if token_ids_a is None:
           return token_ids_a + [self.sep_token_id]
        __lowerCamelCase:				str     =	[self.sep_token_id]
        return token_ids_a + sep + token_ids_a + sep






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin



class        a					(			_UpperCAmelCase   ):
     UpperCAmelCase__ :  Dict						      =	"Speech2TextFeatureExtractor"
     UpperCAmelCase__ :  str						      =	"Speech2TextTokenizer"





     def __init__(    self					:				Optional[Any]    ,	SCREAMING_SNAKE_CASE_					:				List[str]    ,	SCREAMING_SNAKE_CASE_					:				Optional[int]							):
        super().__init__(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							)
        __lowerCamelCase:				Any     =	self.feature_extractor
        __lowerCamelCase:				List[Any]     =	False





     def __call__(    self					:				List[str]    ,	*SCREAMING_SNAKE_CASE_					:				Union[str, Any]    ,	**SCREAMING_SNAKE_CASE_					:				List[Any]							):
        # For backward compatibility
        if self._in_target_context_manager:
           return self.current_processor(*SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)

        if "raw_speech" in kwargs:
           warnings.warn("""Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."""							)
           __lowerCamelCase:				Dict     =	kwargs.pop("""raw_speech"""							)
        else:
           __lowerCamelCase:				int     =	kwargs.pop("""audio"""    ,	SCREAMING_SNAKE_CASE_							)
        __lowerCamelCase:				Dict     =	kwargs.pop("""sampling_rate"""    ,	SCREAMING_SNAKE_CASE_							)
        __lowerCamelCase:				int     =	kwargs.pop("""text"""    ,	SCREAMING_SNAKE_CASE_							)
        if len(SCREAMING_SNAKE_CASE_							) > 0:
           __lowerCamelCase:				Dict     =	args[0]
           __lowerCamelCase:				Optional[Any]     =	args[1:]

        if audio is None and text is None:
           raise ValueError("""You need to specify either an `audio` or `text` input to process."""							)

        if audio is not None:
           __lowerCamelCase:				List[str]     =	self.feature_extractor(SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_    ,	sampling_rate=SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)
        if text is not None:
           __lowerCamelCase:				Any     =	self.tokenizer(SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)

        if text is None:
           return inputs
        elif audio is None:
           return encodings
        else:
           __lowerCamelCase:				List[Any]     =	encodings["""input_ids"""]
           return inputs





     def  SCREAMING_SNAKE_CASE__       (    self					:				List[str]    ,	*SCREAMING_SNAKE_CASE_					:				Optional[Any]    ,	**SCREAMING_SNAKE_CASE_					:				str							):
        return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)





     def  SCREAMING_SNAKE_CASE__       (    self					:				Tuple    ,	*SCREAMING_SNAKE_CASE_					:				Any    ,	**SCREAMING_SNAKE_CASE_					:				List[Any]							):
        return self.tokenizer.decode(*SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)







     @contextmanager
     def  SCREAMING_SNAKE_CASE__       (    self					:				Any							):
        warnings.warn(
            """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
            """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
            """your audio inputs, or in a separate call."""							)
        __lowerCamelCase:				Any     =	True
        __lowerCamelCase:				Any     =	self.tokenizer
        yield
        __lowerCamelCase:				Any     =	self.feature_extractor
        __lowerCamelCase:				Optional[Any]     =	False


