'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase    :     Dict         =       logging.get_logger(__name__)

_UpperCAmelCase    :     Dict         =       '''▁'''

_UpperCAmelCase    :     Optional[Any]         =       {'''vocab_file''': '''sentencepiece.bpe.model'''}

_UpperCAmelCase    :     Any         =       {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_UpperCAmelCase    :     Dict         =       {
    '''facebook/mbart-large-50-one-to-many-mmt''': 10_24,
}

# fmt: off
_UpperCAmelCase    :     Tuple         =       ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']







class lowercase_       (		_UpperCamelCase			):







   """simple docstring"""

   __lowerCAmelCase         =   VOCAB_FILES_NAMES
   __lowerCAmelCase         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase         =   PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase         =   ["input_ids", "attention_mask"]

   __lowerCAmelCase         =   []
   __lowerCAmelCase         =   []
   def __init__(      self      :			List[str], UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			List[str]=None, UpperCamelCase__      :			Optional[int]=None, UpperCamelCase__      :			int="</s>", UpperCamelCase__      :			List[Any]="</s>", UpperCamelCase__      :			Optional[int]="<s>", UpperCamelCase__      :			Tuple="<unk>", UpperCamelCase__      :			Union[str, Any]="<pad>", UpperCamelCase__      :			str="<mask>", UpperCamelCase__      :			Optional[Dict[str, Any]] = None, **UpperCamelCase__      :			Optional[int], )  ->			None:
        # Mask token behave like a normal word, i.e. include the space before it
        _A    		=  AddedToken(UpperCamelCase__, lstrip=UpperCamelCase__, rstrip=UpperCamelCase__     ) if isinstance(UpperCamelCase__, UpperCamelCase__     ) else mask_token

        _A    		=  {} if sp_model_kwargs is None else sp_model_kwargs

        _A    		=  kwargs.get('additional_special_tokens', []     )
        kwargs["additional_special_tokens"] += [
            code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=UpperCamelCase__, tgt_lang=UpperCamelCase__, eos_token=UpperCamelCase__, unk_token=UpperCamelCase__, sep_token=UpperCamelCase__, cls_token=UpperCamelCase__, pad_token=UpperCamelCase__, mask_token=UpperCamelCase__, sp_model_kwargs=self.sp_model_kwargs, **UpperCamelCase__, )

        _A    		=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
        self.sp_model.Load(str(UpperCamelCase__     )     )
        _A    		=  vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # Mimic fairseq token-to-id alignment for the first 4 token
        _A    		=  {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        _A    		=  1

        _A    		=  len(self.sp_model     )
        _A    		=  {
            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase__     )
        }
        _A    		=  {v: k for k, v in self.lang_code_to_id.items()}
        _A    		=  len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

        self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
        _A    		=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}

        _A    		=  src_lang if src_lang is not None else 'en_XX'
        _A    		=  self.lang_code_to_id[self._src_lang]
        _A    		=  tgt_lang
        self.set_src_lang_special_tokens(self._src_lang     )
   @property
   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			int:
        return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token
   @property
   def 	__UpperCAmelCase   (      self      :			int     )  ->			str:
        return self._src_lang
   @src_lang.setter
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			str     )  ->			None:
        _A    		=  new_src_lang
        self.set_src_lang_special_tokens(self._src_lang     )
   def __getstate__(      self      :			Tuple     )  ->			Dict:
        _A    		=  self.__dict__.copy()
        _A    		=  None
        return state
   def __setstate__(      self      :			int, UpperCamelCase__      :			Dict     )  ->			None:
        _A    		=  d

        # for backward compatibility
        if not hasattr(self, 'sp_model_kwargs'     ):
             _A    		=  {}

        _A    		=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
        self.sp_model.Load(self.vocab_file     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			Dict:
        _A    		=  {self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
        vocab.update(self.added_tokens_encoder     )
        return vocab
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			str     )  ->			List[str]:
        return self.sp_model.encode(UpperCamelCase__, out_type=UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Tuple, UpperCamelCase__      :			str     )  ->			int:
        if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
        _A    		=  self.sp_model.PieceToId(UpperCamelCase__     )

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
   def 	__UpperCAmelCase   (      self      :			Optional[int], UpperCamelCase__      :			int     )  ->			str:
        if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset     )
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			Optional[Any]     )  ->			Optional[Any]:
        _A    		=  []
        _A    		=  ''
        _A    		=  False
        for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                  if not prev_is_special:
                       out_string += " "
                  out_string += self.sp_model.decode(UpperCamelCase__     ) + token
                  _A    		=  True
                  _A    		=  []
             else:
                  current_sub_tokens.append(UpperCamelCase__     )
                  _A    		=  False
        out_string += self.sp_model.decode(UpperCamelCase__     )
        return out_string.strip()
   def 	__UpperCAmelCase   (      self      :			Optional[int], UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[str] = None     )  ->			Tuple[str]:
        if not os.path.isdir(UpperCamelCase__     ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
             return
        _A    		=  os.path.join(
            UpperCamelCase__, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

        if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
             copyfile(self.vocab_file, UpperCamelCase__     )
        elif not os.path.isfile(self.vocab_file     ):
             with open(UpperCamelCase__, 'wb'     ) as fi:
                  _A    		=  self.sp_model.serialized_model_proto()
                  fi.write(UpperCamelCase__     )

        return (out_vocab_file,)
   def 	__UpperCAmelCase   (      self      :			List[Any], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None, UpperCamelCase__      :			bool = False     )  ->			List[int]:

        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase__, token_ids_a=UpperCamelCase__, already_has_special_tokens=UpperCamelCase__     )

        _A    		=  [1] * len(self.prefix_tokens     )
        _A    		=  [1] * len(self.suffix_tokens     )
        if token_ids_a is None:
             return prefix_ones + ([0] * len(UpperCamelCase__     )) + suffix_ones
        return prefix_ones + ([0] * len(UpperCamelCase__     )) + ([0] * len(UpperCamelCase__     )) + suffix_ones
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None     )  ->			List[int]:
        if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[str], UpperCamelCase__      :			Optional[str], **UpperCamelCase__      :			Any     )  ->			str:
        if src_lang is None or tgt_lang is None:
             raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'     )
        _A    		=  src_lang
        _A    		=  self(UpperCamelCase__, add_special_tokens=UpperCamelCase__, return_tensors=UpperCamelCase__, **UpperCamelCase__     )
        _A    		=  self.convert_tokens_to_ids(UpperCamelCase__     )
        _A    		=  tgt_lang_id
        return inputs
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			List[str], UpperCamelCase__      :			str = "en_XX", UpperCamelCase__      :			Optional[List[str]] = None, UpperCamelCase__      :			str = "ro_RO", **UpperCamelCase__      :			Dict, )  ->			BatchEncoding:
        _A    		=  src_lang
        _A    		=  tgt_lang
        return super().prepare_seqaseq_batch(UpperCamelCase__, UpperCamelCase__, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			int:
        return self.set_src_lang_special_tokens(self.src_lang     )
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			Any:
        return self.set_tgt_lang_special_tokens(self.tgt_lang     )
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			str     )  ->			None:
        _A    		=  self.lang_code_to_id[src_lang]
        _A    		=  [self.cur_lang_code_id]
        _A    		=  [self.eos_token_id]

   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			str     )  ->			None:
        _A    		=  self.lang_code_to_id[tgt_lang]
        _A    		=  [self.cur_lang_code_id]
        _A    		=  [self.eos_token_id]





import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					'MCTCTFeatureExtractor'
	lowerCAmelCase__   =					'AutoTokenizer'



	def __init__(       self		,    lowercase		,    lowercase					)							->			str:
					super().__init__(lowercase		,    lowercase					)
					lowerCamelCase_         =     self.feature_extractor
					lowerCamelCase_         =     False



	def __call__(       self		,    *lowercase		,    **lowercase					)							->			List[str]:
					# For backward compatibility
					if self._in_target_context_manager:
									return self.current_processor(*lowercase		,    **lowercase					)

					if "raw_speech" in kwargs:
									warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."					)
									lowerCamelCase_         =     kwargs.pop("raw_speech"					)
					else:
									lowerCamelCase_         =     kwargs.pop("audio"		,    lowercase					)
					lowerCamelCase_         =     kwargs.pop("sampling_rate"		,    lowercase					)
					lowerCamelCase_         =     kwargs.pop("text"		,    lowercase					)
					if len(lowercase					) > 0:
									lowerCamelCase_         =     args[0]
									lowerCamelCase_         =     args[1:]

					if audio is None and text is None:
									raise ValueError("You need to specify either an `audio` or `text` input to process."					)

					if audio is not None:
									lowerCamelCase_         =     self.feature_extractor(lowercase		,    *lowercase		,    sampling_rate=lowercase		,    **lowercase					)
					if text is not None:
									lowerCamelCase_         =     self.tokenizer(lowercase		,    **lowercase					)

					if text is None:
									return inputs
					elif audio is None:
									return encodings
					else:
									lowerCamelCase_         =     encodings["input_ids"]
									return inputs



	def        SCREAMING_SNAKE_CASE_(       self		,    *lowercase		,    **lowercase					)							->			Optional[Any]:
					return self.tokenizer.batch_decode(*lowercase		,    **lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self		,    *lowercase		,    **lowercase					)							->			Optional[Any]:
					# For backward compatibility
					if self._in_target_context_manager:
									return self.current_processor.pad(*lowercase		,    **lowercase					)

					lowerCamelCase_         =     kwargs.pop("input_features"		,    lowercase					)
					lowerCamelCase_         =     kwargs.pop("labels"		,    lowercase					)
					if len(lowercase					) > 0:
									lowerCamelCase_         =     args[0]
									lowerCamelCase_         =     args[1:]

					if input_features is not None:
									lowerCamelCase_         =     self.feature_extractor.pad(lowercase		,    *lowercase		,    **lowercase					)
					if labels is not None:
									lowerCamelCase_         =     self.tokenizer.pad(lowercase		,    **lowercase					)

					if labels is None:
									return input_features
					elif input_features is None:
									return labels
					else:
									lowerCamelCase_         =     labels["input_ids"]
									return input_features



	def        SCREAMING_SNAKE_CASE_(       self		,    *lowercase		,    **lowercase					)							->			List[str]:
					return self.tokenizer.decode(*lowercase		,    **lowercase					)



	@contextmanager
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					warnings.warn(
					    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
					    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
					    "your audio inputs, or in a separate call."					)
					lowerCamelCase_         =     True
					lowerCamelCase_         =     self.tokenizer
					yield
					lowerCamelCase_         =     self.feature_extractor
					lowerCamelCase_         =     False


