




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_mbartaa import MBartaaTokenizer
else:
      __SCREAMING_SNAKE_CASE			:      Union[str, Any]      			=None


__SCREAMING_SNAKE_CASE			:      str      			=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE			:      Optional[Any]      			={"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

__SCREAMING_SNAKE_CASE			:      Tuple      			={
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

__SCREAMING_SNAKE_CASE			:      str      			={
    """facebook/mbart-large-50-one-to-many-mmt""": 1_024,
}

# fmt: off
__SCREAMING_SNAKE_CASE			:      Dict      			=["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]






class 	A	(						a_						):
     """simple docstring"""

     _snake_case:		Dict   =VOCAB_FILES_NAMES
     _snake_case:		int   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _snake_case:		Union[str, Any]   =PRETRAINED_VOCAB_FILES_MAP
     _snake_case:		Optional[Any]   =["input_ids", "attention_mask"]
     _snake_case:		Union[str, Any]   =MBartaaTokenizer

     _snake_case:		List[int]   =[]
     _snake_case:		List[int]   =[]




     def __init__(			self     :      Dict		,     A__     :      Any=None		,     A__     :      Union[str, Any]=None		,     A__     :      int=None		,     A__     :      List[str]=None		,     A__     :      List[str]="</s>"		,     A__     :      List[Any]="</s>"		,     A__     :      Optional[int]="<s>"		,     A__     :      Tuple="<unk>"		,     A__     :      Any="<pad>"		,     A__     :      Any="<mask>"		,     **A__     :      int		,     ) ->				Dict:
           lowerCamelCase__					     = AddedToken(A__		,     lstrip=A__		,     rstrip=A__	) if isinstance(A__		,     A__	) else mask_token

           lowerCamelCase__					     = kwargs.get('additional_special_tokens'		,     []	)
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               A__		,     src_lang=A__		,     tgt_lang=A__		,     tokenizer_file=A__		,     eos_token=A__		,     sep_token=A__		,     cls_token=A__		,     unk_token=A__		,     pad_token=A__		,     mask_token=A__		,     **A__		,     )

           lowerCamelCase__					     = vocab_file
           lowerCamelCase__					     = False if not self.vocab_file else True

           lowerCamelCase__					     = {
               lang_code: self.convert_tokens_to_ids(A__	) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           lowerCamelCase__					     = src_lang if src_lang is not None else """en_XX"""
           lowerCamelCase__					     = tgt_lang
           lowerCamelCase__					     = self.lang_code_to_id[self._src_lang]
           self.set_src_lang_special_tokens(self._src_lang	)




     @property
     def        __snake_case      (			self     :      Optional[Any]	) ->				str:
           return self._src_lang




     @src_lang.setter
     def        __snake_case      (			self     :      List[str]		,     A__     :      str	) ->				None:
           lowerCamelCase__					     = new_src_lang
           self.set_src_lang_special_tokens(self._src_lang	)




     def        __snake_case      (			self     :      Tuple		,     A__     :      List[int]		,     A__     :      Optional[List[int]] = None	) ->				List[int]:







           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




     def        __snake_case      (			self     :      Optional[Any]		,     A__     :      List[str]		,     A__     :      str = "en_XX"		,     A__     :      Optional[List[str]] = None		,     A__     :      str = "ro_RO"		,     **A__     :      Dict		,     ) ->				BatchEncoding:
           lowerCamelCase__					     = src_lang
           lowerCamelCase__					     = tgt_lang
           return super().prepare_seqaseq_batch(A__		,     A__		,     **A__	)




     def        __snake_case      (			self     :      Union[str, Any]	) ->				int:
           return self.set_src_lang_special_tokens(self.src_lang	)




     def        __snake_case      (			self     :      Optional[Any]	) ->				Optional[int]:
           return self.set_tgt_lang_special_tokens(self.tgt_lang	)




     def        __snake_case      (			self     :      Dict		,     A__     :      str	) ->				None:
           lowerCamelCase__					     = self.convert_tokens_to_ids(A__	)
           lowerCamelCase__					     = [self.cur_lang_code_id]
           lowerCamelCase__					     = [self.eos_token_id]

           lowerCamelCase__					     = self.convert_ids_to_tokens(self.prefix_tokens	)
           lowerCamelCase__					     = self.convert_ids_to_tokens(self.suffix_tokens	)

           lowerCamelCase__					     = processors.TemplateProcessing(
               single=prefix_tokens_str + ['$A'] + suffix_tokens_str		,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str		,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,     self.prefix_tokens + self.suffix_tokens	)	)		,     )




     def        __snake_case      (			self     :      Any		,     A__     :      str	) ->				None:
           lowerCamelCase__					     = self.convert_tokens_to_ids(A__	)
           lowerCamelCase__					     = [self.cur_lang_code_id]
           lowerCamelCase__					     = [self.eos_token_id]

           lowerCamelCase__					     = self.convert_ids_to_tokens(self.prefix_tokens	)
           lowerCamelCase__					     = self.convert_ids_to_tokens(self.suffix_tokens	)

           lowerCamelCase__					     = processors.TemplateProcessing(
               single=prefix_tokens_str + ['$A'] + suffix_tokens_str		,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str		,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,     self.prefix_tokens + self.suffix_tokens	)	)		,     )




     def        __snake_case      (			self     :      Optional[Any]		,     A__     :      Tuple		,     A__     :      str		,     A__     :      Optional[str]		,     A__     :      Optional[str]		,     **A__     :      List[str]	) ->				Optional[int]:







           if src_lang is None or tgt_lang is None:
                 raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
           lowerCamelCase__					     = src_lang
           lowerCamelCase__					     = self(A__		,     add_special_tokens=A__		,     return_tensors=A__		,     **A__	)
           lowerCamelCase__					     = self.convert_tokens_to_ids(A__	)
           lowerCamelCase__					     = tgt_lang_id
           return inputs




     def        __snake_case      (			self     :      Union[str, Any]		,     A__     :      str		,     A__     :      Optional[str] = None	) ->				Tuple[str]:







           if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                     'tokenizer.'	)

           if not os.path.isdir(A__	):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
                 return
           lowerCamelCase__					     = os.path.join(
               A__		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

           if os.path.abspath(self.vocab_file	) != os.path.abspath(A__	):
                 copyfile(self.vocab_file		,     A__	)

           return (out_vocab_file,)


import importlib.metadata
import subprocess
import sys





def 	snake_case_      (__A     :						bool = False     )			->	str:
   __lowerCAmelCase  :			Any					    =						False
   if "IPython" in sys.modules:
      __lowerCAmelCase  :			Optional[int]					    =						"""google.colab""" in str(sys.modules["""IPython"""].get_ipython()     )

   if in_colab:
      if upgrade:
         __lowerCAmelCase  :			int					    =						["""pip""", """install""", """-U""", """torch"""]
         subprocess.run(__A					,   check=__A     )
      # get the current version of torch
      __lowerCAmelCase  :			List[Any]					    =						importlib.metadata.version("""torch"""     )
      __lowerCAmelCase  :			List[Any]					    =						torch_version[: torch_version.rindex("""."""     )]
      __lowerCAmelCase  :			int					    =						f'''https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl'''
      __lowerCAmelCase  :			List[Any]					    =						["""pip""", """install""", xla_wheel]
      subprocess.run(__A					,   check=__A     )
   else:
      raise RuntimeError("""`install_xla` utility works only on google colab."""     )


