

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_mbartaa import MBartaaTokenizer
else:
   __UpperCAmelCase      = None


__UpperCAmelCase      = logging.get_logger(__name__)

__UpperCAmelCase      = {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

__UpperCAmelCase      = {
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

__UpperCAmelCase      = {
    """facebook/mbart-large-50-one-to-many-mmt""": 1_024,
}

# fmt: off
__UpperCAmelCase      = ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]



class SCREAMING_SNAKE_CASE	(  a_ ):
  """simple docstring"""






  lowerCamelCase    :			Dict						      =VOCAB_FILES_NAMES
  lowerCamelCase    :			int						      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase    :			Union[str, Any]						      =PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase    :			Optional[Any]						      =["input_ids", "attention_mask"]
  lowerCamelCase    :			Union[str, Any]						      =MBartaaTokenizer

  lowerCamelCase    :			List[int]						      =[]
  lowerCamelCase    :			List[int]						      =[]







  def __init__(			self :    Dict       ,							lowerCAmelCase :    Any=None       ,							lowerCAmelCase :    Union[str, Any]=None       ,							lowerCAmelCase :    int=None       ,							lowerCAmelCase :    List[str]=None       ,							lowerCAmelCase :    List[str]="</s>"       ,							lowerCAmelCase :    List[Any]="</s>"       ,							lowerCAmelCase :    Optional[int]="<s>"       ,							lowerCAmelCase :    Tuple="<unk>"       ,							lowerCAmelCase :    Any="<pad>"       ,							lowerCAmelCase :    Any="<mask>"       ,							**lowerCAmelCase :    int       ,							)							->	Dict:






     """simple docstring"""







     __lowerCAmelCase  :			int					    =						AddedToken(lowerCAmelCase       ,							lstrip=lowerCAmelCase       ,							rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase       ,							lowerCAmelCase    ) else mask_token

     __lowerCAmelCase  :			str					    =						kwargs.get("""additional_special_tokens"""       ,							[]    )
     kwargs["additional_special_tokens"] += [
         code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
     ]

     super().__init__(
         lowerCAmelCase       ,							src_lang=lowerCAmelCase       ,							tgt_lang=lowerCAmelCase       ,							tokenizer_file=lowerCAmelCase       ,							eos_token=lowerCAmelCase       ,							sep_token=lowerCAmelCase       ,							cls_token=lowerCAmelCase       ,							unk_token=lowerCAmelCase       ,							pad_token=lowerCAmelCase       ,							mask_token=lowerCAmelCase       ,							**lowerCAmelCase       ,							)

     __lowerCAmelCase  :			Optional[int]					    =						vocab_file
     __lowerCAmelCase  :			str					    =						False if not self.vocab_file else True

     __lowerCAmelCase  :			Any					    =						{
         lang_code: self.convert_tokens_to_ids(lowerCAmelCase    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
     }

     __lowerCAmelCase  :			List[str]					    =						src_lang if src_lang is not None else """en_XX"""
     __lowerCAmelCase  :			str					    =						tgt_lang
     __lowerCAmelCase  :			List[str]					    =						self.lang_code_to_id[self._src_lang]
     self.set_src_lang_special_tokens(self._src_lang    )







  @property
  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	str:






     """simple docstring"""







     return self._src_lang







  @src_lang.setter
  def        SCREAMING_SNAKE_CASE    (			self :    List[str]       ,							lowerCAmelCase :    str    )							->	None:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						new_src_lang
     self.set_src_lang_special_tokens(self._src_lang    )







  def        SCREAMING_SNAKE_CASE    (			self :    Tuple       ,							lowerCAmelCase :    List[int]       ,							lowerCAmelCase :    Optional[List[int]] = None    )							->	List[int]:






     """simple docstring"""







     if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]       ,							lowerCAmelCase :    List[str]       ,							lowerCAmelCase :    str = "en_XX"       ,							lowerCAmelCase :    Optional[List[str]] = None       ,							lowerCAmelCase :    str = "ro_RO"       ,							**lowerCAmelCase :    Dict       ,							)							->	BatchEncoding:






     """simple docstring"""







     __lowerCAmelCase  :			Dict					    =						src_lang
     __lowerCAmelCase  :			Optional[Any]					    =						tgt_lang
     return super().prepare_seqaseq_batch(lowerCAmelCase       ,							lowerCAmelCase       ,							**lowerCAmelCase    )







  def        SCREAMING_SNAKE_CASE    (			self :    Union[str, Any]    )							->	int:






     """simple docstring"""







     return self.set_src_lang_special_tokens(self.src_lang    )







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]    )							->	Optional[int]:






     """simple docstring"""







     return self.set_tgt_lang_special_tokens(self.tgt_lang    )







  def        SCREAMING_SNAKE_CASE    (			self :    Dict       ,							lowerCAmelCase :    str    )							->	None:






     """simple docstring"""







     __lowerCAmelCase  :			Union[str, Any]					    =						self.convert_tokens_to_ids(lowerCAmelCase    )
     __lowerCAmelCase  :			Tuple					    =						[self.cur_lang_code_id]
     __lowerCAmelCase  :			List[str]					    =						[self.eos_token_id]

     __lowerCAmelCase  :			List[str]					    =						self.convert_ids_to_tokens(self.prefix_tokens    )
     __lowerCAmelCase  :			List[str]					    =						self.convert_ids_to_tokens(self.suffix_tokens    )

     __lowerCAmelCase  :			Any					    =						processors.TemplateProcessing(
         single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens    )    )       ,							)







  def        SCREAMING_SNAKE_CASE    (			self :    Any       ,							lowerCAmelCase :    str    )							->	None:






     """simple docstring"""







     __lowerCAmelCase  :			Any					    =						self.convert_tokens_to_ids(lowerCAmelCase    )
     __lowerCAmelCase  :			Optional[int]					    =						[self.cur_lang_code_id]
     __lowerCAmelCase  :			Optional[Any]					    =						[self.eos_token_id]

     __lowerCAmelCase  :			Union[str, Any]					    =						self.convert_ids_to_tokens(self.prefix_tokens    )
     __lowerCAmelCase  :			Optional[Any]					    =						self.convert_ids_to_tokens(self.suffix_tokens    )

     __lowerCAmelCase  :			Dict					    =						processors.TemplateProcessing(
         single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens    )    )       ,							)







  def        SCREAMING_SNAKE_CASE    (			self :    Optional[Any]       ,							lowerCAmelCase :    Tuple       ,							lowerCAmelCase :    str       ,							lowerCAmelCase :    Optional[str]       ,							lowerCAmelCase :    Optional[str]       ,							**lowerCAmelCase :    List[str]    )							->	Optional[int]:






     """simple docstring"""







     if src_lang is None or tgt_lang is None:
        raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
     __lowerCAmelCase  :			Dict					    =						src_lang
     __lowerCAmelCase  :			int					    =						self(lowerCAmelCase       ,							add_special_tokens=lowerCAmelCase       ,							return_tensors=lowerCAmelCase       ,							**lowerCAmelCase    )
     __lowerCAmelCase  :			Optional[int]					    =						self.convert_tokens_to_ids(lowerCAmelCase    )
     __lowerCAmelCase  :			int					    =						tgt_lang_id
     return inputs







  def        SCREAMING_SNAKE_CASE    (			self :    Union[str, Any]       ,							lowerCAmelCase :    str       ,							lowerCAmelCase :    Optional[str] = None    )							->	Tuple[str]:






     """simple docstring"""







     if not self.can_save_slow_tokenizer:
        raise ValueError(
            """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
            """tokenizer."""    )

     if not os.path.isdir(lowerCAmelCase    ):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
        return
     __lowerCAmelCase  :			List[Any]					    =						os.path.join(
         lowerCAmelCase       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

     if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCAmelCase    ):
        copyfile(self.vocab_file       ,							lowerCAmelCase    )

     return (out_vocab_file,)





from .prime_check import Test

Test()


