



"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  lowerCamelCase          =						None


lowerCamelCase          =						logging.get_logger(__name__)

lowerCamelCase          =						{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase          =						{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase          =						{
    """facebook/mbart-large-50-one-to-many-mmt""": 1_024,
}

# fmt: off
lowerCamelCase          =						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]
class       lowercase__   (    UpperCamelCase__      ):







       '''simple docstring'''


       UpperCamelCase						=     VOCAB_FILES_NAMES
       UpperCamelCase						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase						=     PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase						=     ['''input_ids''', '''attention_mask''']
       UpperCamelCase						=     MBartaaTokenizer

       UpperCamelCase						=     []
       UpperCamelCase						=     []



       def __init__(							self			:  Any		,				_UpperCAmelCase			:  List[str]=None		,				_UpperCAmelCase			:  int=None		,				_UpperCAmelCase			:  int=None		,				_UpperCAmelCase			:  Union[str, Any]=None		,				_UpperCAmelCase			:  List[str]="</s>"		,				_UpperCAmelCase			:  Tuple="</s>"		,				_UpperCAmelCase			:  Union[str, Any]="<s>"		,				_UpperCAmelCase			:  str="<unk>"		,				_UpperCAmelCase			:  str="<pad>"		,				_UpperCAmelCase			:  Any="<mask>"		,				**_UpperCAmelCase			:  List[Any]		,				) ->						List[str]:


             '''simple docstring'''




             UpperCAmelCase_			 =							AddedToken(SCREAMING_SNAKE_CASE__		,				lstrip=SCREAMING_SNAKE_CASE__		,				rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__    ) else mask_token

             UpperCAmelCase_			 =							kwargs.get("additional_special_tokens"		,				[]    )
             kwargs["additional_special_tokens"] += [
                 code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
             ]

             super().__init__(
                 SCREAMING_SNAKE_CASE__		,				src_lang=SCREAMING_SNAKE_CASE__		,				tgt_lang=SCREAMING_SNAKE_CASE__		,				tokenizer_file=SCREAMING_SNAKE_CASE__		,				eos_token=SCREAMING_SNAKE_CASE__		,				sep_token=SCREAMING_SNAKE_CASE__		,				cls_token=SCREAMING_SNAKE_CASE__		,				unk_token=SCREAMING_SNAKE_CASE__		,				pad_token=SCREAMING_SNAKE_CASE__		,				mask_token=SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__		,				)

             UpperCAmelCase_			 =							vocab_file
             UpperCAmelCase_			 =							False if not self.vocab_file else True

             UpperCAmelCase_			 =							{
                 lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
             }

             UpperCAmelCase_			 =							src_lang if src_lang is not None else "en_XX"
             UpperCAmelCase_			 =							tgt_lang
             UpperCAmelCase_			 =							self.lang_code_to_id[self._src_lang]
             self.set_src_lang_special_tokens(self._src_lang    )



       @property
       def      lowercase__				(							self			:  Optional[Any]    ) ->						str:


             '''simple docstring'''




             return self._src_lang



       @src_lang.setter
       def      lowercase__				(							self			:  int		,				_UpperCAmelCase			:  str    ) ->						None:


             '''simple docstring'''




             UpperCAmelCase_			 =							new_src_lang
             self.set_src_lang_special_tokens(self._src_lang    )



       def      lowercase__				(							self			:  Dict		,				_UpperCAmelCase			:  List[int]		,				_UpperCAmelCase			:  Optional[List[int]] = None    ) ->						List[int]:


             '''simple docstring'''




             if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + self.suffix_tokens
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



       def      lowercase__				(							self			:  List[Any]		,				_UpperCAmelCase			:  List[str]		,				_UpperCAmelCase			:  str = "en_XX"		,				_UpperCAmelCase			:  Optional[List[str]] = None		,				_UpperCAmelCase			:  str = "ro_RO"		,				**_UpperCAmelCase			:  List[Any]		,				) ->						BatchEncoding:


             '''simple docstring'''




             UpperCAmelCase_			 =							src_lang
             UpperCAmelCase_			 =							tgt_lang
             return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__    )



       def      lowercase__				(							self			:  Any    ) ->						Dict:


             '''simple docstring'''




             return self.set_src_lang_special_tokens(self.src_lang    )



       def      lowercase__				(							self			:  str    ) ->						int:


             '''simple docstring'''




             return self.set_tgt_lang_special_tokens(self.tgt_lang    )



       def      lowercase__				(							self			:  Tuple		,				_UpperCAmelCase			:  str    ) ->						None:


             '''simple docstring'''




             UpperCAmelCase_			 =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )
             UpperCAmelCase_			 =							[self.cur_lang_code_id]
             UpperCAmelCase_			 =							[self.eos_token_id]

             UpperCAmelCase_			 =							self.convert_ids_to_tokens(self.prefix_tokens    )
             UpperCAmelCase_			 =							self.convert_ids_to_tokens(self.suffix_tokens    )

             UpperCAmelCase_			 =							processors.TemplateProcessing(
                 single=prefix_tokens_str + ["$A"] + suffix_tokens_str		,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str		,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,				self.prefix_tokens + self.suffix_tokens    )    )		,				)



       def      lowercase__				(							self			:  Tuple		,				_UpperCAmelCase			:  str    ) ->						None:


             '''simple docstring'''




             UpperCAmelCase_			 =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )
             UpperCAmelCase_			 =							[self.cur_lang_code_id]
             UpperCAmelCase_			 =							[self.eos_token_id]

             UpperCAmelCase_			 =							self.convert_ids_to_tokens(self.prefix_tokens    )
             UpperCAmelCase_			 =							self.convert_ids_to_tokens(self.suffix_tokens    )

             UpperCAmelCase_			 =							processors.TemplateProcessing(
                 single=prefix_tokens_str + ["$A"] + suffix_tokens_str		,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str		,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,				self.prefix_tokens + self.suffix_tokens    )    )		,				)



       def      lowercase__				(							self			:  Union[str, Any]		,				_UpperCAmelCase			:  Optional[Any]		,				_UpperCAmelCase			:  str		,				_UpperCAmelCase			:  Optional[str]		,				_UpperCAmelCase			:  Optional[str]		,				**_UpperCAmelCase			:  List[Any]    ) ->						Optional[int]:


             '''simple docstring'''




             if src_lang is None or tgt_lang is None:
                   raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"    )
             UpperCAmelCase_			 =							src_lang
             UpperCAmelCase_			 =							self(SCREAMING_SNAKE_CASE__		,				add_special_tokens=SCREAMING_SNAKE_CASE__		,				return_tensors=SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__    )
             UpperCAmelCase_			 =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )
             UpperCAmelCase_			 =							tgt_lang_id
             return inputs







       def      lowercase__				(							self			:  List[Any]		,				_UpperCAmelCase			:  str		,				_UpperCAmelCase			:  Optional[str] = None    ) ->						Tuple[str]:


             '''simple docstring'''




             if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                       "tokenizer."    )

             if not os.path.isdir(SCREAMING_SNAKE_CASE__    ):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
                   return
             UpperCAmelCase_			 =							os.path.join(
                 SCREAMING_SNAKE_CASE__		,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

             if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE__    ):
                   copyfile(self.vocab_file		,				SCREAMING_SNAKE_CASE__    )

             return (out_vocab_file,)










import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
     from PIL import Image





class  __lowerCamelCase       (    unittest.TestCase	,				UpperCamelCase__    ):




    """simple docstring"""

    def    a      (       self     :     Any   )      ->       Tuple:
         lowerCAmelCase__		= load_tool("image-captioning"   )
         self.tool.setup()
         lowerCAmelCase__		= load_tool("image-captioning"   ,	remote=SCREAMING_SNAKE_CASE__   )


    def    a      (       self     :     int   )      ->       List[str]:
         lowerCAmelCase__		= Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"   )   ) / "000000039769.png"   )
         lowerCAmelCase__		= self.tool(SCREAMING_SNAKE_CASE__   )
         self.assertEqual(SCREAMING_SNAKE_CASE__   ,	"two cats sleeping on a couch"   )


    def    a      (       self     :     List[str]   )      ->       int:
         lowerCAmelCase__		= Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"   )   ) / "000000039769.png"   )
         lowerCAmelCase__		= self.remote_tool(SCREAMING_SNAKE_CASE__   )
         self.assertEqual(SCREAMING_SNAKE_CASE__   ,	"two cats sleeping on a couch"   )


    def    a      (       self     :     Any   )      ->       Any:
         lowerCAmelCase__		= Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"   )   ) / "000000039769.png"   )
         lowerCAmelCase__		= self.tool(image=SCREAMING_SNAKE_CASE__   )
         self.assertEqual(SCREAMING_SNAKE_CASE__   ,	"two cats sleeping on a couch"   )


    def    a      (       self     :     Dict   )      ->       Any:
         lowerCAmelCase__		= Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO"   )   ) / "000000039769.png"   )
         lowerCAmelCase__		= self.remote_tool(image=SCREAMING_SNAKE_CASE__   )
         self.assertEqual(SCREAMING_SNAKE_CASE__   ,	"two cats sleeping on a couch"   )
