




"""simple docstring"""

import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a							      =     get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
class 							lowercase_			(	__lowerCAmelCase     ,      unittest.TestCase				):



 '''simple docstring'''
 UpperCAmelCase       :	Optional[Any]     =      XLMProphetNetTokenizer
 UpperCAmelCase       :	Tuple     =      False
 UpperCAmelCase       :	List[str]     =      True

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _A     =  XLMProphetNetTokenizer(_UpperCAmelCase       ,				keep_accents=_UpperCAmelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)

 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  '[PAD]'
       _A     =  0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase						)       ,				_UpperCAmelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase						)       ,				_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]       ,				'[PAD]'						)
       self.assertEqual(vocab_keys[1]       ,				'[CLS]'						)
       self.assertEqual(vocab_keys[-1]       ,				'j'						)
       self.assertEqual(len(_UpperCAmelCase						)       ,				1_012						)

 def      lowerCAmelCase_				(					self			:			Dict						):
       self.assertEqual(self.get_tokenizer().vocab_size       ,				1_012						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  XLMProphetNetTokenizer(_UpperCAmelCase       ,				keep_accents=_UpperCAmelCase						)

       _A     =  tokenizer.tokenize('This is a test'						)
       self.assertListEqual(_UpperCAmelCase       ,				['▁This', '▁is', '▁a', '▁t', 'est']						)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)       ,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,				)

       _A     =  tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       self.assertListEqual(
           _UpperCAmelCase       ,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]       ,				)
       _A     =  tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
       self.assertListEqual(
           _UpperCAmelCase       ,				[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
           ]       ,				)

       _A     =  tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
       self.assertListEqual(
           _UpperCAmelCase       ,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '[UNK]',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '[UNK]',
               '.',
           ]       ,				)

 @cached_property
 def      lowerCAmelCase_				(					self			:			Any						):
       return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'						)

 @slow
 def      lowerCAmelCase_				(					self			:			Tuple						):
       _A     =  'Hello World!'
       _A     =  [35_389, 6_672, 49, 2]
       self.assertListEqual(_UpperCAmelCase       ,				self.big_tokenizer.encode(_UpperCAmelCase						)						)



 @slow
 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       # fmt: off
       _A     =  {'input_ids': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_UpperCAmelCase       ,				model_name='microsoft/xprophetnet-large-wiki100-cased'       ,				revision='1acad1643ddd54a44df6a1b797ada8373685d90e'       ,				)






'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_nllb import NllbTokenizer
else:
       __a  			=						None


__a  			=						logging.get_logger(__name__)


__a  			=						{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__a  			=						{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

__a  			=						{
    "facebook/nllb-large-en-ro": 1024,
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
__a  			=						["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]







class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    lowercase   =						VOCAB_FILES_NAMES
    lowercase   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase   =						PRETRAINED_VOCAB_FILES_MAP
    lowercase   =						["input_ids", "attention_mask"]
    lowercase   =						NllbTokenizer

    lowercase   =						[]
    lowercase   =						[]




    def __init__(       self   :	List[str]					,	snake_case_   :	int=None					,	snake_case_   :	Optional[int]=None					,	snake_case_   :	Dict="<s>"					,	snake_case_   :	Optional[Any]="</s>"					,	snake_case_   :	Union[str, Any]="</s>"					,	snake_case_   :	Optional[int]="<s>"					,	snake_case_   :	Any="<unk>"					,	snake_case_   :	Tuple="<pad>"					,	snake_case_   :	Any="<mask>"					,	snake_case_   :	Union[str, Any]=None					,	snake_case_   :	Tuple=None					,	snake_case_   :	Union[str, Any]=None					,	snake_case_   :	Optional[int]=False					,	**snake_case_   :	List[Any]					,	):
          # Mask token behave like a normal word, i.e. include the space before it
          snake_case__    :       Any       		= AddedToken(snake_case_					,	lstrip=snake_case_					,	rstrip=snake_case_		) if isinstance(snake_case_					,	snake_case_		) else mask_token
          snake_case__    :       str       		= legacy_behaviour
          super().__init__(
              vocab_file=snake_case_					,	tokenizer_file=snake_case_					,	bos_token=snake_case_					,	eos_token=snake_case_					,	sep_token=snake_case_					,	cls_token=snake_case_					,	unk_token=snake_case_					,	pad_token=snake_case_					,	mask_token=snake_case_					,	src_lang=snake_case_					,	tgt_lang=snake_case_					,	additional_special_tokens=snake_case_					,	legacy_behaviour=snake_case_					,	**snake_case_					,	)

          snake_case__    :       Optional[Any]       		= vocab_file
          snake_case__    :       str       		= False if not self.vocab_file else True

          snake_case__    :       List[str]       		= FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
                # Only add those special tokens if they are not already there.
                _additional_special_tokens.extend(
                    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

          self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}		)
          snake_case__    :       Optional[int]       		= {
              lang_code: self.convert_tokens_to_ids(snake_case_		) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          snake_case__    :       Any       		= src_lang if src_lang is not None else """eng_Latn"""
          snake_case__    :       Optional[int]       		= self.convert_tokens_to_ids(self._src_lang		)
          snake_case__    :       Any       		= tgt_lang
          self.set_src_lang_special_tokens(self._src_lang		)




    @property
    def lowerCamelCase			(       self   :	str		):
          return self._src_lang




    @src_lang.setter
    def lowerCamelCase			(       self   :	str					,	snake_case_   :	str		):
          snake_case__    :       Any       		= new_src_lang
          self.set_src_lang_special_tokens(self._src_lang		)




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	List[int]					,	snake_case_   :	Optional[List[int]] = None		):
          if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




    def lowerCamelCase			(       self   :	Any					,	snake_case_   :	List[int]					,	snake_case_   :	Optional[List[int]] = None		):
          snake_case__    :       int       		= [self.sep_token_id]
          snake_case__    :       Optional[int]       		= [self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]




    def lowerCamelCase			(       self   :	List[str]					,	snake_case_   :	str					,	snake_case_   :	str					,	snake_case_   :	Optional[str]					,	snake_case_   :	Optional[str]					,	**snake_case_   :	List[Any]		):
          if src_lang is None or tgt_lang is None:
                raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""		)
          snake_case__    :       Any       		= src_lang
          snake_case__    :       str       		= self(snake_case_					,	add_special_tokens=snake_case_					,	return_tensors=snake_case_					,	**snake_case_		)
          snake_case__    :       Dict       		= self.convert_tokens_to_ids(snake_case_		)
          snake_case__    :       str       		= tgt_lang_id
          return inputs




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	List[str]					,	snake_case_   :	str = "eng_Latn"					,	snake_case_   :	Optional[List[str]] = None					,	snake_case_   :	str = "fra_Latn"					,	**snake_case_   :	str					,	):
          snake_case__    :       str       		= src_lang
          snake_case__    :       List[Any]       		= tgt_lang
          return super().prepare_seqaseq_batch(snake_case_					,	snake_case_					,	**snake_case_		)




    def lowerCamelCase			(       self   :	List[str]		):
          return self.set_src_lang_special_tokens(self.src_lang		)




    def lowerCamelCase			(       self   :	Any		):
          return self.set_tgt_lang_special_tokens(self.tgt_lang		)




    def lowerCamelCase			(       self   :	Optional[Any]					,	snake_case_   :	List[str]		):
          snake_case__    :       List[Any]       		= self.convert_tokens_to_ids(snake_case_		)

          if self.legacy_behaviour:
                snake_case__    :       Tuple       		= []
                snake_case__    :       Optional[int]       		= [self.eos_token_id, self.cur_lang_code]
          else:
                snake_case__    :       Tuple       		= [self.cur_lang_code]
                snake_case__    :       int       		= [self.eos_token_id]

          snake_case__    :       List[str]       		= self.convert_ids_to_tokens(self.prefix_tokens		)
          snake_case__    :       Tuple       		= self.convert_ids_to_tokens(self.suffix_tokens		)

          snake_case__    :       Union[str, Any]       		= processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens		)		)					,	)




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	str		):
          snake_case__    :       List[str]       		= self.convert_tokens_to_ids(snake_case_		)
          if self.legacy_behaviour:
                snake_case__    :       int       		= []
                snake_case__    :       Optional[Any]       		= [self.eos_token_id, self.cur_lang_code]
          else:
                snake_case__    :       Dict       		= [self.cur_lang_code]
                snake_case__    :       Dict       		= [self.eos_token_id]

          snake_case__    :       Dict       		= self.convert_ids_to_tokens(self.prefix_tokens		)
          snake_case__    :       Any       		= self.convert_ids_to_tokens(self.suffix_tokens		)

          snake_case__    :       List[Any]       		= processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens		)		)					,	)




    def lowerCamelCase			(       self   :	str					,	snake_case_   :	str					,	snake_case_   :	Optional[str] = None		):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer."""		)

          if not os.path.isdir(snake_case_		):
                logger.error(f"Vocabulary path ({save_directory}) should be a directory."		)
                return
          snake_case__    :       Dict       		= os.path.join(
              snake_case_					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

          if os.path.abspath(self.vocab_file		) != os.path.abspath(snake_case_		):
                copyfile(self.vocab_file					,	snake_case_		)

          return (out_vocab_file,)
