


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_nllb import NllbTokenizer
else:
							snake_case_						:			Dict													=							None


snake_case_						:			List[Any]													=							logging.get_logger(__name__)


snake_case_						:			Tuple													=							{'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case_						:			List[Any]													=							{
    '''vocab_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'''
        ),
    },
}

snake_case_						:			Optional[int]													=							{
    '''facebook/nllb-large-en-ro''': 10_24,
    '''facebook/nllb-200-distilled-600M''': 10_24,
}

# fmt: off
snake_case_						:			Optional[int]													=							['''ace_Arab''', '''ace_Latn''', '''acm_Arab''', '''acq_Arab''', '''aeb_Arab''', '''afr_Latn''', '''ajp_Arab''', '''aka_Latn''', '''amh_Ethi''', '''apc_Arab''', '''arb_Arab''', '''ars_Arab''', '''ary_Arab''', '''arz_Arab''', '''asm_Beng''', '''ast_Latn''', '''awa_Deva''', '''ayr_Latn''', '''azb_Arab''', '''azj_Latn''', '''bak_Cyrl''', '''bam_Latn''', '''ban_Latn''', '''bel_Cyrl''', '''bem_Latn''', '''ben_Beng''', '''bho_Deva''', '''bjn_Arab''', '''bjn_Latn''', '''bod_Tibt''', '''bos_Latn''', '''bug_Latn''', '''bul_Cyrl''', '''cat_Latn''', '''ceb_Latn''', '''ces_Latn''', '''cjk_Latn''', '''ckb_Arab''', '''crh_Latn''', '''cym_Latn''', '''dan_Latn''', '''deu_Latn''', '''dik_Latn''', '''dyu_Latn''', '''dzo_Tibt''', '''ell_Grek''', '''eng_Latn''', '''epo_Latn''', '''est_Latn''', '''eus_Latn''', '''ewe_Latn''', '''fao_Latn''', '''pes_Arab''', '''fij_Latn''', '''fin_Latn''', '''fon_Latn''', '''fra_Latn''', '''fur_Latn''', '''fuv_Latn''', '''gla_Latn''', '''gle_Latn''', '''glg_Latn''', '''grn_Latn''', '''guj_Gujr''', '''hat_Latn''', '''hau_Latn''', '''heb_Hebr''', '''hin_Deva''', '''hne_Deva''', '''hrv_Latn''', '''hun_Latn''', '''hye_Armn''', '''ibo_Latn''', '''ilo_Latn''', '''ind_Latn''', '''isl_Latn''', '''ita_Latn''', '''jav_Latn''', '''jpn_Jpan''', '''kab_Latn''', '''kac_Latn''', '''kam_Latn''', '''kan_Knda''', '''kas_Arab''', '''kas_Deva''', '''kat_Geor''', '''knc_Arab''', '''knc_Latn''', '''kaz_Cyrl''', '''kbp_Latn''', '''kea_Latn''', '''khm_Khmr''', '''kik_Latn''', '''kin_Latn''', '''kir_Cyrl''', '''kmb_Latn''', '''kon_Latn''', '''kor_Hang''', '''kmr_Latn''', '''lao_Laoo''', '''lvs_Latn''', '''lij_Latn''', '''lim_Latn''', '''lin_Latn''', '''lit_Latn''', '''lmo_Latn''', '''ltg_Latn''', '''ltz_Latn''', '''lua_Latn''', '''lug_Latn''', '''luo_Latn''', '''lus_Latn''', '''mag_Deva''', '''mai_Deva''', '''mal_Mlym''', '''mar_Deva''', '''min_Latn''', '''mkd_Cyrl''', '''plt_Latn''', '''mlt_Latn''', '''mni_Beng''', '''khk_Cyrl''', '''mos_Latn''', '''mri_Latn''', '''zsm_Latn''', '''mya_Mymr''', '''nld_Latn''', '''nno_Latn''', '''nob_Latn''', '''npi_Deva''', '''nso_Latn''', '''nus_Latn''', '''nya_Latn''', '''oci_Latn''', '''gaz_Latn''', '''ory_Orya''', '''pag_Latn''', '''pan_Guru''', '''pap_Latn''', '''pol_Latn''', '''por_Latn''', '''prs_Arab''', '''pbt_Arab''', '''quy_Latn''', '''ron_Latn''', '''run_Latn''', '''rus_Cyrl''', '''sag_Latn''', '''san_Deva''', '''sat_Beng''', '''scn_Latn''', '''shn_Mymr''', '''sin_Sinh''', '''slk_Latn''', '''slv_Latn''', '''smo_Latn''', '''sna_Latn''', '''snd_Arab''', '''som_Latn''', '''sot_Latn''', '''spa_Latn''', '''als_Latn''', '''srd_Latn''', '''srp_Cyrl''', '''ssw_Latn''', '''sun_Latn''', '''swe_Latn''', '''swh_Latn''', '''szl_Latn''', '''tam_Taml''', '''tat_Cyrl''', '''tel_Telu''', '''tgk_Cyrl''', '''tgl_Latn''', '''tha_Thai''', '''tir_Ethi''', '''taq_Latn''', '''taq_Tfng''', '''tpi_Latn''', '''tsn_Latn''', '''tso_Latn''', '''tuk_Latn''', '''tum_Latn''', '''tur_Latn''', '''twi_Latn''', '''tzm_Tfng''', '''uig_Arab''', '''ukr_Cyrl''', '''umb_Latn''', '''urd_Arab''', '''uzn_Latn''', '''vec_Latn''', '''vie_Latn''', '''war_Latn''', '''wol_Latn''', '''xho_Latn''', '''ydd_Hebr''', '''yor_Latn''', '''yue_Hant''', '''zho_Hans''', '''zho_Hant''', '''zul_Latn''']

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						VOCAB_FILES_NAMES
						UpperCAmelCase             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase             =						["input_ids", "attention_mask"]
						UpperCAmelCase             =						NllbTokenizer

						UpperCAmelCase             =						[]
						UpperCAmelCase             =						[]




						def __init__(       self					:      Optional[int]				,    _a					:      Union[str, Any]=None				,    _a					:      str=None				,    _a					:      Any="<s>"				,    _a					:      int="</s>"				,    _a					:      Union[str, Any]="</s>"				,    _a					:      Tuple="<s>"				,    _a					:      Any="<unk>"				,    _a					:      str="<pad>"				,    _a					:      int="<mask>"				,    _a					:      str=None				,    _a					:      int=None				,    _a					:      Optional[Any]=None				,    _a					:      Optional[Any]=False				,    **_a					:      List[str]				,    )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AddedToken(_a				,    lstrip=_a				,    rstrip=_a  ) if isinstance(_a				,    _a  ) else mask_token
										_SCREAMING_SNAKE_CASE			  =legacy_behaviour
										super().__init__(
										    vocab_file=_a				,    tokenizer_file=_a				,    bos_token=_a				,    eos_token=_a				,    sep_token=_a				,    cls_token=_a				,    unk_token=_a				,    pad_token=_a				,    mask_token=_a				,    src_lang=_a				,    tgt_lang=_a				,    additional_special_tokens=_a				,    legacy_behaviour=_a				,    **_a				,    )

										_SCREAMING_SNAKE_CASE			  =vocab_file
										_SCREAMING_SNAKE_CASE			  =False if not self.vocab_file else True

										_SCREAMING_SNAKE_CASE			  =FAIRSEQ_LANGUAGE_CODES.copy()

										if additional_special_tokens is not None:
														# Only add those special tokens if they are not already there.
														_additional_special_tokens.extend(
														    [t for t in additional_special_tokens if t not in _additional_special_tokens]  )

										self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}  )
										_SCREAMING_SNAKE_CASE			  ={
										    lang_code: self.convert_tokens_to_ids(_a  ) for lang_code in FAIRSEQ_LANGUAGE_CODES
										}

										_SCREAMING_SNAKE_CASE			  =src_lang if src_lang is not None else '''eng_Latn'''
										_SCREAMING_SNAKE_CASE			  =self.convert_tokens_to_ids(self._src_lang  )
										_SCREAMING_SNAKE_CASE			  =tgt_lang
										self.set_src_lang_special_tokens(self._src_lang  )




						@property
						def 							__UpperCamelCase				(       self					:      str  )							->     str:


										"""simple docstring"""





										return self._src_lang




						@src_lang.setter
						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      str  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =new_src_lang
										self.set_src_lang_special_tokens(self._src_lang  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


										"""simple docstring"""





										if token_ids_a is None:
														return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE			  =[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep  ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      str				,    _a					:      str				,    _a					:      Optional[str]				,    _a					:      Optional[str]				,    **_a					:      Optional[int]  )							->     Union[str, Any]:


										"""simple docstring"""





										if src_lang is None or tgt_lang is None:
														raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''  )
										_SCREAMING_SNAKE_CASE			  =src_lang
										_SCREAMING_SNAKE_CASE			  =self(_a				,    add_special_tokens=_a				,    return_tensors=_a				,    **_a  )
										_SCREAMING_SNAKE_CASE			  =self.convert_tokens_to_ids(_a  )
										_SCREAMING_SNAKE_CASE			  =tgt_lang_id
										return inputs




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      List[str]				,    _a					:      str = "eng_Latn"				,    _a					:      Optional[List[str]] = None				,    _a					:      str = "fra_Latn"				,    **_a					:      str				,    )							->     BatchEncoding:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =src_lang
										_SCREAMING_SNAKE_CASE			  =tgt_lang
										return super().prepare_seqaseq_batch(_a				,    _a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]  )							->     str:


										"""simple docstring"""





										return self.set_src_lang_special_tokens(self.src_lang  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     str:


										"""simple docstring"""





										return self.set_tgt_lang_special_tokens(self.tgt_lang  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      Tuple  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.convert_tokens_to_ids(_a  )

										if self.legacy_behaviour:
														_SCREAMING_SNAKE_CASE			  =[]
														_SCREAMING_SNAKE_CASE			  =[self.eos_token_id, self.cur_lang_code]
										else:
														_SCREAMING_SNAKE_CASE			  =[self.cur_lang_code]
														_SCREAMING_SNAKE_CASE			  =[self.eos_token_id]

										_SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(self.prefix_tokens  )
										_SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(self.suffix_tokens  )

										_SCREAMING_SNAKE_CASE			  =processors.TemplateProcessing(
										    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,    pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,    special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,    self.prefix_tokens + self.suffix_tokens  )  )				,    )




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      str  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.convert_tokens_to_ids(_a  )
										if self.legacy_behaviour:
														_SCREAMING_SNAKE_CASE			  =[]
														_SCREAMING_SNAKE_CASE			  =[self.eos_token_id, self.cur_lang_code]
										else:
														_SCREAMING_SNAKE_CASE			  =[self.cur_lang_code]
														_SCREAMING_SNAKE_CASE			  =[self.eos_token_id]

										_SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(self.prefix_tokens  )
										_SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(self.suffix_tokens  )

										_SCREAMING_SNAKE_CASE			  =processors.TemplateProcessing(
										    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,    pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,    special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,    self.prefix_tokens + self.suffix_tokens  )  )				,    )




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


										"""simple docstring"""





										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''  )

										if not os.path.isdir(_a  ):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory."  )
														return
										_SCREAMING_SNAKE_CASE			  =os.path.join(
										    _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

										if os.path.abspath(self.vocab_file  ) != os.path.abspath(_a  ):
														copyfile(self.vocab_file				,    _a  )

										return (out_vocab_file,)



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						ProphetNetTokenizer
						UpperCAmelCase             =						False




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     int:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']  )
										with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''  ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Union[str, Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''UNwant\u00E9d,running'''
										_SCREAMING_SNAKE_CASE			  ='''unwanted, running'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class(self.vocab_file  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''UNwant\u00E9d,running'''  )
										self.assertListEqual(_a				,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_a  )				,    [9, 6, 7, 12, 10, 11]  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )				,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''h\u00E9llo''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    never_split=['''[UNK]''']  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										_SCREAMING_SNAKE_CASE			  ={}
										for i, token in enumerate(_a  ):
														_SCREAMING_SNAKE_CASE			  =i
										_SCREAMING_SNAKE_CASE			  =WordpieceTokenizer(vocab=_a				,    unk_token='''[UNK]'''  )

										self.assertListEqual(tokenizer.tokenize(''''''  )				,    []  )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )				,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )				,    ['''[UNK]''', '''runn''', '''##ing''']  )




						@require_torch
						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										_SCREAMING_SNAKE_CASE			  =[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
										_SCREAMING_SNAKE_CASE			  =tokenizer(_a				,    padding=_a				,    return_tensors='''pt'''  )
										self.assertIsInstance(_a				,    _a  )
										_SCREAMING_SNAKE_CASE			  =list(batch.input_ids.numpy()[0]  )
										self.assertListEqual(_a				,    _a  )

										self.assertEqual((2, 9)				,    batch.input_ids.shape  )
										self.assertEqual((2, 9)				,    batch.attention_mask.shape  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Optional[int]:


										"""simple docstring"""





										self.assertTrue(_is_whitespace(''' '''  )  )
										self.assertTrue(_is_whitespace('''\t'''  )  )
										self.assertTrue(_is_whitespace('''\r'''  )  )
										self.assertTrue(_is_whitespace('''\n'''  )  )
										self.assertTrue(_is_whitespace('''\u00A0'''  )  )

										self.assertFalse(_is_whitespace('''A'''  )  )
										self.assertFalse(_is_whitespace('''-'''  )  )




						def 							__UpperCamelCase				(       self					:      int  )							->     Dict:


										"""simple docstring"""





										self.assertTrue(_is_control('''\u0005'''  )  )

										self.assertFalse(_is_control('''A'''  )  )
										self.assertFalse(_is_control(''' '''  )  )
										self.assertFalse(_is_control('''\t'''  )  )
										self.assertFalse(_is_control('''\r'''  )  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     List[str]:


										"""simple docstring"""





										self.assertTrue(_is_punctuation('''-'''  )  )
										self.assertTrue(_is_punctuation('''$'''  )  )
										self.assertTrue(_is_punctuation('''`'''  )  )
										self.assertTrue(_is_punctuation('''.'''  )  )

										self.assertFalse(_is_punctuation('''A'''  )  )
										self.assertFalse(_is_punctuation(''' '''  )  )




						@slow
						def 							__UpperCamelCase				(       self					:      Tuple  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''sequence builders'''				,    add_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''multi-sequence build'''				,    add_special_tokens=_a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a				,    _a  )

										assert encoded_sentence == text + [102]
										assert encoded_pair == text + [102] + text_a + [102]