



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class        snake_case__    (lowerCamelCase__ ,     unittest.TestCase		):







						"""simple docstring"""




						__lowerCAmelCase					:Optional[Any]      =					CanineTokenizer
						__lowerCAmelCase					:Optional[int]      =					False

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									super().setUp()
									a__       :						Union[str, Any]							=  CanineTokenizer()
									tokenizer.save_pretrained(self.tmpdirname  )

						@cached_property
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Dict:
									"""simple docstring"""




									return CanineTokenizer.from_pretrained("""google/canine-s"""  )

						def 							SCREAMING_SNAKE_CASE__(       self ,       **__lowercase  ) ->				List[str]:
									"""simple docstring"""




									a__       :						Optional[Any]							=  self.tokenizer_class.from_pretrained(self.tmpdirname ,       **lowercase__  )
									a__       :						Optional[int]							=  1_0_2_4
									return tokenizer

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[str]:
									"""simple docstring"""




									a__       :						str							=  self.canine_tokenizer
									a__       :						List[str]							=  ["""Life is like a box of chocolates.""", """You never know what you\'re gonna get."""]
									# fmt: off
									a__       :						Dict							=  [5_7_3_4_4, 7_6, 1_0_5, 1_0_2, 1_0_1, 3_2, 1_0_5, 1_1_5, 3_2, 1_0_8, 1_0_5, 1_0_7, 1_0_1, 3_2, 9_7, 3_2, 9_8, 1_1_1, 1_2_0, 3_2, 1_1_1, 1_0_2, 3_2, 9_9, 1_0_4, 1_1_1, 9_9, 1_1_1, 1_0_8, 9_7, 1_1_6, 1_0_1, 1_1_5, 4_6, 5_7_3_4_5, 0, 0, 0, 0]
									# fmt: on
									a__       :						Union[str, Any]							=  tokenizer(lowercase__ ,       padding=lowercase__ ,       return_tensors="""pt"""  )
									self.assertIsInstance(lowercase__ ,       lowercase__  )

									a__       :						Any							=  list(batch.input_ids.numpy()[0]  )

									self.assertListEqual(lowercase__ ,       lowercase__  )

									self.assertEqual((2, 3_9) ,       batch.input_ids.shape  )
									self.assertEqual((2, 3_9) ,       batch.attention_mask.shape  )

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									a__       :						int							=  self.canine_tokenizer
									a__       :						str							=  ["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
									a__       :						Optional[int]							=  tokenizer(lowercase__ ,       padding=lowercase__ ,       return_tensors="""pt"""  )
									# check if input_ids, attention_mask and token_type_ids are returned
									self.assertIn("""input_ids""" ,       lowercase__  )
									self.assertIn("""attention_mask""" ,       lowercase__  )
									self.assertIn("""token_type_ids""" ,       lowercase__  )

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[str]:
									"""simple docstring"""




									a__       :						int							=  self.canine_tokenizer
									a__       :						Any							=  [
									    """What\'s the weater?""",
									    """It\'s about 25 degrees.""",
									]
									a__       :						Any							=  tokenizer(
									    text_target=lowercase__ ,       max_length=3_2 ,       padding="""max_length""" ,       truncation=lowercase__ ,       return_tensors="""pt"""  )
									self.assertEqual(3_2 ,       targets["""input_ids"""].shape[1]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Any:
									"""simple docstring"""




									a__       :						str							=  self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															self.assertNotEqual(tokenizer.model_max_length ,       4_2  )

															   # Now let's start the test
									a__       :						str							=  self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															# Isolate this from the other tests because we save additional tokens/etc
															a__       :						List[str]							=  tempfile.mkdtemp()

															a__       :						Tuple							=  """ He is very happy, UNwant\u00E9d,running"""
															a__       :						Optional[int]							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															tokenizer.save_pretrained(lowercase__  )

															a__       :						List[Any]							=  tokenizer.__class__.from_pretrained(lowercase__  )
															a__       :						Optional[Any]							=  after_tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															self.assertListEqual(lowercase__ ,       lowercase__  )

															shutil.rmtree(lowercase__  )

									a__       :						Tuple							=  self.get_tokenizers(model_max_length=4_2  )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															# Isolate this from the other tests because we save additional tokens/etc
															a__       :						Union[str, Any]							=  tempfile.mkdtemp()

															a__       :						List[str]							=  """ He is very happy, UNwant\u00E9d,running"""

															a__       :						List[str]							=  tokenizer.additional_special_tokens

															# We can add a new special token for Canine as follows:
															a__       :						List[Any]							=  chr(0xe007  )
															additional_special_tokens.append(lowercase__  )
															tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}  )
															a__       :						List[str]							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															tokenizer.save_pretrained(lowercase__  )

															a__       :						Dict							=  tokenizer.__class__.from_pretrained(lowercase__  )
															a__       :						Dict							=  after_tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															self.assertListEqual(lowercase__ ,       lowercase__  )
															self.assertIn(lowercase__ ,       after_tokenizer.additional_special_tokens  )
															self.assertEqual(after_tokenizer.model_max_length ,       4_2  )

															a__       :						List[Any]							=  tokenizer.__class__.from_pretrained(lowercase__ ,       model_max_length=4_3  )
															self.assertEqual(tokenizer.model_max_length ,       4_3  )

															shutil.rmtree(lowercase__  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									a__       :						Dict							=  self.get_tokenizers(do_lower_case=lowercase__  )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															a__	,						a__       :						Union[str, Any]							=  self.get_clean_sequence(lowercase__  )

															# a special token for Canine can be defined as follows:
															a__       :						List[str]							=  0xe005
															a__       :						Any							=  chr(lowercase__  )

															tokenizer.add_special_tokens({"""cls_token""": special_token}  )
															a__       :						Optional[Any]							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															self.assertEqual(len(lowercase__  ) ,       1  )

															a__       :						Dict							=  tokenizer.decode(ids + encoded_special_token ,       clean_up_tokenization_spaces=lowercase__  )
															a__       :						Tuple							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )

															a__       :						Any							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															a__       :						Optional[Any]							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															self.assertEqual(lowercase__ ,       input_encoded + special_token_id  )

															a__       :						Optional[Any]							=  tokenizer.decode(lowercase__ ,       skip_special_tokens=lowercase__  )
															self.assertTrue(special_token not in decoded  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									a__       :						Optional[int]							=  self.get_tokenizers(do_lower_case=lowercase__  )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															a__       :						Optional[Any]							=  chr(0xe005  )
															a__       :						int							=  chr(0xe006  )

															# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
															tokenizer.add_tokens([SPECIAL_TOKEN_1] ,       special_tokens=lowercase__  )
															# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
															# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
															tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}  )

															a__       :						List[str]							=  tokenizer.tokenize(lowercase__  )
															a__       :						List[str]							=  tokenizer.tokenize(lowercase__  )

															self.assertEqual(len(lowercase__  ) ,       1  )
															self.assertEqual(len(lowercase__  ) ,       1  )
															self.assertEqual(token_a[0] ,       lowercase__  )
															self.assertEqual(token_a[0] ,       lowercase__  )

						@require_tokenizers
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									a__       :						Optional[Any]							=  self.get_tokenizers(do_lower_case=lowercase__  )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															# a special token for Canine can be defined as follows:
															a__       :						Any							=  0xe006
															a__       :						Tuple							=  chr(lowercase__  )

															a__       :						Dict							=  AddedToken(lowercase__ ,       lstrip=lowercase__  )
															tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}  )

															with tempfile.TemporaryDirectory() as tmp_dir_name:
																		tokenizer.save_pretrained(lowercase__  )
																		tokenizer.from_pretrained(lowercase__  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									a__       :						Any							=  []
									if self.test_slow_tokenizer:
												tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

									if self.test_rust_tokenizer:
												tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
												with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer_utils.save_pretrained(lowercase__  )

															with open(os.path.join(lowercase__ ,       """special_tokens_map.json"""  ) ,       encoding="""utf-8"""  ) as json_file:
																		a__       :						str							=  json.load(lowercase__  )

															with open(os.path.join(lowercase__ ,       """tokenizer_config.json"""  ) ,       encoding="""utf-8"""  ) as json_file:
																		a__       :						List[Any]							=  json.load(lowercase__  )

															# a special token for Canine can be defined as follows:
															a__       :						Dict							=  0xe006
															a__       :						Optional[int]							=  chr(lowercase__  )

															a__       :						int							=  [new_token_a]
															a__       :						List[Any]							=  [new_token_a]

															with open(os.path.join(lowercase__ ,       """special_tokens_map.json"""  ) ,       """w""" ,       encoding="""utf-8"""  ) as outfile:
																		json.dump(lowercase__ ,       lowercase__  )
															with open(os.path.join(lowercase__ ,       """tokenizer_config.json"""  ) ,       """w""" ,       encoding="""utf-8"""  ) as outfile:
																		json.dump(lowercase__ ,       lowercase__  )

															# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
															# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
															# "special_tokens_map.json" files
															a__       :						Any							=  tokenizer_class.from_pretrained(lowercase__ ,       extra_ids=0  )
															self.assertIn(lowercase__ ,       tokenizer_without_change_in_init.additional_special_tokens  )
															# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
															self.assertEqual(
															    [new_token_a] ,       tokenizer_without_change_in_init.convert_ids_to_tokens(
															        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]  )  ) ,       )

															a__       :						str							=  0xe007
															a__       :						Union[str, Any]							=  chr(lowercase__  )
															# Now we test that we can change the value of additional_special_tokens in the from_pretrained
															a__       :						Optional[Any]							=  [AddedToken(lowercase__ ,       lstrip=lowercase__  )]
															a__       :						List[Any]							=  tokenizer_class.from_pretrained(
															    lowercase__ ,       additional_special_tokens=lowercase__ ,       extra_ids=0  )

															self.assertIn(lowercase__ ,       tokenizer.additional_special_tokens  )
															# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
															self.assertEqual(
															    [new_token_a] ,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]  )  )  )

						@require_tokenizers
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									a__       :						List[str]							=  self.get_tokenizers(do_lower_case=lowercase__  )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															a__       :						str							=  """hello world"""
															if self.space_between_special_tokens:
																		a__       :						int							=  """[CLS] hello world [SEP]"""
															else:
																		a__       :						List[str]							=  input
															a__       :						Tuple							=  tokenizer.encode(lowercase__ ,       add_special_tokens=lowercase__  )
															a__       :						int							=  tokenizer.decode(lowercase__ ,       spaces_between_special_tokens=self.space_between_special_tokens  )
															self.assertIn(lowercase__ ,       [output, output.lower()]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									a__       :						Dict							=  self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
															a__       :						int							=  [
															    """bos_token""",
															    """eos_token""",
															    """unk_token""",
															    """sep_token""",
															    """pad_token""",
															    """cls_token""",
															    """mask_token""",
															]

									a__       :						int							=  """a"""
									a__       :						List[str]							=  ord(lowercase__  )

									for attr in attributes_list:
												setattr(lowercase__ ,       attr + """_id""" ,       lowercase__  )
												self.assertEqual(getattr(lowercase__ ,       lowercase__  ) ,       lowercase__  )
												self.assertEqual(getattr(lowercase__ ,       attr + """_id"""  ) ,       lowercase__  )

												setattr(lowercase__ ,       attr + """_id""" ,       lowercase__  )
												self.assertEqual(getattr(lowercase__ ,       lowercase__  ) ,       lowercase__  )
												self.assertEqual(getattr(lowercase__ ,       attr + """_id"""  ) ,       lowercase__  )

									setattr(lowercase__ ,       """additional_special_tokens_ids""" ,       []  )
									self.assertListEqual(getattr(lowercase__ ,       """additional_special_tokens"""  ) ,       []  )
									self.assertListEqual(getattr(lowercase__ ,       """additional_special_tokens_ids"""  ) ,       []  )

									a__       :						Union[str, Any]							=  0xe006
									a__       :						int							=  chr(lowercase__  )
									setattr(lowercase__ ,       """additional_special_tokens_ids""" ,       [additional_special_token_id]  )
									self.assertListEqual(getattr(lowercase__ ,       """additional_special_tokens"""  ) ,       [additional_special_token]  )
									self.assertListEqual(getattr(lowercase__ ,       """additional_special_tokens_ids"""  ) ,       [additional_special_token_id]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Tuple:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[str]:
									"""simple docstring"""




									pass

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Tuple:
									"""simple docstring"""




									pass









import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin

class        snake_case__    (A__ ,     unittest.TestCase		):







						"""simple docstring"""




						__lowerCAmelCase					:Union[str, Any]      =					ProphetNetTokenizer
						__lowerCAmelCase					:Any      =					False

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									super().setUp()

									a__       :						Optional[Any]							=  [
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									a__       :						List[str]							=  os.path.join(self.tmpdirname ,       VOCAB_FILES_NAMES["""vocab_file"""]  )
									with open(self.vocab_file ,       """w""" ,       encoding="""utf-8"""  ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				str:
									"""simple docstring"""




									a__       :						Any							=  """UNwant\u00E9d,running"""
									a__       :						Dict							=  """unwanted, running"""
									return input_text, output_text

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Dict:
									"""simple docstring"""




									a__       :						Tuple							=  self.tokenizer_class(self.vocab_file  )

									a__       :						int							=  tokenizer.tokenize("""UNwant\u00E9d,running"""  )
									self.assertListEqual(__lowercase ,       ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]  )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase  ) ,       [9, 6, 7, 1_2, 1_0, 1_1]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									a__       :						str							=  BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""  ) ,       ["""ah""", """\u535A""", """\u63A8""", """zz"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									a__       :						int							=  BasicTokenizer(do_lower_case=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  ) ,       ["""hello""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ) ,       ["""hello"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									a__       :						str							=  BasicTokenizer(do_lower_case=__lowercase ,       strip_accents=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ) ,       ["""hällo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ) ,       ["""h\u00E9llo"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									a__       :						List[str]							=  BasicTokenizer(do_lower_case=__lowercase ,       strip_accents=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ) ,       ["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ) ,       ["""hello"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									a__       :						Optional[Any]							=  BasicTokenizer(do_lower_case=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ) ,       ["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ) ,       ["""hello"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									a__       :						List[str]							=  BasicTokenizer(do_lower_case=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  ) ,       ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[str]:
									"""simple docstring"""




									a__       :						str							=  BasicTokenizer(do_lower_case=__lowercase ,       strip_accents=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ) ,       ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									a__       :						List[str]							=  BasicTokenizer(do_lower_case=__lowercase ,       strip_accents=__lowercase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ) ,       ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									a__       :						Union[str, Any]							=  BasicTokenizer(do_lower_case=__lowercase ,       never_split=["""[UNK]"""]  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""  ) ,       ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									a__       :						Optional[Any]							=  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									a__       :						Dict							=  {}
									for i, token in enumerate(__lowercase  ):
												a__       :						Optional[Any]							=  i
									a__       :						str							=  WordpieceTokenizer(vocab=__lowercase ,       unk_token="""[UNK]"""  )

									self.assertListEqual(tokenizer.tokenize(""""""  ) ,       []  )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""  ) ,       ["""un""", """##want""", """##ed""", """runn""", """##ing"""]  )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""  ) ,       ["""[UNK]""", """runn""", """##ing"""]  )

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									a__       :						List[Any]							=  self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""  )

									a__       :						List[Any]							=  ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
									a__       :						Optional[Any]							=  [1_0_3_7, 2_1_4_6, 2_0_4_2_3, 2_0_0_5, 7_6_8_0, 7_8_4_9, 3_9_8_9, 1_0_1_2, 1_0_2]
									a__       :						List[Any]							=  tokenizer(__lowercase ,       padding=__lowercase ,       return_tensors="""pt"""  )
									self.assertIsInstance(__lowercase ,       __lowercase  )
									a__       :						Optional[int]							=  list(batch.input_ids.numpy()[0]  )
									self.assertListEqual(__lowercase ,       __lowercase  )

									self.assertEqual((2, 9) ,       batch.input_ids.shape  )
									self.assertEqual((2, 9) ,       batch.attention_mask.shape  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[str]:
									"""simple docstring"""




									self.assertTrue(_is_whitespace(""" """  )  )
									self.assertTrue(_is_whitespace("""\t"""  )  )
									self.assertTrue(_is_whitespace("""\r"""  )  )
									self.assertTrue(_is_whitespace("""\n"""  )  )
									self.assertTrue(_is_whitespace("""\u00A0"""  )  )

									self.assertFalse(_is_whitespace("""A"""  )  )
									self.assertFalse(_is_whitespace("""-"""  )  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									self.assertTrue(_is_control("""\u0005"""  )  )

									self.assertFalse(_is_control("""A"""  )  )
									self.assertFalse(_is_control(""" """  )  )
									self.assertFalse(_is_control("""\t"""  )  )
									self.assertFalse(_is_control("""\r"""  )  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									self.assertTrue(_is_punctuation("""-"""  )  )
									self.assertTrue(_is_punctuation("""$"""  )  )
									self.assertTrue(_is_punctuation("""`"""  )  )
									self.assertTrue(_is_punctuation("""."""  )  )

									self.assertFalse(_is_punctuation("""A"""  )  )
									self.assertFalse(_is_punctuation(""" """  )  )

						@slow
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									a__       :						Optional[Any]							=  self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""  )

									a__       :						Dict							=  tokenizer.encode("""sequence builders""" ,       add_special_tokens=__lowercase  )
									a__       :						str							=  tokenizer.encode("""multi-sequence build""" ,       add_special_tokens=__lowercase  )

									a__       :						Optional[Any]							=  tokenizer.build_inputs_with_special_tokens(__lowercase  )
									a__       :						Optional[int]							=  tokenizer.build_inputs_with_special_tokens(__lowercase ,       __lowercase  )

									assert encoded_sentence == text + [1_0_2]
									assert encoded_pair == text + [1_0_2] + text_a + [1_0_2]




