


'''simple docstring'''






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin
class 						__UpperCamelCase	(						lowercase__						,  unittest.TestCase							):
							lowercase			:		str   		=  ProphetNetTokenizer
							lowercase			:		Union[str, Any]   		=  False





							def 		a__   (  self		:Any			):
									super().setUp()

									snake_case_		:							str			       =						[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									snake_case_		:							Tuple			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]			)
									with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""			) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)





							def 		a__   (  self		:str  ,_UpperCamelCase		:Dict			):
									snake_case_		:							Union[str, Any]			       =						"""UNwant\u00E9d,running"""
									snake_case_		:							Tuple			       =						"""unwanted, running"""
									return input_text, output_text





							def 		a__   (  self		:int			):
									snake_case_		:							Dict			       =						self.tokenizer_class(self.vocab_file			)

									snake_case_		:							Any			       =						tokenizer.tokenize("""UNwant\u00E9d,running"""			)
									self.assertListEqual(_UpperCamelCase  ,["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase			)  ,[9, 6, 7, 1_2, 1_0, 1_1]			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							str			       =						BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			)  ,["""ah""", """\u535A""", """\u63A8""", """zz"""]			)





							def 		a__   (  self		:str			):
									snake_case_		:							List[Any]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)  ,["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)  ,["""hello"""]			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							List[str]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase  ,strip_accents=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)  ,["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)  ,["""h\u00E9llo"""]			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Tuple			       =						BasicTokenizer(do_lower_case=_UpperCamelCase  ,strip_accents=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)  ,["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)  ,["""hello"""]			)





							def 		a__   (  self		:Tuple			):
									snake_case_		:							Optional[Any]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)  ,["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)  ,["""hello"""]			)





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							List[str]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)  ,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Dict			       =						BasicTokenizer(do_lower_case=_UpperCamelCase  ,strip_accents=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)  ,["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							Optional[Any]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase  ,strip_accents=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)  ,["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:Tuple			):
									snake_case_		:							List[Any]			       =						BasicTokenizer(do_lower_case=_UpperCamelCase  ,never_split=["""[UNK]"""]			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			)  ,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)





							def 		a__   (  self		:List[Any]			):
									snake_case_		:							Any			       =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									snake_case_		:							Optional[int]			       =						{}
									for i, token in enumerate(_UpperCamelCase			):
											snake_case_		:							str			       =						i
									snake_case_		:							str			       =						WordpieceTokenizer(vocab=_UpperCamelCase  ,unk_token="""[UNK]"""			)

									self.assertListEqual(tokenizer.tokenize(""""""			)  ,[]			)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""			)  ,["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			)  ,["""[UNK]""", """runn""", """##ing"""]			)





							@require_torch
							def 		a__   (  self		:str			):
									snake_case_		:							List[str]			       =						self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""			)

									snake_case_		:							List[str]			       =						["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
									snake_case_		:							Dict			       =						[1_0_3_7, 2_1_4_6, 2_0_4_2_3, 2_0_0_5, 7_6_8_0, 7_8_4_9, 3_9_8_9, 1_0_1_2, 1_0_2]
									snake_case_		:							Any			       =						tokenizer(_UpperCamelCase  ,padding=_UpperCamelCase  ,return_tensors="""pt"""			)
									self.assertIsInstance(_UpperCamelCase  ,_UpperCamelCase			)
									snake_case_		:							List[str]			       =						list(batch.input_ids.numpy()[0]			)
									self.assertListEqual(_UpperCamelCase  ,_UpperCamelCase			)

									self.assertEqual((2, 9)  ,batch.input_ids.shape			)
									self.assertEqual((2, 9)  ,batch.attention_mask.shape			)





							def 		a__   (  self		:Union[str, Any]			):
									self.assertTrue(_is_whitespace(""" """			)			)
									self.assertTrue(_is_whitespace("""\t"""			)			)
									self.assertTrue(_is_whitespace("""\r"""			)			)
									self.assertTrue(_is_whitespace("""\n"""			)			)
									self.assertTrue(_is_whitespace("""\u00A0"""			)			)

									self.assertFalse(_is_whitespace("""A"""			)			)
									self.assertFalse(_is_whitespace("""-"""			)			)





							def 		a__   (  self		:Tuple			):
									self.assertTrue(_is_control("""\u0005"""			)			)

									self.assertFalse(_is_control("""A"""			)			)
									self.assertFalse(_is_control(""" """			)			)
									self.assertFalse(_is_control("""\t"""			)			)
									self.assertFalse(_is_control("""\r"""			)			)





							def 		a__   (  self		:Optional[Any]			):
									self.assertTrue(_is_punctuation("""-"""			)			)
									self.assertTrue(_is_punctuation("""$"""			)			)
									self.assertTrue(_is_punctuation("""`"""			)			)
									self.assertTrue(_is_punctuation("""."""			)			)

									self.assertFalse(_is_punctuation("""A"""			)			)
									self.assertFalse(_is_punctuation(""" """			)			)





							@slow
							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							Dict			       =						self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""			)

									snake_case_		:							Tuple			       =						tokenizer.encode("""sequence builders"""  ,add_special_tokens=_UpperCamelCase			)
									snake_case_		:							int			       =						tokenizer.encode("""multi-sequence build"""  ,add_special_tokens=_UpperCamelCase			)

									snake_case_		:							List[str]			       =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase			)
									snake_case_		:							Dict			       =						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase  ,_UpperCamelCase			)

									assert encoded_sentence == text + [1_0_2]
									assert encoded_pair == text + [1_0_2] + text_a + [1_0_2]



'''simple docstring'''






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 						__UpperCamelCase	(						lowercase__						,  unittest.TestCase							):
							lowercase			:		Dict   		=  TransfoXLTokenizer
							lowercase			:		Optional[Any]   		=  False
							lowercase			:		Dict   		=  False





							def 		a__   (  self		:Union[str, Any]			):
									super().setUp()

									snake_case_		:							Optional[int]			       =						[
									    """<unk>""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """unwanted""",
									    """wa""",
									    """un""",
									    """running""",
									    """,""",
									    """low""",
									    """l""",
									]
									snake_case_		:							Optional[int]			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]			)
									with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""			) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)





							def 		a__   (  self		:List[Any]  ,**_UpperCamelCase		:Optional[Any]			):
									snake_case_		:							Tuple			       =						True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname  ,**_UpperCamelCase			)





							def 		a__   (  self		:Tuple  ,_UpperCamelCase		:Union[str, Any]			):
									snake_case_		:							Any			       =						"""<unk> UNwanted , running"""
									snake_case_		:							Optional[int]			       =						"""<unk> unwanted, running"""
									return input_text, output_text





							def 		a__   (  self		:Dict			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(vocab_file=self.vocab_file  ,lower_case=_UpperCamelCase			)

									snake_case_		:							Dict			       =						tokenizer.tokenize("""<unk> UNwanted , running"""			)
									self.assertListEqual(_UpperCamelCase  ,["""<unk>""", """unwanted""", """,""", """running"""]			)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase			)  ,[0, 4, 8, 7]			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""hello""", """!""", """how""", """are""", """you""", """?"""]			)





							def 		a__   (  self		:Any			):
									snake_case_		:							List[Any]			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							str			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)
									snake_case_		:							List[str]			       =						"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
									snake_case_		:							Optional[int]			       =						[
									    """Hello""",
									    """(""",
									    """bracket""",
									    """)""",
									    """and""",
									    """side""",
									    """@-@""",
									    """scrolled""",
									    """[""",
									    """and""",
									    """]""",
									    """Henry""",
									    """'s""",
									    """$""",
									    """5""",
									    """@,@""",
									    """000""",
									    """with""",
									    """3""",
									    """@.@""",
									    """34""",
									    """m""",
									    """.""",
									    """What""",
									    """'s""",
									    """up""",
									    """!""",
									    """?""",
									]

									self.assertListEqual(tokenizer.tokenize(_UpperCamelCase			)  ,_UpperCamelCase			)

									self.assertEqual(tokenizer.convert_tokens_to_string(_UpperCamelCase			)  ,_UpperCamelCase			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Union[str, Any]			       =						self.get_tokenizer()
									snake_case_		:							Dict			       =						len(_UpperCamelCase			)

									tokenizer.add_tokens(["""new1""", """new2"""]			)
									tokenizer.move_added_token("""new1"""  ,1			)

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(_UpperCamelCase			)  ,original_len + 2			)
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode("""new1"""			)  ,[1]			)
									self.assertEqual(tokenizer.decode([1]			)  ,"""new1"""			)