






"""simple docstring"""







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class     __UpperCAmelCase(   SCREAMING_SNAKE_CASE__  ,  unittest.TestCase							):




						"""simple docstring"""




						__lowerCamelCase				       =					RealmTokenizer
						__lowerCamelCase				       =					RealmTokenizerFast
						__lowerCamelCase				       =					True
						__lowerCamelCase				       =					True
						__lowerCamelCase				       =					filter_non_english




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												super().setUp()

												lowercase__				:      int=       [
												    "[UNK]",
												    "[CLS]",
												    "[SEP]",
												    "[PAD]",
												    "[MASK]",
												    "want",
												    "##want",
												    "##ed",
												    "wa",
												    "un",
												    "runn",
												    "##ing",
												    ",",
												    "low",
												    "lowest",
												]
												lowercase__				:      int=       os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["vocab_file"]			)
												with open(self.vocab_file       ,  "w"       ,  encoding="utf-8"			) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												lowercase__				:      Tuple=       "UNwant\u00E9d,running"
												lowercase__				:      Tuple=       "unwanted, running"
												return input_text, output_text




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       self.tokenizer_class(self.vocab_file			)

												lowercase__				:      Optional[Any]=       tokenizer.tokenize("UNwant\u00E9d,running"			)
												self.assertListEqual(snake_case__       ,  ["un", "##want", "##ed", ",", "runn", "##ing"]			)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__			)       ,  [9, 6, 7, 12, 10, 11]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												if not self.test_rust_tokenizer:
																		return

												lowercase__				:      List[str]=       self.get_tokenizer()
												lowercase__				:      Dict=       self.get_rust_tokenizer()

												lowercase__				:      int=       "UNwant\u00E9d,running"

												lowercase__				:      Dict=       tokenizer.tokenize(snake_case__			)
												lowercase__				:      Optional[Any]=       rust_tokenizer.tokenize(snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)

												lowercase__				:      str=       tokenizer.encode(snake_case__       ,  add_special_tokens=snake_case__			)
												lowercase__				:      Dict=       rust_tokenizer.encode(snake_case__       ,  add_special_tokens=snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)

												lowercase__				:      Dict=       self.get_rust_tokenizer()
												lowercase__				:      List[Any]=       tokenizer.encode(snake_case__			)
												lowercase__				:      Dict=       rust_tokenizer.encode(snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)

												# With lower casing
												lowercase__				:      Union[str, Any]=       self.get_tokenizer(do_lower_case=snake_case__			)
												lowercase__				:      Dict=       self.get_rust_tokenizer(do_lower_case=snake_case__			)

												lowercase__				:      Optional[Any]=       "UNwant\u00E9d,running"

												lowercase__				:      Optional[int]=       tokenizer.tokenize(snake_case__			)
												lowercase__				:      Any=       rust_tokenizer.tokenize(snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)

												lowercase__				:      Optional[int]=       tokenizer.encode(snake_case__       ,  add_special_tokens=snake_case__			)
												lowercase__				:      Tuple=       rust_tokenizer.encode(snake_case__       ,  add_special_tokens=snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)

												lowercase__				:      Optional[Any]=       self.get_rust_tokenizer()
												lowercase__				:      str=       tokenizer.encode(snake_case__			)
												lowercase__				:      Union[str, Any]=       rust_tokenizer.encode(snake_case__			)
												self.assertListEqual(snake_case__       ,  snake_case__			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Dict=       BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)       ,  ["ah", "\u535A", "\u63A8", "zz"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      int=       BasicTokenizer(do_lower_case=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,  ["hello", "!", "how", "are", "you", "?"]			)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,  ["hello"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       BasicTokenizer(do_lower_case=snake_case__       ,  strip_accents=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,  ["hällo", "!", "how", "are", "you", "?"]			)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,  ["h\u00E9llo"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       BasicTokenizer(do_lower_case=snake_case__       ,  strip_accents=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,  ["hallo", "!", "how", "are", "you", "?"]			)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,  ["hello"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       BasicTokenizer(do_lower_case=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,  ["hallo", "!", "how", "are", "you", "?"]			)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)       ,  ["hello"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      List[Any]=       BasicTokenizer(do_lower_case=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)       ,  ["HeLLo", "!", "how", "Are", "yoU", "?"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       BasicTokenizer(do_lower_case=snake_case__       ,  strip_accents=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,  ["HäLLo", "!", "how", "Are", "yoU", "?"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Any=       BasicTokenizer(do_lower_case=snake_case__       ,  strip_accents=snake_case__			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)       ,  ["HaLLo", "!", "how", "Are", "yoU", "?"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[Any]=       BasicTokenizer(do_lower_case=snake_case__       ,  never_split=["[UNK]"]			)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)       ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

												lowercase__				:      Optional[int]=       {}
												for i, token in enumerate(snake_case__			):
																		lowercase__				:      Union[str, Any]=       i
												lowercase__				:      Optional[Any]=       WordpieceTokenizer(vocab=snake_case__       ,  unk_token="[UNK]"			)

												self.assertListEqual(tokenizer.tokenize(""			)       ,  []			)

												self.assertListEqual(tokenizer.tokenize("unwanted running"			)       ,  ["un", "##want", "##ed", "runn", "##ing"]			)

												self.assertListEqual(tokenizer.tokenize("unwantedX running"			)       ,  ["[UNK]", "runn", "##ing"]			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												self.assertTrue(_is_whitespace(" "			)			)
												self.assertTrue(_is_whitespace("\t"			)			)
												self.assertTrue(_is_whitespace("\r"			)			)
												self.assertTrue(_is_whitespace("\n"			)			)
												self.assertTrue(_is_whitespace("\u00A0"			)			)

												self.assertFalse(_is_whitespace("A"			)			)
												self.assertFalse(_is_whitespace("-"			)			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												self.assertTrue(_is_control("\u0005"			)			)

												self.assertFalse(_is_control("A"			)			)
												self.assertFalse(_is_control(" "			)			)
												self.assertFalse(_is_control("\t"			)			)
												self.assertFalse(_is_control("\r"			)			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												self.assertTrue(_is_punctuation("-"			)			)
												self.assertTrue(_is_punctuation("$"			)			)
												self.assertTrue(_is_punctuation("`"			)			)
												self.assertTrue(_is_punctuation("."			)			)

												self.assertFalse(_is_punctuation("A"			)			)
												self.assertFalse(_is_punctuation(" "			)			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       self.get_tokenizer()

												# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
												self.assertListEqual([tokenizer.tokenize(snake_case__			) for t in ["Test", "\xad", "test"]]       ,  [["[UNK]"], [], ["[UNK]"]]			)

												if self.test_rust_tokenizer:
																		lowercase__				:      Union[str, Any]=       self.get_rust_tokenizer()
																		self.assertListEqual(
																		    [rust_tokenizer.tokenize(snake_case__			) for t in ["Test", "\xad", "test"]]       ,  [["[UNK]"], [], ["[UNK]"]]			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Dict=       self.tokenizer_class.from_pretrained("bert-base-uncased"			)

												lowercase__				:      int=       tokenizer.encode("sequence builders"       ,  add_special_tokens=snake_case__			)
												lowercase__				:      Optional[int]=       tokenizer.encode("multi-sequence build"       ,  add_special_tokens=snake_case__			)

												lowercase__				:      Tuple=       tokenizer.build_inputs_with_special_tokens(snake_case__			)
												lowercase__				:      List[str]=       tokenizer.build_inputs_with_special_tokens(snake_case__       ,  snake_case__			)

												assert encoded_sentence == [101] + text + [102]
												assert encoded_pair == [101] + text + [102] + text_a + [102]




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
																								lowercase__				:      Union[str, Any]=       self.rust_tokenizer_class.from_pretrained(snake_case__       ,  **snake_case__			)

																								lowercase__				:      Optional[int]=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																								lowercase__				:      Union[str, Any]=       tokenizer_r.encode_plus(
																								    snake_case__       ,  return_attention_mask=snake_case__       ,  return_token_type_ids=snake_case__       ,  return_offsets_mapping=snake_case__       ,  add_special_tokens=snake_case__       ,  )

																								lowercase__				:      Tuple=       tokenizer_r.do_lower_case if hasattr(snake_case__       ,  "do_lower_case"			) else False
																								lowercase__				:      Union[str, Any]=       (
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "A"),
																								        ((1, 2), ","),
																								        ((3, 5), "na"),
																								        ((5, 6), "##ï"),
																								        ((6, 8), "##ve"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "Allen"),
																								        ((21, 23), "##NL"),
																								        ((23, 24), "##P"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "a"),
																								        ((1, 2), ","),
																								        ((3, 8), "naive"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "allen"),
																								        ((21, 23), "##nl"),
																								        ((23, 24), "##p"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]       ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
																								self.assertEqual([e[0] for e in expected_results]       ,  tokens["offset_mapping"]			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
																								lowercase__				:      Tuple=       self.rust_tokenizer_class.from_pretrained(snake_case__       ,  **snake_case__			)
																								lowercase__				:      Tuple=       self.tokenizer_class.from_pretrained(snake_case__       ,  **snake_case__			)
																								lowercase__				:      List[Any]=       [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

																								lowercase__				:      Optional[int]=       tokenizer_r.batch_encode_candidates(snake_case__       ,  max_length=10       ,  return_tensors="np"			)
																								lowercase__				:      str=       tokenizer_p.batch_encode_candidates(snake_case__       ,  max_length=10       ,  return_tensors="np"			)

																								lowercase__				:      Dict=       (2, 2, 10)

																								self.assertEqual(encoded_sentence_r["input_ids"].shape       ,  snake_case__			)
																								self.assertEqual(encoded_sentence_r["attention_mask"].shape       ,  snake_case__			)
																								self.assertEqual(encoded_sentence_r["token_type_ids"].shape       ,  snake_case__			)

																								self.assertEqual(encoded_sentence_p["input_ids"].shape       ,  snake_case__			)
																								self.assertEqual(encoded_sentence_p["attention_mask"].shape       ,  snake_case__			)
																								self.assertEqual(encoded_sentence_p["token_type_ids"].shape       ,  snake_case__			)














"""simple docstring"""







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
			import torch

			from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer


@require_torch
class     __UpperCAmelCase(   unittest.TestCase							):




						"""simple docstring"""




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       XLMProphetNetForConditionalGeneration.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"			)
												model.to(snake_case__			)

												# encoder-decoder outputs
												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      Optional[Any]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      Dict=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      int=       output[0]
												lowercase__				:      str=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-6.39_86, -8.23_91, 12.51_89], [-6.32_89, -8.08_64, 12.62_11], [-6.24_18, -8.04_45, 12.79_68]]]			).to(snake_case__			)
												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# encoder outputs
												lowercase__				:      Tuple=       model.prophetnet.encoder(snake_case__			)[0]
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-1.42_60, -0.76_28, 0.84_53], [-1.47_19, -0.13_91, 0.78_07], [-1.76_78, 0.01_14, 0.46_46]]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       torch.Size((1, 4, 1024)			)
												self.assertEqual(encoder_outputs.shape       ,  snake_case__			)
												self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# decoder outputs
												lowercase__				:      int=       model.prophetnet.decoder(
												    snake_case__       ,  encoder_hidden_states=snake_case__       ,  )
												lowercase__				:      Union[str, Any]=       decoder_outputs[1].view(1       ,  model.config.ngram       ,  14       ,  -1			)
												lowercase__				:      List[Any]=       model.lm_head(snake_case__			)
												lowercase__				:      str=       predicting_streams_logits[:, 0]
												self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Union[str, Any]=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)

												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      List[str]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      str=       output[0]
												lowercase__				:      Optional[Any]=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												# compare the actual values for a slice.
												lowercase__				:      str=       torch.tensor(
												    [[[-9.22_53, -9.71_73, -6.35_29], [-7.67_01, -9.01_45, -1.93_82], [-8.01_95, -7.00_04, -0.15_23]]]			).to(snake_case__			)

												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)
												lowercase__				:      Optional[int]=       512

												lowercase__				:      int=       XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)

												lowercase__				:      str=       (
												    "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"
												    " January 14, 2020, according to the official portal of the organization. From that day, users of this"
												    " system will not be able to receive security updates, which could make their computers vulnerable to"
												    " cyber attacks."
												)
												lowercase__				:      Tuple=       (
												    "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"
												    " после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"
												    " этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"
												    " к кибератакам."
												)
												lowercase__				:      Optional[int]=       (
												    "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
												)

												lowercase__				:      Dict=       tokenizer(
												    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]       ,  padding=snake_case__       ,  max_length=255       ,  return_tensors="pt"			).input_ids
												lowercase__				:      Any=       input_ids.to(snake_case__			)

												lowercase__				:      List[Any]=       model.generate(
												    snake_case__       ,  num_beams=10       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      str=       [tokenizer.decode(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids]
												lowercase__				:      Dict=       "Microsoft to end Windows 7 free support after January 14, 2020"
												lowercase__				:      int=       "Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"
												lowercase__				:      List[str]=       "微软打算终止对Windows 7操作系统的免费支持"
												self.assertListEqual(
												    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]       ,  snake_case__       ,  )

												lowercase__				:      List[str]=       model.generate(
												    snake_case__       ,  num_beams=1       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      Optional[Any]=       [
												    tokenizer.convert_ids_to_tokens(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids_beama
												]
												lowercase__				:      List[str]=       "▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7".split(" "			)
												lowercase__				:      List[Any]=       "▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года".split(
												    " "			)
												lowercase__				:      int=       "微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持".split(" "			)
												self.assertListEqual(
												    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]       ,  snake_case__       ,  )






