




import os
import re
import shutil
import sys
import tempfile
import unittest

import black


a__		     =     os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
a__		     =     '''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''


class    SCREAMING_SNAKE_CASE_				(       unittest.TestCase   ):



	"""simple docstring"""


	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      tempfile.mkdtemp()
						os.makedirs(os.path.join(self.transformer_dir					,     """models/bert/"""	)	)
						__UpperCamelCase					:	Optional[int]             =      self.transformer_dir
						shutil.copy(
						    os.path.join(lowerCAmelCase					,     """src/transformers/models/bert/modeling_bert.py"""	)					,     os.path.join(self.transformer_dir					,     """models/bert/modeling_bert.py"""	)					,     )






	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	List[Any]             =      """src/transformers"""
						shutil.rmtree(self.transformer_dir	)






	def 					lowerCamelCase__     (					self		:						Tuple					,     lowerCAmelCase		:						Tuple					,     lowerCAmelCase		:						Tuple					,     lowerCAmelCase		:						int					,     lowerCAmelCase		:						str=None	)	->       Any:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      comment + F'''\nclass {class_name}(nn.Module):\n''' + class_code
						if overwrite_result is not None:
											__UpperCamelCase					:	List[str]             =      comment + F'''\nclass {class_name}(nn.Module):\n''' + overwrite_result
						__UpperCamelCase					:	int             =      black.Mode(target_versions={black.TargetVersion.PYaa}					,     line_length=119	)
						__UpperCamelCase					:	List[Any]             =      black.format_str(lowerCAmelCase					,     mode=lowerCAmelCase	)
						__UpperCamelCase					:	int             =      os.path.join(self.transformer_dir					,     """new_code.py"""	)
						with open(lowerCAmelCase					,     """w"""					,     newline="""\n"""	) as f:
											f.write(lowerCAmelCase	)
						if overwrite_result is None:
											self.assertTrue(len(check_copies.is_copy_consistent(lowerCAmelCase	)	) == 0	)
						else:
											check_copies.is_copy_consistent(f.name					,     overwrite=lowerCAmelCase	)
											with open(lowerCAmelCase					,     """r"""	) as f:
																self.assertTrue(f.read()					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						int	)	->       Union[str, Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      check_copies.find_code_in_transformers("""models.bert.modeling_bert.BertLMPredictionHead"""	)
						self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						str	)	->       Union[str, Any]:







						"""simple docstring"""







						self.check_copy_consistency(
						    """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""					,     """BertLMPredictionHead"""					,     REFERENCE_CODE + """\n"""					,     )

						# With no empty line at the end
						self.check_copy_consistency(
						    """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""					,     """BertLMPredictionHead"""					,     lowerCAmelCase					,     )

						# Copy consistency with rename
						self.check_copy_consistency(
						    """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""					,     """TestModelLMPredictionHead"""					,     re.sub("""Bert"""					,     """TestModel"""					,     lowerCAmelCase	)					,     )

						# Copy consistency with a really long name
						__UpperCamelCase					:	List[Any]             =      """TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
						self.check_copy_consistency(
						    F'''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'''					,     F'''{long_class_name}LMPredictionHead'''					,     re.sub("""Bert"""					,     lowerCAmelCase					,     lowerCAmelCase	)					,     )

						# Copy consistency with overwrite
						self.check_copy_consistency(
						    """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""					,     """TestModelLMPredictionHead"""					,     lowerCAmelCase					,     overwrite_result=re.sub("""Bert"""					,     """TestModel"""					,     lowerCAmelCase	)					,     )






	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

						__UpperCamelCase					:	int             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
						    """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
						    """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
						    """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
						    """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
						    """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
						    """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
						    """ method has been applied to compress GPT2 into"""
						    """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
						    """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
						    """ Multilingual BERT into"""
						    """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
						    """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
						    """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
						    """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
						    """ Luong, Quoc V. Le, Christopher D. Manning."""
						)
						__UpperCamelCase					:	Optional[Any]             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
						    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
						    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
						    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
						)
						__UpperCamelCase					:	str             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
						    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
						    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
						    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
						    """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
						    """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
						    """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
						    """ method has been applied to compress GPT2 into"""
						    """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
						    """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
						    """ Multilingual BERT into"""
						    """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
						    """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
						    """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
						    """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
						    """ Christopher D. Manning 发布。\n"""
						)

						__UpperCamelCase      ,			__UpperCamelCase					:	str             =      check_copies.convert_to_localized_md(
						    lowerCAmelCase					,     lowerCAmelCase					,     localized_readme["""format_model_list"""]	)

						self.assertFalse(lowerCAmelCase	)
						self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)

						__UpperCamelCase      ,			__UpperCamelCase					:	Optional[Any]             =      check_copies.convert_to_localized_md(
						    lowerCAmelCase					,     lowerCAmelCase					,     localized_readme["""format_model_list"""]	)

						# Check whether the number of models is equal to README.md after conversion.
						self.assertTrue(lowerCAmelCase	)

						__UpperCamelCase					:	List[Any]             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
						    """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
						    """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
						    """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
						)
						__UpperCamelCase					:	int             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
						    """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
						    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
						    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
						)
						__UpperCamelCase					:	Any             =      (
						    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
						    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
						    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
						    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
						)

						__UpperCamelCase      ,			__UpperCamelCase					:	Any             =      check_copies.convert_to_localized_md(
						    lowerCAmelCase					,     lowerCAmelCase					,     localized_readme["""format_model_list"""]	)

						# Check if the model link is synchronized.
						self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)












import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase	,  unittest.TestCase   ):



	"""simple docstring"""


	__magic_name__   :       str      	=		ProphetNetTokenizer
	__magic_name__   :       Dict      	=		False






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       List[str]:







						"""simple docstring"""







						super().setUp()

						__UpperCamelCase					:	Optional[Any]             =      [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """want""",
						    """##want""",
						    """##ed""",
						    """wa""",
						    """un""",
						    """runn""",
						    """##ing""",
						    """,""",
						    """low""",
						    """lowest""",
						]
						__UpperCamelCase					:	List[Any]             =      os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["""vocab_file"""]	)
						with open(self.vocab_file					,     """w"""					,     encoding="""utf-8"""	) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)






	def 					lowerCamelCase__     (					self		:						Tuple					,     lowerCAmelCase		:						int	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      """UNwant\u00E9d,running"""
						__UpperCamelCase					:	List[str]             =      """unwanted, running"""
						return input_text, output_text






	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       int:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[int]             =      self.tokenizer_class(self.vocab_file	)

						__UpperCamelCase					:	List[str]             =      tokenizer.tokenize("""UNwant\u00E9d,running"""	)
						self.assertListEqual(lowerCAmelCase					,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase	)					,     [9, 6, 7, 12, 10, 11]	)






	def 					lowerCamelCase__     (					self		:						str	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	)					,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]	)






	def 					lowerCamelCase__     (					self		:						int	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      BasicTokenizer(do_lower_case=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)					,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)					,     ["""hello"""]	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	List[Any]             =      BasicTokenizer(do_lower_case=lowerCAmelCase					,     strip_accents=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)					,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)					,     ["""h\u00E9llo"""]	)






	def 					lowerCamelCase__     (					self		:						Any	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Tuple             =      BasicTokenizer(do_lower_case=lowerCAmelCase					,     strip_accents=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)					,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)					,     ["""hello"""]	)






	def 					lowerCamelCase__     (					self		:						List[str]	)	->       Optional[int]:







						"""simple docstring"""







						__UpperCamelCase					:	Dict             =      BasicTokenizer(do_lower_case=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)					,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)					,     ["""hello"""]	)






	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      BasicTokenizer(do_lower_case=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)					,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	Tuple             =      BasicTokenizer(do_lower_case=lowerCAmelCase					,     strip_accents=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)					,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






	def 					lowerCamelCase__     (					self		:						Optional[int]	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	List[Any]             =      BasicTokenizer(do_lower_case=lowerCAmelCase					,     strip_accents=lowerCAmelCase	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)					,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






	def 					lowerCamelCase__     (					self		:						Optional[Any]	)	->       str:







						"""simple docstring"""







						__UpperCamelCase					:	int             =      BasicTokenizer(do_lower_case=lowerCAmelCase					,     never_split=["""[UNK]"""]	)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	)					,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	int             =      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						__UpperCamelCase					:	List[Any]             =      {}
						for i, token in enumerate(lowerCAmelCase	):
											__UpperCamelCase					:	Dict             =      i
						__UpperCamelCase					:	Any             =      WordpieceTokenizer(vocab=lowerCAmelCase					,     unk_token="""[UNK]"""	)

						self.assertListEqual(tokenizer.tokenize(""""""	)					,     []	)

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""	)					,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	)					,     ["""[UNK]""", """runn""", """##ing"""]	)






	@require_torch
	def 					lowerCamelCase__     (					self		:						Optional[Any]	)	->       Dict:







						"""simple docstring"""







						__UpperCamelCase					:	str             =      self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""	)

						__UpperCamelCase					:	int             =      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
						__UpperCamelCase					:	Optional[int]             =      [1037, 2146, 20423, 2005, 7680, 7849, 3989, 1012, 102]
						__UpperCamelCase					:	Optional[int]             =      tokenizer(lowerCAmelCase					,     padding=lowerCAmelCase					,     return_tensors="""pt"""	)
						self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)
						__UpperCamelCase					:	int             =      list(batch.input_ids.numpy()[0]	)
						self.assertListEqual(lowerCAmelCase					,     lowerCAmelCase	)

						self.assertEqual((2, 9)					,     batch.input_ids.shape	)
						self.assertEqual((2, 9)					,     batch.attention_mask.shape	)






	def 					lowerCamelCase__     (					self		:						int	)	->       List[Any]:







						"""simple docstring"""







						self.assertTrue(_is_whitespace(""" """	)	)
						self.assertTrue(_is_whitespace("""\t"""	)	)
						self.assertTrue(_is_whitespace("""\r"""	)	)
						self.assertTrue(_is_whitespace("""\n"""	)	)
						self.assertTrue(_is_whitespace("""\u00A0"""	)	)

						self.assertFalse(_is_whitespace("""A"""	)	)
						self.assertFalse(_is_whitespace("""-"""	)	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Optional[int]:







						"""simple docstring"""







						self.assertTrue(_is_control("""\u0005"""	)	)

						self.assertFalse(_is_control("""A"""	)	)
						self.assertFalse(_is_control(""" """	)	)
						self.assertFalse(_is_control("""\t"""	)	)
						self.assertFalse(_is_control("""\r"""	)	)






	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Optional[Any]:







						"""simple docstring"""







						self.assertTrue(_is_punctuation("""-"""	)	)
						self.assertTrue(_is_punctuation("""$"""	)	)
						self.assertTrue(_is_punctuation("""`"""	)	)
						self.assertTrue(_is_punctuation("""."""	)	)

						self.assertFalse(_is_punctuation("""A"""	)	)
						self.assertFalse(_is_punctuation(""" """	)	)






	@slow
	def 					lowerCamelCase__     (					self		:						str	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""	)

						__UpperCamelCase					:	Optional[int]             =      tokenizer.encode("""sequence builders"""					,     add_special_tokens=lowerCAmelCase	)
						__UpperCamelCase					:	List[Any]             =      tokenizer.encode("""multi-sequence build"""					,     add_special_tokens=lowerCAmelCase	)

						__UpperCamelCase					:	Optional[Any]             =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase	)
						__UpperCamelCase					:	Union[str, Any]             =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase					,     lowerCAmelCase	)

						assert encoded_sentence == text + [102]
						assert encoded_pair == text + [102] + text_a + [102]






