




import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin
class 			_UpperCamelCase      (				lowerCAmelCase	,       unittest.TestCase		):
      UpperCAmelCase_        =		ProphetNetTokenizer
      UpperCAmelCase_        =		False
      def 	UpperCAmelCase_						(			self   :Union[str, Any]      )     ->			str:
       super().setUp()

       UpperCAmelCase__							 =						[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       UpperCAmelCase__							 =						os.path.join(self.tmpdirname ,      VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file ,      "w" ,      encoding="utf-8"      ) as vocab_writer:
        vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )
      def 	UpperCAmelCase_						(			self   :List[Any] ,      lowerCamelCase   :int      )     ->			Dict:
       UpperCAmelCase__							 =						"UNwant\u00E9d,running"
       UpperCAmelCase__							 =						"unwanted, running"
       return input_text, output_text
      def 	UpperCAmelCase_						(			self   :Tuple      )     ->			Optional[Any]:
       UpperCAmelCase__							 =						self.tokenizer_class(self.vocab_file      )

       UpperCAmelCase__							 =						tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(lowerCamelCase ,      ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase      ) ,      [9, 6, 7, 12, 10, 11]      )
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			Any:
       UpperCAmelCase__							 =						BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      ) ,      ["ah", "\u535A", "\u63A8", "zz"]      )
      def 	UpperCAmelCase_						(			self   :Dict      )     ->			Optional[Any]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      ) ,      ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			Dict:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["h\u00E9llo"]      )
      def 	UpperCAmelCase_						(			self   :Any      )     ->			int:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :Tuple      )     ->			Union[str, Any]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :Optional[int]      )     ->			Any:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      ) ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :Dict      )     ->			int:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			List[str]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :Tuple      )     ->			Optional[int]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      ) ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )
      def 	UpperCAmelCase_						(			self   :str      )     ->			Any:
       UpperCAmelCase__							 =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       UpperCAmelCase__							 =						{}
       for i, token in enumerate(lowerCamelCase      ):
        UpperCAmelCase__							 =						i
       UpperCAmelCase__							 =						WordpieceTokenizer(vocab=lowerCamelCase ,      unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      ) ,      []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      ) ,      ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      ) ,      ["[UNK]", "runn", "##ing"]      )
      @require_torch
      def 	UpperCAmelCase_						(			self   :Optional[int]      )     ->			str:
       UpperCAmelCase__							 =						self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"      )

       UpperCAmelCase__							 =						["A long paragraph for summarization.", "Another paragraph for summarization."]
       UpperCAmelCase__							 =						[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
       UpperCAmelCase__							 =						tokenizer(lowerCamelCase ,      padding=lowerCamelCase ,      return_tensors="pt"      )
       self.assertIsInstance(lowerCamelCase ,      lowerCamelCase      )
       UpperCAmelCase__							 =						list(batch.input_ids.numpy()[0]      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       self.assertEqual((2, 9) ,      batch.input_ids.shape      )
       self.assertEqual((2, 9) ,      batch.attention_mask.shape      )
      def 	UpperCAmelCase_						(			self   :int      )     ->			int:
       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			int:
       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )
      def 	UpperCAmelCase_						(			self   :int      )     ->			int:
       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )







      @slow
      def 	UpperCAmelCase_						(			self   :str      )     ->			str:
       UpperCAmelCase__							 =						self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"      )

       UpperCAmelCase__							 =						tokenizer.encode("sequence builders" ,      add_special_tokens=lowerCamelCase      )
       UpperCAmelCase__							 =						tokenizer.encode("multi-sequence build" ,      add_special_tokens=lowerCamelCase      )

       UpperCAmelCase__							 =						tokenizer.build_inputs_with_special_tokens(lowerCamelCase      )
       UpperCAmelCase__							 =						tokenizer.build_inputs_with_special_tokens(lowerCamelCase ,      lowerCamelCase      )

       assert encoded_sentence == text + [102]
       assert encoded_pair == text + [102] + text_a + [102]









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase						:   Union[str, Any]		=   {
    "configuration_bert": ["BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BertConfig", "BertOnnxConfig"],
    "tokenization_bert": ["BasicTokenizer", "BertTokenizer", "WordpieceTokenizer"],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCAmelCase						:   str		=   ["BertTokenizerFast"]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCAmelCase						:   int		=   [
      "BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "BertForMaskedLM",
      "BertForMultipleChoice",
      "BertForNextSentencePrediction",
      "BertForPreTraining",
      "BertForQuestionAnswering",
      "BertForSequenceClassification",
      "BertForTokenClassification",
      "BertLayer",
      "BertLMHeadModel",
      "BertModel",
      "BertPreTrainedModel",
      "load_tf_weights_in_bert",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCAmelCase						:   Dict		=   [
      "TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFBertEmbeddings",
      "TFBertForMaskedLM",
      "TFBertForMultipleChoice",
      "TFBertForNextSentencePrediction",
      "TFBertForPreTraining",
      "TFBertForQuestionAnswering",
      "TFBertForSequenceClassification",
      "TFBertForTokenClassification",
      "TFBertLMHeadModel",
      "TFBertMainLayer",
      "TFBertModel",
      "TFBertPreTrainedModel",
  ]
try:
  if not is_tensorflow_text_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCAmelCase						:   str		=   ["TFBertTokenizer"]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCAmelCase						:   Optional[Any]		=   [
      "FlaxBertForCausalLM",
      "FlaxBertForMaskedLM",
      "FlaxBertForMultipleChoice",
      "FlaxBertForNextSentencePrediction",
      "FlaxBertForPreTraining",
      "FlaxBertForQuestionAnswering",
      "FlaxBertForSequenceClassification",
      "FlaxBertForTokenClassification",
      "FlaxBertModel",
      "FlaxBertPreTrainedModel",
  ]

if TYPE_CHECKING:
  from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
  from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_bert_fast import BertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_bert import (
        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        BertForMaskedLM,
        BertForMultipleChoice,
        BertForNextSentencePrediction,
        BertForPreTraining,
        BertForQuestionAnswering,
        BertForSequenceClassification,
        BertForTokenClassification,
        BertLayer,
        BertLMHeadModel,
        BertModel,
        BertPreTrainedModel,
        load_tf_weights_in_bert,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_bert import (
        TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFBertEmbeddings,
        TFBertForMaskedLM,
        TFBertForMultipleChoice,
        TFBertForNextSentencePrediction,
        TFBertForPreTraining,
        TFBertForQuestionAnswering,
        TFBertForSequenceClassification,
        TFBertForTokenClassification,
        TFBertLMHeadModel,
        TFBertMainLayer,
        TFBertModel,
        TFBertPreTrainedModel,
    )

  try:
    if not is_tensorflow_text_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_bert_tf import TFBertTokenizer

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_bert import (
        FlaxBertForCausalLM,
        FlaxBertForMaskedLM,
        FlaxBertForMultipleChoice,
        FlaxBertForNextSentencePrediction,
        FlaxBertForPreTraining,
        FlaxBertForQuestionAnswering,
        FlaxBertForSequenceClassification,
        FlaxBertForTokenClassification,
        FlaxBertModel,
        FlaxBertPreTrainedModel,
    )

else:
  import sys

  _lowerCAmelCase						:   Union[str, Any]		=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





