




'''simple docstring'''



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 		lowercase_    ( a__     ,  unittest.TestCase   ):


   """simple docstring"""

   lowerCamelCase_				  =						RealmTokenizer
   lowerCamelCase_				  =						RealmTokenizerFast
   lowerCamelCase_				  =						True
   lowerCamelCase_				  =						True
   lowerCamelCase_				  =						filter_non_english



   def 							lowerCAmelCase_			(    self					:	str    ):

      """simple docstring"""
      super().setUp()

      _SCREAMING_SNAKE_CASE       =      [
          "[UNK]",
          "[CLS]",
          "[SEP]",
          "[PAD]",
          "[MASK]",
          "want",
          "##want",
          "##ed",
          "wa",
          "un",
          "runn",
          "##ing",
          ",",
          "low",
          "lowest",
      ]
      _SCREAMING_SNAKE_CASE       =      os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["vocab_file"]    )
      with open(self.vocab_file  ,      "w"  ,      encoding="utf-8"    ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )



   def 							lowerCAmelCase_			(    self					:	Optional[int]  ,      __lowerCamelCase					:	str    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      "UNwant\u00E9d,running"
      _SCREAMING_SNAKE_CASE       =      "unwanted, running"
      return input_text, output_text



   def 							lowerCAmelCase_			(    self					:	Any    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      self.tokenizer_class(self.vocab_file    )

      _SCREAMING_SNAKE_CASE       =      tokenizer.tokenize("UNwant\u00E9d,running"    )
      self.assertListEqual(_snake_case  ,      ["un", "##want", "##ed", ",", "runn", "##ing"]    )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case    )  ,      [9, 6, 7, 1_2, 1_0, 1_1]    )



   def 							lowerCAmelCase_			(    self					:	Any    ):

      """simple docstring"""






      if not self.test_rust_tokenizer:
         return

      _SCREAMING_SNAKE_CASE       =      self.get_tokenizer()
      _SCREAMING_SNAKE_CASE       =      self.get_rust_tokenizer()

      _SCREAMING_SNAKE_CASE       =      "UNwant\u00E9d,running"

      _SCREAMING_SNAKE_CASE       =      tokenizer.tokenize(_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.tokenize(_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )

      _SCREAMING_SNAKE_CASE       =      tokenizer.encode(_snake_case  ,      add_special_tokens=_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.encode(_snake_case  ,      add_special_tokens=_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )

      _SCREAMING_SNAKE_CASE       =      self.get_rust_tokenizer()
      _SCREAMING_SNAKE_CASE       =      tokenizer.encode(_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.encode(_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )

      # With lower casing
      _SCREAMING_SNAKE_CASE       =      self.get_tokenizer(do_lower_case=_snake_case    )
      _SCREAMING_SNAKE_CASE       =      self.get_rust_tokenizer(do_lower_case=_snake_case    )

      _SCREAMING_SNAKE_CASE       =      "UNwant\u00E9d,running"

      _SCREAMING_SNAKE_CASE       =      tokenizer.tokenize(_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.tokenize(_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )

      _SCREAMING_SNAKE_CASE       =      tokenizer.encode(_snake_case  ,      add_special_tokens=_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.encode(_snake_case  ,      add_special_tokens=_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )

      _SCREAMING_SNAKE_CASE       =      self.get_rust_tokenizer()
      _SCREAMING_SNAKE_CASE       =      tokenizer.encode(_snake_case    )
      _SCREAMING_SNAKE_CASE       =      rust_tokenizer.encode(_snake_case    )
      self.assertListEqual(_snake_case  ,      _snake_case    )



   def 							lowerCAmelCase_			(    self					:	List[str]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"    )  ,      ["ah", "\u535A", "\u63A8", "zz"]    )



   def 							lowerCAmelCase_			(    self					:	Tuple    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )  ,      ["hello", "!", "how", "are", "you", "?"]    )
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )  ,      ["hello"]    )



   def 							lowerCAmelCase_			(    self					:	Dict    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case  ,      strip_accents=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )  ,      ["hällo", "!", "how", "are", "you", "?"]    )
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )  ,      ["h\u00E9llo"]    )



   def 							lowerCAmelCase_			(    self					:	Any    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case  ,      strip_accents=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )  ,      ["hallo", "!", "how", "are", "you", "?"]    )
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )  ,      ["hello"]    )



   def 							lowerCAmelCase_			(    self					:	Optional[Any]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )  ,      ["hallo", "!", "how", "are", "you", "?"]    )
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )  ,      ["hello"]    )



   def 							lowerCAmelCase_			(    self					:	List[str]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )  ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]    )



   def 							lowerCAmelCase_			(    self					:	Union[str, Any]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case  ,      strip_accents=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )  ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]    )



   def 							lowerCAmelCase_			(    self					:	Union[str, Any]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case  ,      strip_accents=_snake_case    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )  ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]    )



   def 							lowerCAmelCase_			(    self					:	List[Any]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      BasicTokenizer(do_lower_case=_snake_case  ,      never_split=["[UNK]"]    )

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"    )  ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]    )



   def 							lowerCAmelCase_			(    self					:	Optional[int]    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

      _SCREAMING_SNAKE_CASE       =      {}
      for i, token in enumerate(_snake_case    ):
         _SCREAMING_SNAKE_CASE       =      i
      _SCREAMING_SNAKE_CASE       =      WordpieceTokenizer(vocab=_snake_case  ,      unk_token="[UNK]"    )

      self.assertListEqual(tokenizer.tokenize(""    )  ,      []    )

      self.assertListEqual(tokenizer.tokenize("unwanted running"    )  ,      ["un", "##want", "##ed", "runn", "##ing"]    )

      self.assertListEqual(tokenizer.tokenize("unwantedX running"    )  ,      ["[UNK]", "runn", "##ing"]    )



   def 							lowerCAmelCase_			(    self					:	int    ):

      """simple docstring"""
      self.assertTrue(_is_whitespace(" "    )    )
      self.assertTrue(_is_whitespace("\t"    )    )
      self.assertTrue(_is_whitespace("\r"    )    )
      self.assertTrue(_is_whitespace("\n"    )    )
      self.assertTrue(_is_whitespace("\u00A0"    )    )

      self.assertFalse(_is_whitespace("A"    )    )
      self.assertFalse(_is_whitespace("-"    )    )



   def 							lowerCAmelCase_			(    self					:	Tuple    ):

      """simple docstring"""
      self.assertTrue(_is_control("\u0005"    )    )

      self.assertFalse(_is_control("A"    )    )
      self.assertFalse(_is_control(" "    )    )
      self.assertFalse(_is_control("\t"    )    )
      self.assertFalse(_is_control("\r"    )    )



   def 							lowerCAmelCase_			(    self					:	Dict    ):

      """simple docstring"""
      self.assertTrue(_is_punctuation("-"    )    )
      self.assertTrue(_is_punctuation("$"    )    )
      self.assertTrue(_is_punctuation("`"    )    )
      self.assertTrue(_is_punctuation("."    )    )

      self.assertFalse(_is_punctuation("A"    )    )
      self.assertFalse(_is_punctuation(" "    )    )



   def 							lowerCAmelCase_			(    self					:	Any    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_snake_case    ) for t in ["Test", "\xad", "test"]]  ,      [["[UNK]"], [], ["[UNK]"]]    )

      if self.test_rust_tokenizer:
         _SCREAMING_SNAKE_CASE       =      self.get_rust_tokenizer()
         self.assertListEqual(
             [rust_tokenizer.tokenize(_snake_case    ) for t in ["Test", "\xad", "test"]]  ,      [["[UNK]"], [], ["[UNK]"]]    )



   @slow
   def 							lowerCAmelCase_			(    self					:	Tuple    ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE       =      self.tokenizer_class.from_pretrained("bert-base-uncased"    )

      _SCREAMING_SNAKE_CASE       =      tokenizer.encode("sequence builders"  ,      add_special_tokens=_snake_case    )
      _SCREAMING_SNAKE_CASE       =      tokenizer.encode("multi-sequence build"  ,      add_special_tokens=_snake_case    )

      _SCREAMING_SNAKE_CASE       =      tokenizer.build_inputs_with_special_tokens(_snake_case    )
      _SCREAMING_SNAKE_CASE       =      tokenizer.build_inputs_with_special_tokens(_snake_case  ,      _snake_case    )

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



   def 							lowerCAmelCase_			(    self					:	Dict    ):

      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
            _SCREAMING_SNAKE_CASE       =      self.rust_tokenizer_class.from_pretrained(_snake_case  ,      **_snake_case    )

            _SCREAMING_SNAKE_CASE       =      F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            _SCREAMING_SNAKE_CASE       =      tokenizer_r.encode_plus(
                _snake_case  ,      return_attention_mask=_snake_case  ,      return_token_type_ids=_snake_case  ,      return_offsets_mapping=_snake_case  ,      add_special_tokens=_snake_case  ,      )

            _SCREAMING_SNAKE_CASE       =      tokenizer_r.do_lower_case if hasattr(_snake_case  ,      "do_lower_case"    ) else False
            _SCREAMING_SNAKE_CASE       =      (
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "A"),
                    ((1, 2), ","),
                    ((3, 5), "na"),
                    ((5, 6), "##ï"),
                    ((6, 8), "##ve"),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), "Allen"),
                    ((2_1, 2_3), "##NL"),
                    ((2_3, 2_4), "##P"),
                    ((2_5, 3_3), "sentence"),
                    ((3_3, 3_4), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "a"),
                    ((1, 2), ","),
                    ((3, 8), "naive"),
                    ((9, 1_5), tokenizer_r.mask_token),
                    ((1_6, 2_1), "allen"),
                    ((2_1, 2_3), "##nl"),
                    ((2_3, 2_4), "##p"),
                    ((2_5, 3_3), "sentence"),
                    ((3_3, 3_4), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]  ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]    )    )
            self.assertEqual([e[0] for e in expected_results]  ,      tokens["offset_mapping"]    )



   @slow
   def 							lowerCAmelCase_			(    self					:	List[str]    ):

      """simple docstring"""






      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
            _SCREAMING_SNAKE_CASE       =      self.rust_tokenizer_class.from_pretrained(_snake_case  ,      **_snake_case    )
            _SCREAMING_SNAKE_CASE       =      self.tokenizer_class.from_pretrained(_snake_case  ,      **_snake_case    )
            _SCREAMING_SNAKE_CASE       =      [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

            _SCREAMING_SNAKE_CASE       =      tokenizer_r.batch_encode_candidates(_snake_case  ,      max_length=1_0  ,      return_tensors="np"    )
            _SCREAMING_SNAKE_CASE       =      tokenizer_p.batch_encode_candidates(_snake_case  ,      max_length=1_0  ,      return_tensors="np"    )

            _SCREAMING_SNAKE_CASE       =      (2, 2, 1_0)

            self.assertEqual(encoded_sentence_r["input_ids"].shape  ,      _snake_case    )
            self.assertEqual(encoded_sentence_r["attention_mask"].shape  ,      _snake_case    )
            self.assertEqual(encoded_sentence_r["token_type_ids"].shape  ,      _snake_case    )

            self.assertEqual(encoded_sentence_p["input_ids"].shape  ,      _snake_case    )
            self.assertEqual(encoded_sentence_p["attention_mask"].shape  ,      _snake_case    )
            self.assertEqual(encoded_sentence_p["token_type_ids"].shape  ,      _snake_case    )








"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__UpperCamelCase			:		List[Any]		    =   {'''configuration_yoso''': ['''YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''YosoConfig''']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase			:		Dict		    =   [
           '''YOSO_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''YosoForMaskedLM''',
           '''YosoForMultipleChoice''',
           '''YosoForQuestionAnswering''',
           '''YosoForSequenceClassification''',
           '''YosoForTokenClassification''',
           '''YosoLayer''',
           '''YosoModel''',
           '''YosoPreTrainedModel''',
       ]


if TYPE_CHECKING:
       from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_yoso import (
                  YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
                  YosoForMaskedLM,
                  YosoForMultipleChoice,
                  YosoForQuestionAnswering,
                  YosoForSequenceClassification,
                  YosoForTokenClassification,
                  YosoLayer,
                  YosoModel,
                  YosoPreTrainedModel,
              )


else:
       import sys

       __UpperCamelCase			:		Any		    =   _LazyModule(__name__, globals()['''__file__'''], _import_structure)





