

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


a_     :    List[str]     		=   {
    """configuration_roberta_prelayernorm""": [
        """ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """RobertaPreLayerNormConfig""",
        """RobertaPreLayerNormOnnxConfig""",
    ],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      a_     :    Optional[int]     		=   [
          """ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """RobertaPreLayerNormForCausalLM""",
          """RobertaPreLayerNormForMaskedLM""",
          """RobertaPreLayerNormForMultipleChoice""",
          """RobertaPreLayerNormForQuestionAnswering""",
          """RobertaPreLayerNormForSequenceClassification""",
          """RobertaPreLayerNormForTokenClassification""",
          """RobertaPreLayerNormModel""",
          """RobertaPreLayerNormPreTrainedModel""",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      a_     :    List[str]     		=   [
          """TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """TFRobertaPreLayerNormForCausalLM""",
          """TFRobertaPreLayerNormForMaskedLM""",
          """TFRobertaPreLayerNormForMultipleChoice""",
          """TFRobertaPreLayerNormForQuestionAnswering""",
          """TFRobertaPreLayerNormForSequenceClassification""",
          """TFRobertaPreLayerNormForTokenClassification""",
          """TFRobertaPreLayerNormMainLayer""",
          """TFRobertaPreLayerNormModel""",
          """TFRobertaPreLayerNormPreTrainedModel""",
      ]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      a_     :    List[Any]     		=   [
          """FlaxRobertaPreLayerNormForCausalLM""",
          """FlaxRobertaPreLayerNormForMaskedLM""",
          """FlaxRobertaPreLayerNormForMultipleChoice""",
          """FlaxRobertaPreLayerNormForQuestionAnswering""",
          """FlaxRobertaPreLayerNormForSequenceClassification""",
          """FlaxRobertaPreLayerNormForTokenClassification""",
          """FlaxRobertaPreLayerNormModel""",
          """FlaxRobertaPreLayerNormPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_roberta_prelayernorm import (
          ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
          RobertaPreLayerNormConfig,
          RobertaPreLayerNormOnnxConfig,
      )

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_roberta_prelayernorm import (
                ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                RobertaPreLayerNormForCausalLM,
                RobertaPreLayerNormForMaskedLM,
                RobertaPreLayerNormForMultipleChoice,
                RobertaPreLayerNormForQuestionAnswering,
                RobertaPreLayerNormForSequenceClassification,
                RobertaPreLayerNormForTokenClassification,
                RobertaPreLayerNormModel,
                RobertaPreLayerNormPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_roberta_prelayernorm import (
                TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFRobertaPreLayerNormForCausalLM,
                TFRobertaPreLayerNormForMaskedLM,
                TFRobertaPreLayerNormForMultipleChoice,
                TFRobertaPreLayerNormForQuestionAnswering,
                TFRobertaPreLayerNormForSequenceClassification,
                TFRobertaPreLayerNormForTokenClassification,
                TFRobertaPreLayerNormMainLayer,
                TFRobertaPreLayerNormModel,
                TFRobertaPreLayerNormPreTrainedModel,
            )

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_roberta_prelayernorm import (
                FlaxRobertaPreLayerNormForCausalLM,
                FlaxRobertaPreLayerNormForMaskedLM,
                FlaxRobertaPreLayerNormForMultipleChoice,
                FlaxRobertaPreLayerNormForQuestionAnswering,
                FlaxRobertaPreLayerNormForSequenceClassification,
                FlaxRobertaPreLayerNormForTokenClassification,
                FlaxRobertaPreLayerNormModel,
                FlaxRobertaPreLayerNormPreTrainedModel,
            )

else:
      import sys

      a_     :    Union[str, Any]     		=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    _snake_case (						UpperCAmelCase_	,    unittest.TestCase				):
 __lowerCAmelCase	:  Union[str, Any]	   =      RoCBertTokenizer
 __lowerCAmelCase	:  Union[str, Any]	   =      None
 __lowerCAmelCase	:  str	   =      False
 __lowerCAmelCase	:  List[Any]	   =      True
 __lowerCAmelCase	:  Optional[int]	   =      filter_non_english


 def   lowercase__  (       self):


   '''simple docstring'''




   super().setUp()

   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
   lowercase__	: Dict     		=   {}
   lowercase__	: Tuple     		=   {}
   for i, value in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Tuple     		=   i
     lowercase__	: Any     		=   i
   lowercase__	: str     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__	: Union[str, Any]     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_shape_file"""])
   lowercase__	: Tuple     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_pronunciation_file"""])
   with open(self.vocab_file       ,	"""w"""       ,	encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))
   with open(self.word_shape_file       ,	"""w"""       ,	encoding="""utf-8""") as word_shape_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)
   with open(self.word_pronunciation_file       ,	"""w"""       ,	encoding="""utf-8""") as word_pronunciation_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[int]     		=   tokenizer.tokenize("""你好[SEP]你是谁""")
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""])
   self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: int     		=   RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")       ,	["""ah""", """\u535A""", """\u63A8""", """zz"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""hello""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hällo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""h\u00E9llo"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: str     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	never_split=["""[UNK]"""])

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   lowercase__	: Optional[int]     		=   {}
   for i, token in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Optional[Any]     		=   i
   lowercase__	: Union[str, Any]     		=   RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_       ,	unk_token="""[UNK]""")

   self.assertListEqual(tokenizer.tokenize("""""")       ,	[])

   self.assertListEqual(tokenizer.tokenize("""unwanted running""")       ,	["""un""", """##want""", """##ed""", """runn""", """##ing"""])

   self.assertListEqual(tokenizer.tokenize("""unwantedX running""")       ,	["""[UNK]""", """runn""", """##ing"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_whitespace(""" """))
   self.assertTrue(_is_whitespace("""\t"""))
   self.assertTrue(_is_whitespace("""\r"""))
   self.assertTrue(_is_whitespace("""\n"""))
   self.assertTrue(_is_whitespace("""\u00A0"""))

   self.assertFalse(_is_whitespace("""A"""))
   self.assertFalse(_is_whitespace("""-"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_control("""\u0005"""))

   self.assertFalse(_is_control("""A"""))
   self.assertFalse(_is_control(""" """))
   self.assertFalse(_is_control("""\t"""))
   self.assertFalse(_is_control("""\r"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_punctuation("""-"""))
   self.assertTrue(_is_punctuation("""$"""))
   self.assertTrue(_is_punctuation("""`"""))
   self.assertTrue(_is_punctuation("""."""))

   self.assertFalse(_is_punctuation("""A"""))
   self.assertFalse(_is_punctuation(""" """))


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])

   if self.test_rust_tokenizer:
     lowercase__	: int     		=   self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])


 def   lowercase__  (       self):


   '''simple docstring'''




   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: str     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
       lowercase__	: List[str]     		=   tokenizer_r.encode_plus(
           SCREAMING_SNAKE_CASE_       ,	return_attention_mask=SCREAMING_SNAKE_CASE_       ,	return_token_type_ids=SCREAMING_SNAKE_CASE_       ,	return_offsets_mapping=SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_       ,	)

       lowercase__	: str     		=   tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_       ,	"""do_lower_case""") else False
       lowercase__	: Optional[Any]     		=   (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """A"""),
               ((1, 2), ""","""),
               ((3, 5), """na"""),
               ((5, 6), """##ï"""),
               ((6, 8), """##ve"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """Allen"""),
               ((21, 23), """##NL"""),
               ((23, 24), """##P"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """a"""),
               ((1, 2), ""","""),
               ((3, 8), """naive"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """allen"""),
               ((21, 23), """##nl"""),
               ((23, 24), """##p"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]       ,	tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
       self.assertEqual([e[0] for e in expected_results]       ,	tokens["""offset_mapping"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   ["""的""", """人""", """有"""]
   lowercase__	: List[str]     		=   """""".join(SCREAMING_SNAKE_CASE_)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: Union[str, Any]     		=   True
       lowercase__	: Tuple     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: List[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[Any]     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: str     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: List[str]     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

       lowercase__	: Any     		=   False
       lowercase__	: Optional[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Tuple     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that only the first Chinese character is not preceded by "##".
       lowercase__	: Any     		=   [
           f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_)
       ]
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)


 @slow
 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[Any]     		=   tokenizer.encode("""你好"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
   lowercase__	: Any     		=   tokenizer.encode("""你是谁"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

   lowercase__	: Optional[Any]     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_)
   lowercase__	: Tuple     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]



 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_)
   for tokenizer in tokenizers:
     with self.subTest(f'{tokenizer.__class__.__name__}'):
       lowercase__	: Optional[int]     		=   """你好，你是谁"""
       lowercase__	: List[Any]     		=   tokenizer.tokenize(SCREAMING_SNAKE_CASE_)
       lowercase__	: Union[str, Any]     		=   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Any     		=   tokenizer.prepare_for_model(
           SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Dict     		=   tokenizer.encode_plus(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)






