"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							a					(		lowerCAmelCase_						,				unittest.TestCase ):
    _snake_case  :	Optional[Any]  				=				RoCBertTokenizer
    _snake_case  :	List[str]  				=				None
    _snake_case  :	Any  				=				False
    _snake_case  :	int  				=				True
    _snake_case  :	Any  				=				filter_non_english


    def   lowerCAmelCase_	(       self						:	str       ):
      super().setUp()

      _UpperCAmelCase    			=   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
      _UpperCAmelCase    			=   {}
      _UpperCAmelCase    			=   {}
      for i, value in enumerate(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   i
        _UpperCAmelCase    			=   i
      _UpperCAmelCase    			=   os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""vocab_file"""]       )
      _UpperCAmelCase    			=   os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""word_shape_file"""]       )
      _UpperCAmelCase    			=   os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""word_pronunciation_file"""]       )
      with open(self.vocab_file ,					"""w""" ,					encoding="""utf-8"""       ) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
      with open(self.word_shape_file ,					"""w""" ,					encoding="""utf-8"""       ) as word_shape_writer:
        json.dump(__lowerCAmelCase ,					__lowerCAmelCase ,					ensure_ascii=__lowerCAmelCase       )
      with open(self.word_pronunciation_file ,					"""w""" ,					encoding="""utf-8"""       ) as word_pronunciation_writer:
        json.dump(__lowerCAmelCase ,					__lowerCAmelCase ,					ensure_ascii=__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   self.tokenizer_class(self.vocab_file ,					self.word_shape_file ,					self.word_pronunciation_file       )

      _UpperCAmelCase    			=   tokenizer.tokenize("""你好[SEP]你是谁"""       )
      self.assertListEqual(__lowerCAmelCase ,					["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]       )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase       ) ,					[5, 6, 2, 5, 7, 8]       )
      self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__lowerCAmelCase       ) ,					[5, 6, 2, 5, 7, 8]       )
      self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__lowerCAmelCase       ) ,					[5, 6, 2, 5, 7, 8]       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       ) ,					["""ah""", """\u535A""", """\u63A8""", """zz"""]       )


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       ) ,					["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       ) ,					["""hello"""]       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase ,					strip_accents=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       ) ,					["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       ) ,					["""h\u00E9llo"""]       )


    def   lowerCAmelCase_	(       self						:	Tuple       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase ,					strip_accents=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       ) ,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       ) ,					["""hello"""]       )


    def   lowerCAmelCase_	(       self						:	str       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       ) ,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       ) ,					["""hello"""]       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       ) ,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )


    def   lowerCAmelCase_	(       self						:	str       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase ,					strip_accents=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       ) ,					["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase ,					strip_accents=__lowerCAmelCase       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       ) ,					["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )


    def   lowerCAmelCase_	(       self						:	int       ):
      _UpperCAmelCase    			=   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase ,					never_split=["""[UNK]"""]       )

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       ) ,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

      _UpperCAmelCase    			=   {}
      for i, token in enumerate(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   i
      _UpperCAmelCase    			=   RoCBertWordpieceTokenizer(vocab=__lowerCAmelCase ,					unk_token="""[UNK]"""       )

      self.assertListEqual(tokenizer.tokenize(""""""       ) ,					[]       )

      self.assertListEqual(tokenizer.tokenize("""unwanted running"""       ) ,					["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )

      self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       ) ,					["""[UNK]""", """runn""", """##ing"""]       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      self.assertTrue(_is_whitespace(""" """       )       )
      self.assertTrue(_is_whitespace("""\t"""       )       )
      self.assertTrue(_is_whitespace("""\r"""       )       )
      self.assertTrue(_is_whitespace("""\n"""       )       )
      self.assertTrue(_is_whitespace("""\u00A0"""       )       )

      self.assertFalse(_is_whitespace("""A"""       )       )
      self.assertFalse(_is_whitespace("""-"""       )       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.assertTrue(_is_control("""\u0005"""       )       )

      self.assertFalse(_is_control("""A"""       )       )
      self.assertFalse(_is_control(""" """       )       )
      self.assertFalse(_is_control("""\t"""       )       )
      self.assertFalse(_is_control("""\r"""       )       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      self.assertTrue(_is_punctuation("""-"""       )       )
      self.assertTrue(_is_punctuation("""$"""       )       )
      self.assertTrue(_is_punctuation("""`"""       )       )
      self.assertTrue(_is_punctuation("""."""       )       )

      self.assertFalse(_is_punctuation("""A"""       )       )
      self.assertFalse(_is_punctuation(""" """       )       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__lowerCAmelCase       ) for t in ["""Test""", """\xad""", """test"""]] ,					[["""[UNK]"""], [], ["""[UNK]"""]]       )

      if self.test_rust_tokenizer:
        _UpperCAmelCase    			=   self.get_rust_tokenizer()
        self.assertListEqual(
            [rust_tokenizer.tokenize(__lowerCAmelCase       ) for t in ["""Test""", """\xad""", """test"""]] ,					[["""[UNK]"""], [], ["""[UNK]"""]]       )


    def   lowerCAmelCase_	(       self						:	int       ):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
          _UpperCAmelCase    			=   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase ,					**__lowerCAmelCase       )

          _UpperCAmelCase    			=   f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          _UpperCAmelCase    			=   tokenizer_r.encode_plus(
              __lowerCAmelCase ,					return_attention_mask=__lowerCAmelCase ,					return_token_type_ids=__lowerCAmelCase ,					return_offsets_mapping=__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase ,					)

          _UpperCAmelCase    			=   tokenizer_r.do_lower_case if hasattr(__lowerCAmelCase ,					"""do_lower_case"""       ) else False
          _UpperCAmelCase    			=   (
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), """A"""),
                  ((1, 2), ""","""),
                  ((3, 5), """na"""),
                  ((5, 6), """##ï"""),
                  ((6, 8), """##ve"""),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), """Allen"""),
                  ((21, 23), """##NL"""),
                  ((23, 24), """##P"""),
                  ((25, 33), """sentence"""),
                  ((33, 34), """."""),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), """a"""),
                  ((1, 2), ""","""),
                  ((3, 8), """naive"""),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), """allen"""),
                  ((21, 23), """##nl"""),
                  ((23, 24), """##p"""),
                  ((25, 33), """sentence"""),
                  ((33, 34), """."""),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results] ,					tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
          self.assertEqual([e[0] for e in expected_results] ,					tokens["""offset_mapping"""]       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      _UpperCAmelCase    			=   ["""的""", """人""", """有"""]
      _UpperCAmelCase    			=   """""".join(__lowerCAmelCase       )
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
          _UpperCAmelCase    			=   True
          _UpperCAmelCase    			=   self.tokenizer_class.from_pretrained(__lowerCAmelCase ,					**__lowerCAmelCase       )
          _UpperCAmelCase    			=   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase ,					**__lowerCAmelCase       )

          _UpperCAmelCase    			=   tokenizer_p.encode(__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer_r.encode(__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )

          _UpperCAmelCase    			=   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase       )

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__lowerCAmelCase ,					__lowerCAmelCase       )
          self.assertListEqual(__lowerCAmelCase ,					__lowerCAmelCase       )

          _UpperCAmelCase    			=   False
          _UpperCAmelCase    			=   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase ,					**__lowerCAmelCase       )
          _UpperCAmelCase    			=   self.tokenizer_class.from_pretrained(__lowerCAmelCase ,					**__lowerCAmelCase       )

          _UpperCAmelCase    			=   tokenizer_r.encode(__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer_p.encode(__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )

          _UpperCAmelCase    			=   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase       )

          # it is expected that only the first Chinese character is not preceded by "##".
          _UpperCAmelCase    			=   [
              f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCAmelCase       )
          ]
          self.assertListEqual(__lowerCAmelCase ,					__lowerCAmelCase       )
          self.assertListEqual(__lowerCAmelCase ,					__lowerCAmelCase       )


    @slow
    def   lowerCAmelCase_	(       self						:	Tuple       ):
      _UpperCAmelCase    			=   self.tokenizer_class(self.vocab_file ,					self.word_shape_file ,					self.word_pronunciation_file       )

      _UpperCAmelCase    			=   tokenizer.encode("""你好""" ,					add_special_tokens=__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.encode("""你是谁""" ,					add_special_tokens=__lowerCAmelCase       )

      _UpperCAmelCase    			=   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase ,					__lowerCAmelCase       )

      assert encoded_sentence == [1] + text + [2]
      assert encoded_pair == [1] + text + [2] + text_a + [2]


    def   lowerCAmelCase_	(       self						:	str       ):
      _UpperCAmelCase    			=   self.get_tokenizers(do_lower_case=__lowerCAmelCase       )
      for tokenizer in tokenizers:
        with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
          _UpperCAmelCase    			=   """你好，你是谁"""
          _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer.convert_tokens_to_ids(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer.convert_tokens_to_shape_ids(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer.convert_tokens_to_pronunciation_ids(__lowerCAmelCase       )
          _UpperCAmelCase    			=   tokenizer.prepare_for_model(
              __lowerCAmelCase ,					__lowerCAmelCase ,					__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )

          _UpperCAmelCase    			=   tokenizer.encode_plus(__lowerCAmelCase ,					add_special_tokens=__lowerCAmelCase       )

          self.assertEqual(__lowerCAmelCase ,					__lowerCAmelCase       )

"""simple docstring"""


from transformers import BertTokenizerFast

from .custom_tokenization import CustomTokenizer


class 							a					(		lowerCAmelCase_ ):
    _snake_case  :	Dict  				=				CustomTokenizer
    pass
