






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class  lowerCAmelCase_       (				snake_case__     ,    unittest.TestCase					):
       UpperCamelCase_		:Optional[int]		      =    RoCBertTokenizer
       UpperCamelCase_		:List[Any]		      =    None
       UpperCamelCase_		:Union[str, Any]		      =    False
       UpperCamelCase_		:Dict		      =    True
       UpperCamelCase_		:Union[str, Any]		      =    filter_non_english




       def 						__snake_case					(   self     :   Any	):
           super().setUp()

           lowerCAmelCase__      			=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
           lowerCAmelCase__      			=						{}
           lowerCAmelCase__      			=						{}
           for i, value in enumerate(SCREAMING_SNAKE_CASE_	):
               lowerCAmelCase__      			=						i
               lowerCAmelCase__      			=						i
           lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''vocab_file''']	)
           lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''word_shape_file''']	)
           lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''word_pronunciation_file''']	)
           with open(self.vocab_file ,							'''w''' ,							encoding='''utf-8'''	) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
           with open(self.word_shape_file ,							'''w''' ,							encoding='''utf-8'''	) as word_shape_writer:
               json.dump(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							ensure_ascii=SCREAMING_SNAKE_CASE_	)
           with open(self.word_pronunciation_file ,							'''w''' ,							encoding='''utf-8'''	) as word_pronunciation_writer:
               json.dump(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							ensure_ascii=SCREAMING_SNAKE_CASE_	)




       def 						__snake_case					(   self     :   Any	):
           lowerCAmelCase__      			=						self.tokenizer_class(self.vocab_file ,							self.word_shape_file ,							self.word_pronunciation_file	)

           lowerCAmelCase__      			=						tokenizer.tokenize('''你好[SEP]你是谁'''	)
           self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)
           self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)
           self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)




       def 						__snake_case					(   self     :   Optional[int]	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	) ,							['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)




       def 						__snake_case					(   self     :   List[Any]	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	) ,							['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




       def 						__snake_case					(   self     :   Optional[int]	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''h\u00E9llo''']	)




       def 						__snake_case					(   self     :   int	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




       def 						__snake_case					(   self     :   Optional[Any]	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




       def 						__snake_case					(   self     :   Any	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	) ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




       def 						__snake_case					(   self     :   int	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




       def 						__snake_case					(   self     :   Tuple	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




       def 						__snake_case					(   self     :   Optional[Any]	):
           lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							never_split=['''[UNK]''']	)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	) ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)




       def 						__snake_case					(   self     :   Optional[Any]	):
           lowerCAmelCase__      			=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           lowerCAmelCase__      			=						{}
           for i, token in enumerate(SCREAMING_SNAKE_CASE_	):
               lowerCAmelCase__      			=						i
           lowerCAmelCase__      			=						RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_ ,							unk_token='''[UNK]'''	)

           self.assertListEqual(tokenizer.tokenize(''''''	) ,							[]	)

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''	) ,							['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	) ,							['''[UNK]''', '''runn''', '''##ing''']	)




       def 						__snake_case					(   self     :   Optional[Any]	):
           self.assertTrue(_is_whitespace(''' '''	)	)
           self.assertTrue(_is_whitespace('''\t'''	)	)
           self.assertTrue(_is_whitespace('''\r'''	)	)
           self.assertTrue(_is_whitespace('''\n'''	)	)
           self.assertTrue(_is_whitespace('''\u00A0'''	)	)

           self.assertFalse(_is_whitespace('''A'''	)	)
           self.assertFalse(_is_whitespace('''-'''	)	)




       def 						__snake_case					(   self     :   Tuple	):
           self.assertTrue(_is_control('''\u0005'''	)	)

           self.assertFalse(_is_control('''A'''	)	)
           self.assertFalse(_is_control(''' '''	)	)
           self.assertFalse(_is_control('''\t'''	)	)
           self.assertFalse(_is_control('''\r'''	)	)




       def 						__snake_case					(   self     :   int	):
           self.assertTrue(_is_punctuation('''-'''	)	)
           self.assertTrue(_is_punctuation('''$'''	)	)
           self.assertTrue(_is_punctuation('''`'''	)	)
           self.assertTrue(_is_punctuation('''.'''	)	)

           self.assertFalse(_is_punctuation('''A'''	)	)
           self.assertFalse(_is_punctuation(''' '''	)	)




       def 						__snake_case					(   self     :   str	):
           lowerCAmelCase__      			=						self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_	) for t in ['''Test''', '''\xad''', '''test''']] ,							[['''[UNK]'''], [], ['''[UNK]''']]	)

           if self.test_rust_tokenizer:
               lowerCAmelCase__      			=						self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_	) for t in ['''Test''', '''\xad''', '''test''']] ,							[['''[UNK]'''], [], ['''[UNK]''']]	)




       def 						__snake_case					(   self     :   Dict	):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                   lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                   lowerCAmelCase__      			=						tokenizer_r.encode_plus(
                       SCREAMING_SNAKE_CASE_ ,							return_attention_mask=SCREAMING_SNAKE_CASE_ ,							return_token_type_ids=SCREAMING_SNAKE_CASE_ ,							return_offsets_mapping=SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_ ,							)

                   lowerCAmelCase__      			=						tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_ ,							'''do_lower_case'''	) else False
                   lowerCAmelCase__      			=						(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results] ,							tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
                   self.assertEqual([e[0] for e in expected_results] ,							tokens['''offset_mapping''']	)




       def 						__snake_case					(   self     :   Optional[Any]	):
           lowerCAmelCase__      			=						['''的''', '''人''', '''有''']
           lowerCAmelCase__      			=						''''''.join(SCREAMING_SNAKE_CASE_	)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                   lowerCAmelCase__      			=						True
                   lowerCAmelCase__      			=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						tokenizer_p.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer_r.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)
                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						False
                   lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						tokenizer_r.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer_p.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)

                   # it is expected that only the first Chinese character is not preceded by "##".
                   lowerCAmelCase__      			=						[
                       f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_	)
                   ]
                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)
                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)




       @slow
       def 						__snake_case					(   self     :   Tuple	):
           lowerCAmelCase__      			=						self.tokenizer_class(self.vocab_file ,							self.word_shape_file ,							self.word_pronunciation_file	)

           lowerCAmelCase__      			=						tokenizer.encode('''你好''' ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
           lowerCAmelCase__      			=						tokenizer.encode('''你是谁''' ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

           lowerCAmelCase__      			=						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_	)
           lowerCAmelCase__      			=						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)

           assert encoded_sentence == [1] + text + [2]
           assert encoded_pair == [1] + text + [2] + text_a + [2]




       def 						__snake_case					(   self     :   Union[str, Any]	):
           lowerCAmelCase__      			=						self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_	)
           for tokenizer in tokenizers:
               with self.subTest(f'{tokenizer.__class__.__name__}'	):
                   lowerCAmelCase__      			=						'''你好，你是谁'''
                   lowerCAmelCase__      			=						tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_	)
                   lowerCAmelCase__      			=						tokenizer.prepare_for_model(
                       SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

                   lowerCAmelCase__      			=						tokenizer.encode_plus(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

                   self.assertEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)













from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_UpperCAmelCase			:		List[str]   				=    {
    "configuration_electra": ["ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP", "ElectraConfig", "ElectraOnnxConfig"],
    "tokenization_electra": ["ElectraTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Dict   				=    ["ElectraTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Optional[Any]   				=    [
			    "ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "ElectraForCausalLM",
			    "ElectraForMaskedLM",
			    "ElectraForMultipleChoice",
			    "ElectraForPreTraining",
			    "ElectraForQuestionAnswering",
			    "ElectraForSequenceClassification",
			    "ElectraForTokenClassification",
			    "ElectraModel",
			    "ElectraPreTrainedModel",
			    "load_tf_weights_in_electra",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Union[str, Any]   				=    [
			    "TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFElectraForMaskedLM",
			    "TFElectraForMultipleChoice",
			    "TFElectraForPreTraining",
			    "TFElectraForQuestionAnswering",
			    "TFElectraForSequenceClassification",
			    "TFElectraForTokenClassification",
			    "TFElectraModel",
			    "TFElectraPreTrainedModel",
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Optional[int]   				=    [
			    "FlaxElectraForCausalLM",
			    "FlaxElectraForMaskedLM",
			    "FlaxElectraForMultipleChoice",
			    "FlaxElectraForPreTraining",
			    "FlaxElectraForQuestionAnswering",
			    "FlaxElectraForSequenceClassification",
			    "FlaxElectraForTokenClassification",
			    "FlaxElectraModel",
			    "FlaxElectraPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
			from .tokenization_electra import ElectraTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_electra_fast import ElectraTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_electra import (
						    ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
						    ElectraForCausalLM,
						    ElectraForMaskedLM,
						    ElectraForMultipleChoice,
						    ElectraForPreTraining,
						    ElectraForQuestionAnswering,
						    ElectraForSequenceClassification,
						    ElectraForTokenClassification,
						    ElectraModel,
						    ElectraPreTrainedModel,
						    load_tf_weights_in_electra,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_electra import (
						    TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFElectraForMaskedLM,
						    TFElectraForMultipleChoice,
						    TFElectraForPreTraining,
						    TFElectraForQuestionAnswering,
						    TFElectraForSequenceClassification,
						    TFElectraForTokenClassification,
						    TFElectraModel,
						    TFElectraPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_electra import (
						    FlaxElectraForCausalLM,
						    FlaxElectraForMaskedLM,
						    FlaxElectraForMultipleChoice,
						    FlaxElectraForPreTraining,
						    FlaxElectraForQuestionAnswering,
						    FlaxElectraForSequenceClassification,
						    FlaxElectraForTokenClassification,
						    FlaxElectraModel,
						    FlaxElectraPreTrainedModel,
						)

else:
			import sys

			_UpperCAmelCase			:		Any   				=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





