
'''simple docstring'''



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCAmelCase_     (		_SCREAMING_SNAKE_CASE       ,							unittest.TestCase ):






						'''simple docstring'''






						_lowercase		:      Optional[int]									=   ElectraTokenizer
						_lowercase		:      int									=   ElectraTokenizerFast
						_lowercase		:      Optional[Any]									=   True
						_lowercase		:      str									=   True
						_lowercase		:      Tuple									=   filter_non_english




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												super().setUp()

												_lowerCAmelCase  						= [
												    "[UNK]",
												    "[CLS]",
												    "[SEP]",
												    "[PAD]",
												    "[MASK]",
												    "want",
												    "##want",
												    "##ed",
												    "wa",
												    "un",
												    "runn",
												    "##ing",
												    ",",
												    "low",
												    "lowest",
												]
												_lowerCAmelCase  						= os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES["""vocab_file"""]      )
												with open(self.vocab_file    ,   """w"""    ,   encoding="""utf-8"""      ) as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )




						def 					_lowercase	(					self    ,   _lowercase      ):

												"""simple docstring"""



												_lowerCAmelCase  						= "UNwant\u00E9d,running"
												_lowerCAmelCase  						= "unwanted, running"
												return input_text, output_text




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= self.tokenizer_class(self.vocab_file      )

												_lowerCAmelCase  						= tokenizer.tokenize("""UNwant\u00E9d,running"""      )
												self.assertListEqual(A_    ,   ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]      )
												self.assertListEqual(tokenizer.convert_tokens_to_ids(A_      )    ,   [9, 6, 7, 12, 10, 11]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												if not self.test_rust_tokenizer:
																		return

												_lowerCAmelCase  						= self.get_tokenizer()
												_lowerCAmelCase  						= self.get_rust_tokenizer()

												_lowerCAmelCase  						= "UNwant\u00E9d,running"

												_lowerCAmelCase  						= tokenizer.tokenize(A_      )
												_lowerCAmelCase  						= rust_tokenizer.tokenize(A_      )
												self.assertListEqual(A_    ,   A_      )

												_lowerCAmelCase  						= tokenizer.encode(A_    ,   add_special_tokens=A_      )
												_lowerCAmelCase  						= rust_tokenizer.encode(A_    ,   add_special_tokens=A_      )
												self.assertListEqual(A_    ,   A_      )

												_lowerCAmelCase  						= self.get_rust_tokenizer()
												_lowerCAmelCase  						= tokenizer.encode(A_      )
												_lowerCAmelCase  						= rust_tokenizer.encode(A_      )
												self.assertListEqual(A_    ,   A_      )

												# With lower casing
												_lowerCAmelCase  						= self.get_tokenizer(do_lower_case=A_      )
												_lowerCAmelCase  						= self.get_rust_tokenizer(do_lower_case=A_      )

												_lowerCAmelCase  						= "UNwant\u00E9d,running"

												_lowerCAmelCase  						= tokenizer.tokenize(A_      )
												_lowerCAmelCase  						= rust_tokenizer.tokenize(A_      )
												self.assertListEqual(A_    ,   A_      )

												_lowerCAmelCase  						= tokenizer.encode(A_    ,   add_special_tokens=A_      )
												_lowerCAmelCase  						= rust_tokenizer.encode(A_    ,   add_special_tokens=A_      )
												self.assertListEqual(A_    ,   A_      )

												_lowerCAmelCase  						= self.get_rust_tokenizer()
												_lowerCAmelCase  						= tokenizer.encode(A_      )
												_lowerCAmelCase  						= rust_tokenizer.encode(A_      )
												self.assertListEqual(A_    ,   A_      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""      )    ,   ["""ah""", """\u535A""", """\u63A8""", """zz"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )    ,   ["""hello""", """!""", """how""", """are""", """you""", """?"""]      )
												self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )    ,   ["""hello"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_    ,   strip_accents=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )    ,   ["""hällo""", """!""", """how""", """are""", """you""", """?"""]      )
												self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )    ,   ["""h\u00E9llo"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_    ,   strip_accents=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )    ,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
												self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )    ,   ["""hello"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )    ,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
												self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )    ,   ["""hello"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )    ,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_    ,   strip_accents=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )    ,   ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_    ,   strip_accents=A_      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )    ,   ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= BasicTokenizer(do_lower_case=A_    ,   never_split=["""[UNK]"""]      )

												self.assertListEqual(
												    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""      )    ,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

												_lowerCAmelCase  						= {}
												for i, token in enumerate(A_      ):
																		_lowerCAmelCase  						= i
												_lowerCAmelCase  						= WordpieceTokenizer(vocab=A_    ,   unk_token="""[UNK]"""      )

												self.assertListEqual(tokenizer.tokenize(""""""      )    ,   []      )

												self.assertListEqual(tokenizer.tokenize("""unwanted running"""      )    ,   ["""un""", """##want""", """##ed""", """runn""", """##ing"""]      )

												self.assertListEqual(tokenizer.tokenize("""unwantedX running"""      )    ,   ["""[UNK]""", """runn""", """##ing"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												self.assertTrue(_is_whitespace(""" """      )      )
												self.assertTrue(_is_whitespace("""\t"""      )      )
												self.assertTrue(_is_whitespace("""\r"""      )      )
												self.assertTrue(_is_whitespace("""\n"""      )      )
												self.assertTrue(_is_whitespace("""\u00A0"""      )      )

												self.assertFalse(_is_whitespace("""A"""      )      )
												self.assertFalse(_is_whitespace("""-"""      )      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												self.assertTrue(_is_control("""\u0005"""      )      )

												self.assertFalse(_is_control("""A"""      )      )
												self.assertFalse(_is_control(""" """      )      )
												self.assertFalse(_is_control("""\t"""      )      )
												self.assertFalse(_is_control("""\r"""      )      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												self.assertTrue(_is_punctuation("""-"""      )      )
												self.assertTrue(_is_punctuation("""$"""      )      )
												self.assertTrue(_is_punctuation("""`"""      )      )
												self.assertTrue(_is_punctuation("""."""      )      )

												self.assertFalse(_is_punctuation("""A"""      )      )
												self.assertFalse(_is_punctuation(""" """      )      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= self.get_tokenizer()
												_lowerCAmelCase  						= self.get_rust_tokenizer()

												# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
												self.assertListEqual([tokenizer.tokenize(A_      ) for t in ["""Test""", """\xad""", """test"""]]    ,   [["""[UNK]"""], [], ["""[UNK]"""]]      )

												self.assertListEqual(
												    [rust_tokenizer.tokenize(A_      ) for t in ["""Test""", """\xad""", """test"""]]    ,   [["""[UNK]"""], [], ["""[UNK]"""]]      )




						@slow
						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""      )

												_lowerCAmelCase  						= tokenizer.encode("""sequence builders"""    ,   add_special_tokens=A_      )
												_lowerCAmelCase  						= tokenizer.encode("""multi-sequence build"""    ,   add_special_tokens=A_      )

												_lowerCAmelCase  						= tokenizer.build_inputs_with_special_tokens(A_      )
												_lowerCAmelCase  						= tokenizer.build_inputs_with_special_tokens(A_    ,   A_      )

												assert encoded_sentence == [101] + text + [102]
												assert encoded_pair == [101] + text + [102] + text_a + [102]




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																								_lowerCAmelCase  						= self.rust_tokenizer_class.from_pretrained(A_    ,   **A_      )

																								_lowerCAmelCase  						= F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																								_lowerCAmelCase  						= tokenizer_r.encode_plus(
																								    A_    ,   return_attention_mask=A_    ,   return_token_type_ids=A_    ,   return_offsets_mapping=A_    ,   add_special_tokens=A_    ,   )

																								_lowerCAmelCase  						= tokenizer_r.do_lower_case if hasattr(A_    ,   """do_lower_case"""      ) else False
																								_lowerCAmelCase  						= (
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "A"),
																								        ((1, 2), ","),
																								        ((3, 5), "na"),
																								        ((5, 6), "##ï"),
																								        ((6, 8), "##ve"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "Allen"),
																								        ((21, 23), "##NL"),
																								        ((23, 24), "##P"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "a"),
																								        ((1, 2), ","),
																								        ((3, 8), "naive"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "allen"),
																								        ((21, 23), "##nl"),
																								        ((23, 24), "##p"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]    ,   tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]      )      )
																								self.assertEqual([e[0] for e in expected_results]    ,   tokens["""offset_mapping"""]      )




						def 					_lowercase	(					self      ):

												"""simple docstring"""



												_lowerCAmelCase  						= ["的", "人", "有"]
												_lowerCAmelCase  						= "".join(A_      )
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																								_lowerCAmelCase  						= True
																								_lowerCAmelCase  						= self.tokenizer_class.from_pretrained(A_    ,   **A_      )
																								_lowerCAmelCase  						= self.rust_tokenizer_class.from_pretrained(A_    ,   **A_      )

																								_lowerCAmelCase  						= tokenizer_p.encode(A_    ,   add_special_tokens=A_      )
																								_lowerCAmelCase  						= tokenizer_r.encode(A_    ,   add_special_tokens=A_      )

																								_lowerCAmelCase  						= tokenizer_r.convert_ids_to_tokens(A_      )
																								_lowerCAmelCase  						= tokenizer_p.convert_ids_to_tokens(A_      )

																								# it is expected that each Chinese character is not preceded by "##"
																								self.assertListEqual(A_    ,   A_      )
																								self.assertListEqual(A_    ,   A_      )

																								_lowerCAmelCase  						= False
																								_lowerCAmelCase  						= self.rust_tokenizer_class.from_pretrained(A_    ,   **A_      )
																								_lowerCAmelCase  						= self.tokenizer_class.from_pretrained(A_    ,   **A_      )

																								_lowerCAmelCase  						= tokenizer_r.encode(A_    ,   add_special_tokens=A_      )
																								_lowerCAmelCase  						= tokenizer_p.encode(A_    ,   add_special_tokens=A_      )

																								_lowerCAmelCase  						= tokenizer_r.convert_ids_to_tokens(A_      )
																								_lowerCAmelCase  						= tokenizer_p.convert_ids_to_tokens(A_      )

																								# it is expected that only the first Chinese character is not preceded by "##".
																								_lowerCAmelCase  						= [
																								    F'##{token}' if idx != 0 else token for idx, token in enumerate(A_      )
																								]
																								self.assertListEqual(A_    ,   A_      )
																								self.assertListEqual(A_    ,   A_      )



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE_						=					{'configuration_hubert': ['HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'HubertConfig']}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_						=					[
					    'HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'HubertForCTC',
					    'HubertForSequenceClassification',
					    'HubertModel',
					    'HubertPreTrainedModel',
					]


try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_						=					[
					    'TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'TFHubertForCTC',
					    'TFHubertModel',
					    'TFHubertPreTrainedModel',
					]

if TYPE_CHECKING:
					from .configuration_hubert import HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, HubertConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_hubert import (
										    HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    HubertForCTC,
										    HubertForSequenceClassification,
										    HubertModel,
										    HubertPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_hubert import (
										    TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFHubertForCTC,
										    TFHubertModel,
										    TFHubertPreTrainedModel,
										)


else:
					import sys

					SCREAMING_SNAKE_CASE_						=					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


