



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 							a_  (		_snake_case					, unittest.TestCase				):
						UpperCamelCase__      :						Dict				       =ElectraTokenizer
						UpperCamelCase__      :						Union[str, Any]				       =ElectraTokenizerFast
						UpperCamelCase__      :						List[str]				       =True
						UpperCamelCase__      :						str				       =True
						UpperCamelCase__      :						Union[str, Any]				       =filter_non_english

						def 	__a       (    self	:List[Any])		->					Union[str, Any]:
												super().setUp()

												UpperCAmelCase_     							=     [
												    '''[UNK]''',
												    '''[CLS]''',
												    '''[SEP]''',
												    '''[PAD]''',
												    '''[MASK]''',
												    '''want''',
												    '''##want''',
												    '''##ed''',
												    '''wa''',
												    '''un''',
												    '''runn''',
												    '''##ing''',
												    ''',''',
												    '''low''',
												    '''lowest''',
												]
												UpperCAmelCase_     							=     os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''vocab_file'''])
												with open(self.vocab_file   ,						'''w'''   ,						encoding='''utf-8''') as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))

						def 	__a       (    self	:Any   ,						_lowercase	:str)		->					Any:
												UpperCAmelCase_     							=     '''UNwant\u00E9d,running'''
												UpperCAmelCase_     							=     '''unwanted, running'''
												return input_text, output_text

						def 	__a       (    self	:Dict)		->					int:
												UpperCAmelCase_     							=     self.tokenizer_class(self.vocab_file)

												UpperCAmelCase_     							=     tokenizer.tokenize('''UNwant\u00E9d,running''')
												self.assertListEqual(_lowercase   ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase)   ,						[9, 6, 7, 12, 10, 11])

						def 	__a       (    self	:Tuple)		->					str:
												if not self.test_rust_tokenizer:
																		return

												UpperCAmelCase_     							=     self.get_tokenizer()
												UpperCAmelCase_     							=     self.get_rust_tokenizer()

												UpperCAmelCase_     							=     '''UNwant\u00E9d,running'''

												UpperCAmelCase_     							=     tokenizer.tokenize(_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.tokenize(_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

												UpperCAmelCase_     							=     tokenizer.encode(_lowercase   ,						add_special_tokens=_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.encode(_lowercase   ,						add_special_tokens=_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

												UpperCAmelCase_     							=     self.get_rust_tokenizer()
												UpperCAmelCase_     							=     tokenizer.encode(_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.encode(_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

												# With lower casing
												UpperCAmelCase_     							=     self.get_tokenizer(do_lower_case=_lowercase)
												UpperCAmelCase_     							=     self.get_rust_tokenizer(do_lower_case=_lowercase)

												UpperCAmelCase_     							=     '''UNwant\u00E9d,running'''

												UpperCAmelCase_     							=     tokenizer.tokenize(_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.tokenize(_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

												UpperCAmelCase_     							=     tokenizer.encode(_lowercase   ,						add_special_tokens=_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.encode(_lowercase   ,						add_special_tokens=_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

												UpperCAmelCase_     							=     self.get_rust_tokenizer()
												UpperCAmelCase_     							=     tokenizer.encode(_lowercase)
												UpperCAmelCase_     							=     rust_tokenizer.encode(_lowercase)
												self.assertListEqual(_lowercase   ,						_lowercase)

						def 	__a       (    self	:int)		->					Optional[Any]:
												UpperCAmelCase_     							=     BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')   ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])

						def 	__a       (    self	:Optional[Any])		->					Union[str, Any]:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')   ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')   ,						['''hello'''])

						def 	__a       (    self	:Optional[Any])		->					Tuple:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase   ,						strip_accents=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')   ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')   ,						['''h\u00E9llo'''])

						def 	__a       (    self	:Optional[Any])		->					int:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase   ,						strip_accents=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')   ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')   ,						['''hello'''])

						def 	__a       (    self	:Union[str, Any])		->					Tuple:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')   ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')   ,						['''hello'''])

						def 	__a       (    self	:List[Any])		->					List[str]:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')   ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])

						def 	__a       (    self	:List[str])		->					Union[str, Any]:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase   ,						strip_accents=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')   ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])

						def 	__a       (    self	:str)		->					Union[str, Any]:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase   ,						strip_accents=_lowercase)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')   ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])

						def 	__a       (    self	:List[str])		->					List[Any]:
												UpperCAmelCase_     							=     BasicTokenizer(do_lower_case=_lowercase   ,						never_split=['''[UNK]'''])

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')   ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])

						def 	__a       (    self	:Tuple)		->					List[Any]:
												UpperCAmelCase_     							=     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

												UpperCAmelCase_     							=     {}
												for i, token in enumerate(_lowercase):
																		UpperCAmelCase_     							=     i
												UpperCAmelCase_     							=     WordpieceTokenizer(vocab=_lowercase   ,						unk_token='''[UNK]''')

												self.assertListEqual(tokenizer.tokenize('''''')   ,						[])

												self.assertListEqual(tokenizer.tokenize('''unwanted running''')   ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

												self.assertListEqual(tokenizer.tokenize('''unwantedX running''')   ,						['''[UNK]''', '''runn''', '''##ing'''])

						def 	__a       (    self	:str)		->					List[Any]:
												self.assertTrue(_is_whitespace(''' '''))
												self.assertTrue(_is_whitespace('''\t'''))
												self.assertTrue(_is_whitespace('''\r'''))
												self.assertTrue(_is_whitespace('''\n'''))
												self.assertTrue(_is_whitespace('''\u00A0'''))

												self.assertFalse(_is_whitespace('''A'''))
												self.assertFalse(_is_whitespace('''-'''))

						def 	__a       (    self	:Tuple)		->					List[str]:
												self.assertTrue(_is_control('''\u0005'''))

												self.assertFalse(_is_control('''A'''))
												self.assertFalse(_is_control(''' '''))
												self.assertFalse(_is_control('''\t'''))
												self.assertFalse(_is_control('''\r'''))

						def 	__a       (    self	:Tuple)		->					int:
												self.assertTrue(_is_punctuation('''-'''))
												self.assertTrue(_is_punctuation('''$'''))
												self.assertTrue(_is_punctuation('''`'''))
												self.assertTrue(_is_punctuation('''.'''))

												self.assertFalse(_is_punctuation('''A'''))
												self.assertFalse(_is_punctuation(''' '''))

						def 	__a       (    self	:str)		->					Optional[Any]:
												UpperCAmelCase_     							=     self.get_tokenizer()
												UpperCAmelCase_     							=     self.get_rust_tokenizer()

												# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
												self.assertListEqual([tokenizer.tokenize(_lowercase) for t in ['''Test''', '''\xad''', '''test''']]   ,						[['''[UNK]'''], [], ['''[UNK]''']])

												self.assertListEqual(
												    [rust_tokenizer.tokenize(_lowercase) for t in ['''Test''', '''\xad''', '''test''']]   ,						[['''[UNK]'''], [], ['''[UNK]''']])

						@slow
						def 	__a       (    self	:Any)		->					int:
												UpperCAmelCase_     							=     self.tokenizer_class.from_pretrained('''google/electra-base-discriminator''')

												UpperCAmelCase_     							=     tokenizer.encode('''sequence builders'''   ,						add_special_tokens=_lowercase)
												UpperCAmelCase_     							=     tokenizer.encode('''multi-sequence build'''   ,						add_special_tokens=_lowercase)

												UpperCAmelCase_     							=     tokenizer.build_inputs_with_special_tokens(_lowercase)
												UpperCAmelCase_     							=     tokenizer.build_inputs_with_special_tokens(_lowercase   ,						_lowercase)

												assert encoded_sentence == [101] + text + [102]
												assert encoded_pair == [101] + text + [102] + text_a + [102]

						def 	__a       (    self	:int)		->					List[Any]:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
																								UpperCAmelCase_     							=     self.rust_tokenizer_class.from_pretrained(_lowercase   ,						**_lowercase)

																								UpperCAmelCase_     							=     f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																								UpperCAmelCase_     							=     tokenizer_r.encode_plus(
																								    _lowercase   ,						return_attention_mask=_lowercase   ,						return_token_type_ids=_lowercase   ,						return_offsets_mapping=_lowercase   ,						add_special_tokens=_lowercase   ,						)

																								UpperCAmelCase_     							=     tokenizer_r.do_lower_case if hasattr(_lowercase   ,						'''do_lower_case''') else False
																								UpperCAmelCase_     							=     (
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), '''A'''),
																								        ((1, 2), ''','''),
																								        ((3, 5), '''na'''),
																								        ((5, 6), '''##ï'''),
																								        ((6, 8), '''##ve'''),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), '''Allen'''),
																								        ((21, 23), '''##NL'''),
																								        ((23, 24), '''##P'''),
																								        ((25, 33), '''sentence'''),
																								        ((33, 34), '''.'''),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), '''a'''),
																								        ((1, 2), ''','''),
																								        ((3, 8), '''naive'''),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), '''allen'''),
																								        ((21, 23), '''##nl'''),
																								        ((23, 24), '''##p'''),
																								        ((25, 33), '''sentence'''),
																								        ((33, 34), '''.'''),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]   ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
																								self.assertEqual([e[0] for e in expected_results]   ,						tokens['''offset_mapping'''])




						def 	__a       (    self	:Union[str, Any])		->					List[str]:
												UpperCAmelCase_     							=     ['''的''', '''人''', '''有''']
												UpperCAmelCase_     							=     ''''''.join(_lowercase)
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
																								UpperCAmelCase_     							=     True
																								UpperCAmelCase_     							=     self.tokenizer_class.from_pretrained(_lowercase   ,						**_lowercase)
																								UpperCAmelCase_     							=     self.rust_tokenizer_class.from_pretrained(_lowercase   ,						**_lowercase)

																								UpperCAmelCase_     							=     tokenizer_p.encode(_lowercase   ,						add_special_tokens=_lowercase)
																								UpperCAmelCase_     							=     tokenizer_r.encode(_lowercase   ,						add_special_tokens=_lowercase)

																								UpperCAmelCase_     							=     tokenizer_r.convert_ids_to_tokens(_lowercase)
																								UpperCAmelCase_     							=     tokenizer_p.convert_ids_to_tokens(_lowercase)

																								# it is expected that each Chinese character is not preceded by "##"
																								self.assertListEqual(_lowercase   ,						_lowercase)
																								self.assertListEqual(_lowercase   ,						_lowercase)

																								UpperCAmelCase_     							=     False
																								UpperCAmelCase_     							=     self.rust_tokenizer_class.from_pretrained(_lowercase   ,						**_lowercase)
																								UpperCAmelCase_     							=     self.tokenizer_class.from_pretrained(_lowercase   ,						**_lowercase)

																								UpperCAmelCase_     							=     tokenizer_r.encode(_lowercase   ,						add_special_tokens=_lowercase)
																								UpperCAmelCase_     							=     tokenizer_p.encode(_lowercase   ,						add_special_tokens=_lowercase)

																								UpperCAmelCase_     							=     tokenizer_r.convert_ids_to_tokens(_lowercase)
																								UpperCAmelCase_     							=     tokenizer_p.convert_ids_to_tokens(_lowercase)

																								# it is expected that only the first Chinese character is not preceded by "##".
																								UpperCAmelCase_     							=     [
																								    f"##{token}" if idx != 0 else token for idx, token in enumerate(_lowercase)
																								]
																								self.assertListEqual(_lowercase   ,						_lowercase)
																								self.assertListEqual(_lowercase   ,						_lowercase)










from dataclasses import dataclass
from typing import Callable, Optional

import datasets







@dataclass
class 							a_  (		datasets.BuilderConfig				):
						UpperCamelCase__      :						Optional[Callable]				       =None
						UpperCamelCase__      :						Optional[dict]				       =None
						UpperCamelCase__      :						Optional[datasets.Features]				       =None

						def 	__a       (    self	:List[str])		->					List[str]:
												assert self.generator is not None, "generator must be specified"

												if self.gen_kwargs is None:
																		UpperCAmelCase_     							=     {}







class 							a_  (		datasets.GeneratorBasedBuilder				):
						UpperCamelCase__      :						List[str]				       =GeneratorConfig

						def 	__a       (    self	:List[str])		->					List[Any]:
												return datasets.DatasetInfo(features=self.config.features)

						def 	__a       (    self	:Optional[Any]   ,						_lowercase	:List[str])		->					Optional[Any]:
												return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,						gen_kwargs=self.config.gen_kwargs)]




						def 	__a       (    self	:Optional[int]   ,						**_lowercase	:str)		->					Union[str, Any]:
												for idx, ex in enumerate(self.config.generator(**_lowercase)):
																		yield idx, ex





