

"""simple docstring"""




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      _UpperCAmelCase   (							_UpperCamelCase		,    unittest.TestCase       ):
	a__		: Optional[Any]					=					MobileBertTokenizer
	a__		: Optional[Any]					=					MobileBertTokenizerFast
	a__		: Optional[int]					=					True
	a__		: Any					=					True
	a__		: Dict					=					filter_non_english
	a__		: Optional[int]					=					'google/mobilebert-uncased'



	def        a  (  self     :       Tuple						):
		super().setUp()

		__UpperCAmelCase      		= [
		    '[UNK]',
		    '[CLS]',
		    '[SEP]',
		    '[PAD]',
		    '[MASK]',
		    'want',
		    '##want',
		    '##ed',
		    'wa',
		    'un',
		    'runn',
		    '##ing',
		    ',',
		    'low',
		    'lowest',
		]
		__UpperCAmelCase      		= os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['''vocab_file''']						)
		with open(self.vocab_file			,    '''w'''			,    encoding='''utf-8'''						) as vocab_writer:
			vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)

		__UpperCAmelCase      		= [
		    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
		    for tokenizer_def in self.tokenizers_list
		]



	def        a  (  self     :       List[Any]			,    _lowercase     :       int						):
		__UpperCAmelCase      		= 'UNwant\u00E9d,running'
		__UpperCAmelCase      		= 'unwanted, running'
		return input_text, output_text



	def        a  (  self     :       Union[str, Any]						):
		__UpperCAmelCase      		= self.tokenizer_class(self.vocab_file						)

		__UpperCAmelCase      		= tokenizer.tokenize('''UNwant\u00E9d,running'''						)
		self.assertListEqual(_UpperCAmelCase			,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']						)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)			,    [9, 6, 7, 12, 10, 11]						)



	def        a  (  self     :       Optional[Any]						):
		if not self.test_rust_tokenizer:
			return

		__UpperCAmelCase      		= self.get_tokenizer()
		__UpperCAmelCase      		= self.get_rust_tokenizer()

		__UpperCAmelCase      		= 'UNwant\u00E9d,running'

		__UpperCAmelCase      		= tokenizer.tokenize(_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.tokenize(_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

		__UpperCAmelCase      		= tokenizer.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

		__UpperCAmelCase      		= self.get_rust_tokenizer()
		__UpperCAmelCase      		= tokenizer.encode(_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.encode(_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

		# With lower casing
		__UpperCAmelCase      		= self.get_tokenizer(do_lower_case=_UpperCAmelCase						)
		__UpperCAmelCase      		= self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase						)

		__UpperCAmelCase      		= 'UNwant\u00E9d,running'

		__UpperCAmelCase      		= tokenizer.tokenize(_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.tokenize(_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

		__UpperCAmelCase      		= tokenizer.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

		__UpperCAmelCase      		= self.get_rust_tokenizer()
		__UpperCAmelCase      		= tokenizer.encode(_UpperCAmelCase						)
		__UpperCAmelCase      		= rust_tokenizer.encode(_UpperCAmelCase						)
		self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)



	def        a  (  self     :       List[Any]						):
		__UpperCAmelCase      		= BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''						)			,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']						)



	def        a  (  self     :       List[Any]						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''						)			,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
		self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)			,    ['''hello''']						)



	def        a  (  self     :       str						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase			,    strip_accents=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)			,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
		self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)			,    ['''h\u00E9llo''']						)



	def        a  (  self     :       List[str]						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase			,    strip_accents=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)			,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
		self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)			,    ['''hello''']						)



	def        a  (  self     :       str						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)			,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']						)
		self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''						)			,    ['''hello''']						)



	def        a  (  self     :       Union[str, Any]						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''						)			,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)



	def        a  (  self     :       Any						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase			,    strip_accents=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)			,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)



	def        a  (  self     :       List[Any]						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase			,    strip_accents=_UpperCAmelCase						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''						)			,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)



	def        a  (  self     :       List[Any]						):
		__UpperCAmelCase      		= BasicTokenizer(do_lower_case=_UpperCAmelCase			,    never_split=['''[UNK]''']						)

		self.assertListEqual(
		    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''						)			,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']						)



	def        a  (  self     :       Optional[int]						):
		__UpperCAmelCase      		= ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

		__UpperCAmelCase      		= {}
		for i, token in enumerate(_UpperCAmelCase						):
			__UpperCAmelCase      		= i
		__UpperCAmelCase      		= WordpieceTokenizer(vocab=_UpperCAmelCase			,    unk_token='''[UNK]'''						)

		self.assertListEqual(tokenizer.tokenize(''''''						)			,    []						)

		self.assertListEqual(tokenizer.tokenize('''unwanted running'''						)			,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']						)

		self.assertListEqual(tokenizer.tokenize('''unwantedX running'''						)			,    ['''[UNK]''', '''runn''', '''##ing''']						)



	def        a  (  self     :       Optional[Any]						):
		self.assertTrue(_is_whitespace(''' '''						)						)
		self.assertTrue(_is_whitespace('''\t'''						)						)
		self.assertTrue(_is_whitespace('''\r'''						)						)
		self.assertTrue(_is_whitespace('''\n'''						)						)
		self.assertTrue(_is_whitespace('''\u00A0'''						)						)

		self.assertFalse(_is_whitespace('''A'''						)						)
		self.assertFalse(_is_whitespace('''-'''						)						)



	def        a  (  self     :       Optional[int]						):
		self.assertTrue(_is_control('''\u0005'''						)						)

		self.assertFalse(_is_control('''A'''						)						)
		self.assertFalse(_is_control(''' '''						)						)
		self.assertFalse(_is_control('''\t'''						)						)
		self.assertFalse(_is_control('''\r'''						)						)



	def        a  (  self     :       Optional[Any]						):
		self.assertTrue(_is_punctuation('''-'''						)						)
		self.assertTrue(_is_punctuation('''$'''						)						)
		self.assertTrue(_is_punctuation('''`'''						)						)
		self.assertTrue(_is_punctuation('''.'''						)						)

		self.assertFalse(_is_punctuation('''A'''						)						)
		self.assertFalse(_is_punctuation(''' '''						)						)



	def        a  (  self     :       List[str]						):
		__UpperCAmelCase      		= self.get_tokenizer()
		__UpperCAmelCase      		= self.get_rust_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase						) for t in ['''Test''', '''\xad''', '''test''']]			,    [['''[UNK]'''], [], ['''[UNK]''']]						)

		self.assertListEqual(
		    [rust_tokenizer.tokenize(_UpperCAmelCase						) for t in ['''Test''', '''\xad''', '''test''']]			,    [['''[UNK]'''], [], ['''[UNK]''']]						)



	@slow
	def        a  (  self     :       List[str]						):
		__UpperCAmelCase      		= self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''						)

		__UpperCAmelCase      		= tokenizer.encode('''sequence builders'''			,    add_special_tokens=_UpperCAmelCase						)
		__UpperCAmelCase      		= tokenizer.encode('''multi-sequence build'''			,    add_special_tokens=_UpperCAmelCase						)

		__UpperCAmelCase      		= tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
		__UpperCAmelCase      		= tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase			,    _UpperCAmelCase						)

		assert encoded_sentence == [1_01] + text + [1_02]
		assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



	def        a  (  self     :       Dict						):
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
				__UpperCAmelCase      		= self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase			,    **_UpperCAmelCase						)

				__UpperCAmelCase      		= F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
				__UpperCAmelCase      		= tokenizer_r.encode_plus(
				    _UpperCAmelCase			,    return_attention_mask=_UpperCAmelCase			,    return_token_type_ids=_UpperCAmelCase			,    return_offsets_mapping=_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase			,    )

				__UpperCAmelCase      		= tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase			,    '''do_lower_case'''						) else False
				__UpperCAmelCase      		= (
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'A'),
				        ((1, 2), ','),
				        ((3, 5), 'na'),
				        ((5, 6), '##ï'),
				        ((6, 8), '##ve'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'Allen'),
				        ((21, 23), '##NL'),
				        ((23, 24), '##P'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'a'),
				        ((1, 2), ','),
				        ((3, 8), 'naive'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'allen'),
				        ((21, 23), '##nl'),
				        ((23, 24), '##p'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results]			,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']						)						)
				self.assertEqual([e[0] for e in expected_results]			,    tokens['''offset_mapping''']						)



	def        a  (  self     :       int						):
		__UpperCAmelCase      		= ['的', '人', '有']
		__UpperCAmelCase      		= ''.join(_UpperCAmelCase						)
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
				__UpperCAmelCase      		= True
				__UpperCAmelCase      		= self.tokenizer_class.from_pretrained(_UpperCAmelCase			,    **_UpperCAmelCase						)
				__UpperCAmelCase      		= self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase			,    **_UpperCAmelCase						)

				__UpperCAmelCase      		= tokenizer_p.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
				__UpperCAmelCase      		= tokenizer_r.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)

				__UpperCAmelCase      		= tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase						)
				__UpperCAmelCase      		= tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase						)

				# it is expected that each Chinese character is not preceded by "##"
				self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)
				self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)

				__UpperCAmelCase      		= False
				__UpperCAmelCase      		= self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase			,    **_UpperCAmelCase						)
				__UpperCAmelCase      		= self.tokenizer_class.from_pretrained(_UpperCAmelCase			,    **_UpperCAmelCase						)

				__UpperCAmelCase      		= tokenizer_r.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)
				__UpperCAmelCase      		= tokenizer_p.encode(_UpperCAmelCase			,    add_special_tokens=_UpperCAmelCase						)

				__UpperCAmelCase      		= tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase						)
				__UpperCAmelCase      		= tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase						)

				# it is expected that only the first Chinese character is not preceded by "##".
				__UpperCAmelCase      		= [
				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_UpperCAmelCase						)
				]
				self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)
				self.assertListEqual(_UpperCAmelCase			,    _UpperCAmelCase						)





'''simple docstring'''

from abc import ABC, abstractmethod
from argparse import ArgumentParser

class       __magic_name__		(       _UpperCamelCase  ):







							@staticmethod
							@abstractmethod
							def 				__lowercase     (				_UpperCAmelCase   :			ArgumentParser						):
									raise NotImplementedError()







							@abstractmethod
							def 				__lowercase     (				self   :			str						):
									raise NotImplementedError()




