
'''simple docstring'''


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 	__lowercase				(  __lowerCAmelCase      ,     unittest.TestCase    ):
			lowerCamelCase			:		List[str]	      =			BertTokenizer
			lowerCamelCase			:		Any	      =			BertTokenizerFast
			lowerCamelCase			:		List[Any]	      =			True
			lowerCamelCase			:		str	      =			True
			lowerCamelCase			:		List[str]	      =			filter_non_english

			def        UpperCAmelCase__ (self      ):
								super().setUp()

								lowerCamelCase_					:      int     =		[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								lowerCamelCase_					:      Tuple     =		os.path.join(self.tmpdirname			,   VOCAB_FILES_NAMES['''vocab_file''']      )
								with open(self.vocab_file			,   '''w'''			,   encoding='''utf-8'''      ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )

			def        UpperCAmelCase__ (self			,   A      ):
								lowerCamelCase_					:      List[str]     =		'''UNwant\u00E9d,running'''
								lowerCamelCase_					:      Dict     =		'''unwanted, running'''
								return input_text, output_text

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      int     =		self.tokenizer_class(self.vocab_file      )

								lowerCamelCase_					:      Optional[Any]     =		tokenizer.tokenize('''UNwant\u00E9d,running'''      )
								self.assertListEqual(lowerCamelCase__			,   ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__      )			,   [9, 6, 7, 1_2, 1_0, 1_1]      )

			def        UpperCAmelCase__ (self      ):






								if not self.test_rust_tokenizer:
													return

								lowerCamelCase_					:      str     =		self.get_tokenizer()
								lowerCamelCase_					:      Dict     =		self.get_rust_tokenizer()

								lowerCamelCase_					:      List[str]     =		'''UNwant\u00E9d,running'''

								lowerCamelCase_					:      Optional[Any]     =		tokenizer.tokenize(lowerCamelCase__      )
								lowerCamelCase_					:      Any     =		rust_tokenizer.tokenize(lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

								lowerCamelCase_					:      Any     =		tokenizer.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
								lowerCamelCase_					:      Dict     =		rust_tokenizer.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

								lowerCamelCase_					:      Optional[int]     =		self.get_rust_tokenizer()
								lowerCamelCase_					:      List[Any]     =		tokenizer.encode(lowerCamelCase__      )
								lowerCamelCase_					:      List[str]     =		rust_tokenizer.encode(lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

								# With lower casing
								lowerCamelCase_					:      Tuple     =		self.get_tokenizer(do_lower_case=lowerCamelCase__      )
								lowerCamelCase_					:      str     =		self.get_rust_tokenizer(do_lower_case=lowerCamelCase__      )

								lowerCamelCase_					:      List[Any]     =		'''UNwant\u00E9d,running'''

								lowerCamelCase_					:      Union[str, Any]     =		tokenizer.tokenize(lowerCamelCase__      )
								lowerCamelCase_					:      Dict     =		rust_tokenizer.tokenize(lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

								lowerCamelCase_					:      List[Any]     =		tokenizer.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
								lowerCamelCase_					:      Any     =		rust_tokenizer.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

								lowerCamelCase_					:      Dict     =		self.get_rust_tokenizer()
								lowerCamelCase_					:      str     =		tokenizer.encode(lowerCamelCase__      )
								lowerCamelCase_					:      List[Any]     =		rust_tokenizer.encode(lowerCamelCase__      )
								self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Tuple     =		BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )			,   ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		BasicTokenizer(do_lower_case=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )			,   ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )			,   ['''hello''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Any     =		BasicTokenizer(do_lower_case=lowerCamelCase__			,   strip_accents=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )			,   ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )			,   ['''h\u00E9llo''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Any     =		BasicTokenizer(do_lower_case=lowerCamelCase__			,   strip_accents=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )			,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )			,   ['''hello''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      str     =		BasicTokenizer(do_lower_case=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )			,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )			,   ['''hello''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Dict     =		BasicTokenizer(do_lower_case=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )			,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Any     =		BasicTokenizer(do_lower_case=lowerCamelCase__			,   strip_accents=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )			,   ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		BasicTokenizer(do_lower_case=lowerCamelCase__			,   strip_accents=lowerCamelCase__      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )			,   ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		BasicTokenizer(do_lower_case=lowerCamelCase__			,   never_split=['''[UNK]''']      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )			,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Dict     =		BasicTokenizer()
								lowerCamelCase_					:      List[str]     =		'''a\n\'ll !!to?\'d of, can\'t.'''
								lowerCamelCase_					:      Optional[Any]     =		['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
								self.assertListEqual(tokenizer.tokenize(lowerCamelCase__      )			,   lowerCamelCase__      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Union[str, Any]     =		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								lowerCamelCase_					:      List[Any]     =		{}
								for i, token in enumerate(lowerCamelCase__      ):
													lowerCamelCase_					:      str     =		i
								lowerCamelCase_					:      Union[str, Any]     =		WordpieceTokenizer(vocab=lowerCamelCase__			,   unk_token='''[UNK]'''      )

								self.assertListEqual(tokenizer.tokenize(''''''      )			,   []      )

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )			,   ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )			,   ['''[UNK]''', '''runn''', '''##ing''']      )

			def        UpperCAmelCase__ (self      ):
								self.assertTrue(_is_whitespace(''' '''      )      )
								self.assertTrue(_is_whitespace('''\t'''      )      )
								self.assertTrue(_is_whitespace('''\r'''      )      )
								self.assertTrue(_is_whitespace('''\n'''      )      )
								self.assertTrue(_is_whitespace('''\u00A0'''      )      )

								self.assertFalse(_is_whitespace('''A'''      )      )
								self.assertFalse(_is_whitespace('''-'''      )      )

			def        UpperCAmelCase__ (self      ):
								self.assertTrue(_is_control('''\u0005'''      )      )

								self.assertFalse(_is_control('''A'''      )      )
								self.assertFalse(_is_control(''' '''      )      )
								self.assertFalse(_is_control('''\t'''      )      )
								self.assertFalse(_is_control('''\r'''      )      )

			def        UpperCAmelCase__ (self      ):
								self.assertTrue(_is_punctuation('''-'''      )      )
								self.assertTrue(_is_punctuation('''$'''      )      )
								self.assertTrue(_is_punctuation('''`'''      )      )
								self.assertTrue(_is_punctuation('''.'''      )      )

								self.assertFalse(_is_punctuation('''A'''      )      )
								self.assertFalse(_is_punctuation(''' '''      )      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[Any]     =		self.get_tokenizer()
								lowerCamelCase_					:      Optional[Any]     =		self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(lowerCamelCase__      ) for t in ['''Test''', '''\xad''', '''test''']]			,   [['''[UNK]'''], [], ['''[UNK]''']]      )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(lowerCamelCase__      ) for t in ['''Test''', '''\xad''', '''test''']]			,   [['''[UNK]'''], [], ['''[UNK]''']]      )

			@slow
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      List[Any]     =		self.tokenizer_class.from_pretrained('''bert-base-uncased'''      )

								lowerCamelCase_					:      List[str]     =		tokenizer.encode('''sequence builders'''			,   add_special_tokens=lowerCamelCase__      )
								lowerCamelCase_					:      str     =		tokenizer.encode('''multi-sequence build'''			,   add_special_tokens=lowerCamelCase__      )

								lowerCamelCase_					:      Union[str, Any]     =		tokenizer.build_inputs_with_special_tokens(lowerCamelCase__      )
								lowerCamelCase_					:      Optional[Any]     =		tokenizer.build_inputs_with_special_tokens(lowerCamelCase__			,   lowerCamelCase__      )

								assert encoded_sentence == [1_0_1] + text + [1_0_2]
								assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

			def        UpperCAmelCase__ (self      ):






								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																		lowerCamelCase_					:      int     =		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__			,   **lowerCamelCase__      )

																		lowerCamelCase_					:      Dict     =		F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																		lowerCamelCase_					:      Union[str, Any]     =		tokenizer_r.encode_plus(
																		    lowerCamelCase__			,   return_attention_mask=lowerCamelCase__			,   return_token_type_ids=lowerCamelCase__			,   return_offsets_mapping=lowerCamelCase__			,   add_special_tokens=lowerCamelCase__			,   )

																		lowerCamelCase_					:      List[Any]     =		tokenizer_r.do_lower_case if hasattr(lowerCamelCase__			,   '''do_lower_case'''      ) else False
																		lowerCamelCase_					:      Optional[int]     =		(
																		    [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), '''A'''),
																		        ((1, 2), ''','''),
																		        ((3, 5), '''na'''),
																		        ((5, 6), '''##ï'''),
																		        ((6, 8), '''##ve'''),
																		        ((9, 1_5), tokenizer_r.mask_token),
																		        ((1_6, 2_1), '''Allen'''),
																		        ((2_1, 2_3), '''##NL'''),
																		        ((2_3, 2_4), '''##P'''),
																		        ((2_5, 3_3), '''sentence'''),
																		        ((3_3, 3_4), '''.'''),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		    if not do_lower_case
																		    else [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), '''a'''),
																		        ((1, 2), ''','''),
																		        ((3, 8), '''naive'''),
																		        ((9, 1_5), tokenizer_r.mask_token),
																		        ((1_6, 2_1), '''allen'''),
																		        ((2_1, 2_3), '''##nl'''),
																		        ((2_3, 2_4), '''##p'''),
																		        ((2_5, 3_3), '''sentence'''),
																		        ((3_3, 3_4), '''.'''),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		)

																		self.assertEqual(
																		    [e[1] for e in expected_results]			,   tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
																		self.assertEqual([e[0] for e in expected_results]			,   tokens['''offset_mapping''']      )






			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Union[str, Any]     =		['''的''', '''人''', '''有''']
								lowerCamelCase_					:      Optional[int]     =		''''''.join(lowerCamelCase__      )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																		lowerCamelCase_					:      Dict     =		True
																		lowerCamelCase_					:      Any     =		self.tokenizer_class.from_pretrained(lowerCamelCase__			,   **lowerCamelCase__      )
																		lowerCamelCase_					:      Optional[int]     =		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__			,   **lowerCamelCase__      )

																		lowerCamelCase_					:      Optional[int]     =		tokenizer_p.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
																		lowerCamelCase_					:      List[str]     =		tokenizer_r.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )

																		lowerCamelCase_					:      str     =		tokenizer_r.convert_ids_to_tokens(lowerCamelCase__      )
																		lowerCamelCase_					:      str     =		tokenizer_p.convert_ids_to_tokens(lowerCamelCase__      )

																		# it is expected that each Chinese character is not preceded by "##"
																		self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )
																		self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )

																		lowerCamelCase_					:      List[Any]     =		False
																		lowerCamelCase_					:      Dict     =		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__			,   **lowerCamelCase__      )
																		lowerCamelCase_					:      Optional[Any]     =		self.tokenizer_class.from_pretrained(lowerCamelCase__			,   **lowerCamelCase__      )

																		lowerCamelCase_					:      Dict     =		tokenizer_r.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )
																		lowerCamelCase_					:      Optional[Any]     =		tokenizer_p.encode(lowerCamelCase__			,   add_special_tokens=lowerCamelCase__      )

																		lowerCamelCase_					:      List[str]     =		tokenizer_r.convert_ids_to_tokens(lowerCamelCase__      )
																		lowerCamelCase_					:      Tuple     =		tokenizer_p.convert_ids_to_tokens(lowerCamelCase__      )

																		# it is expected that only the first Chinese character is not preceded by "##".
																		lowerCamelCase_					:      Union[str, Any]     =		[
																		    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__      )
																		]
																		self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )
																		self.assertListEqual(lowerCamelCase__			,   lowerCamelCase__      )





'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__lowercase							:   List[Any]					=     {}


try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:   List[Any]					=     ['''MLukeTokenizer''']

if TYPE_CHECKING:
					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_mluke import MLukeTokenizer


else:
					import sys

					__lowercase							:   List[str]					=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


