
"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class        _UpperCAmelCase(       lowerCamelCase ,	unittest.TestCase      ):
		lowercase__         = MobileBertTokenizer
		lowercase__         = MobileBertTokenizerFast
		lowercase__         = True
		lowercase__         = True
		lowercase__         = filter_non_english
		lowercase__         = 'google/mobilebert-uncased'



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							super().setUp()

							_UpperCamelCase									=    [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							_UpperCamelCase									=    os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file'''])
							with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8''') as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))

							_UpperCamelCase									=    [
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							_UpperCamelCase									=    '''UNwant\u00E9d,running'''
							_UpperCamelCase									=    '''unwanted, running'''
							return input_text, output_text



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.tokenizer_class(self.vocab_file)

							_UpperCamelCase									=    tokenizer.tokenize('''UNwant\u00E9d,running''')
							self.assertListEqual(__a							,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)							,						[9, 6, 7, 12, 10, 11])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							if not self.test_rust_tokenizer:
												return

							_UpperCamelCase									=    self.get_tokenizer()
							_UpperCamelCase									=    self.get_rust_tokenizer()

							_UpperCamelCase									=    '''UNwant\u00E9d,running'''

							_UpperCamelCase									=    tokenizer.tokenize(__a)
							_UpperCamelCase									=    rust_tokenizer.tokenize(__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    tokenizer.encode(__a							,						add_special_tokens=__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a							,						add_special_tokens=__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    self.get_rust_tokenizer()
							_UpperCamelCase									=    tokenizer.encode(__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a)
							self.assertListEqual(__a							,						__a)

							# With lower casing
							_UpperCamelCase									=    self.get_tokenizer(do_lower_case=__a)
							_UpperCamelCase									=    self.get_rust_tokenizer(do_lower_case=__a)

							_UpperCamelCase									=    '''UNwant\u00E9d,running'''

							_UpperCamelCase									=    tokenizer.tokenize(__a)
							_UpperCamelCase									=    rust_tokenizer.tokenize(__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    tokenizer.encode(__a							,						add_special_tokens=__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a							,						add_special_tokens=__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    self.get_rust_tokenizer()
							_UpperCamelCase									=    tokenizer.encode(__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a)
							self.assertListEqual(__a							,						__a)



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')							,						['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''h\u00E9llo'''])



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						never_split=['''[UNK]'''])

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							_UpperCamelCase									=    {}
							for i, token in enumerate(__a):
												_UpperCamelCase									=    i
							_UpperCamelCase									=    WordpieceTokenizer(vocab=__a							,						unk_token='''[UNK]''')

							self.assertListEqual(tokenizer.tokenize('''''')							,						[])

							self.assertListEqual(tokenizer.tokenize('''unwanted running''')							,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

							self.assertListEqual(tokenizer.tokenize('''unwantedX running''')							,						['''[UNK]''', '''runn''', '''##ing'''])



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							self.assertTrue(_is_whitespace(''' '''))
							self.assertTrue(_is_whitespace('''\t'''))
							self.assertTrue(_is_whitespace('''\r'''))
							self.assertTrue(_is_whitespace('''\n'''))
							self.assertTrue(_is_whitespace('''\u00A0'''))

							self.assertFalse(_is_whitespace('''A'''))
							self.assertFalse(_is_whitespace('''-'''))



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							self.assertTrue(_is_control('''\u0005'''))

							self.assertFalse(_is_control('''A'''))
							self.assertFalse(_is_control(''' '''))
							self.assertFalse(_is_control('''\t'''))
							self.assertFalse(_is_control('''\r'''))



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							self.assertTrue(_is_punctuation('''-'''))
							self.assertTrue(_is_punctuation('''$'''))
							self.assertTrue(_is_punctuation('''`'''))
							self.assertTrue(_is_punctuation('''.'''))

							self.assertFalse(_is_punctuation('''A'''))
							self.assertFalse(_is_punctuation(''' '''))



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.get_tokenizer()
							_UpperCamelCase									=    self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__a) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']])

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__a) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']])



		@slow
		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''')

							_UpperCamelCase									=    tokenizer.encode('''sequence builders'''							,						add_special_tokens=__a)
							_UpperCamelCase									=    tokenizer.encode('''multi-sequence build'''							,						add_special_tokens=__a)

							_UpperCamelCase									=    tokenizer.build_inputs_with_special_tokens(__a)
							_UpperCamelCase									=    tokenizer.build_inputs_with_special_tokens(__a							,						__a)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	_UpperCamelCase									=    tokenizer_r.encode_plus(
																	    __a							,						return_attention_mask=__a							,						return_token_type_ids=__a							,						return_offsets_mapping=__a							,						add_special_tokens=__a							,						)

																	_UpperCamelCase									=    tokenizer_r.do_lower_case if hasattr(__a							,						'''do_lower_case''') else False
																	_UpperCamelCase									=    (
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]							,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
																	self.assertEqual([e[0] for e in expected_results]							,						tokens['''offset_mapping'''])




		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    ['''的''', '''人''', '''有''']
							_UpperCamelCase									=    ''''''.join(__a)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
																	_UpperCamelCase									=    True
																	_UpperCamelCase									=    self.tokenizer_class.from_pretrained(__a							,						**__a)
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    tokenizer_p.encode(__a							,						add_special_tokens=__a)
																	_UpperCamelCase									=    tokenizer_r.encode(__a							,						add_special_tokens=__a)

																	_UpperCamelCase									=    tokenizer_r.convert_ids_to_tokens(__a)
																	_UpperCamelCase									=    tokenizer_p.convert_ids_to_tokens(__a)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(__a							,						__a)
																	self.assertListEqual(__a							,						__a)

																	_UpperCamelCase									=    False
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)
																	_UpperCamelCase									=    self.tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    tokenizer_r.encode(__a							,						add_special_tokens=__a)
																	_UpperCamelCase									=    tokenizer_p.encode(__a							,						add_special_tokens=__a)

																	_UpperCamelCase									=    tokenizer_r.convert_ids_to_tokens(__a)
																	_UpperCamelCase									=    tokenizer_p.convert_ids_to_tokens(__a)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	_UpperCamelCase									=    [
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a)
																	]
																	self.assertListEqual(__a							,						__a)
																	self.assertListEqual(__a							,						__a)








"""simple docstring"""




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_A     :			Optional[int]  						=			{
    """configuration_mgp_str""": ["""MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MgpstrConfig"""],
    """processing_mgp_str""": ["""MgpstrProcessor"""],
    """tokenization_mgp_str""": ["""MgpstrTokenizer"""],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A     :			Any  						=			[
       """MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """MgpstrModel""",
       """MgpstrPreTrainedModel""",
       """MgpstrForSceneTextRecognition""",
   ]

if TYPE_CHECKING:
   from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig
   from .processing_mgp_str import MgpstrProcessor
   from .tokenization_mgp_str import MgpstrTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mgp_str import (
          MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST,
          MgpstrForSceneTextRecognition,
          MgpstrModel,
          MgpstrPreTrainedModel,
      )
else:
   import sys

   _A     :			Tuple  						=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


