






'''simple docstring'''


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    _SCREAMING_SNAKE_CASE							(		_UpperCamelCase       ,unittest.TestCase	):
			__a							:							Any					     =					BertTokenizer
			__a							:							Union[str, Any]					     =					BertTokenizerFast
			__a							:							str					     =					True
			__a							:							int					     =					True
			__a							:							Tuple					     =					filter_non_english






			def 				A      (   self							:						Union[str, Any]			)							->    str:

									'''simple docstring'''







									super().setUp()

									UpperCamelCase__         = [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									UpperCamelCase__         = os.path.join(self.tmpdirname ,      VOCAB_FILES_NAMES["""vocab_file"""]			)
									with open(self.vocab_file ,      """w""" ,      encoding="""utf-8"""			) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)






			def 				A      (   self							:						Any ,      lowercase							:						Union[str, Any]			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = '''UNwant\u00E9d,running'''
									UpperCamelCase__         = '''unwanted, running'''
									return input_text, output_text






			def 				A      (   self							:						Optional[Any]			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = self.tokenizer_class(self.vocab_file			)

									UpperCamelCase__         = tokenizer.tokenize("""UNwant\u00E9d,running"""			)
									self.assertListEqual(_UpperCAmelCase ,      ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase			) ,      [9, 6, 7, 1_2, 1_0, 1_1]			)






			def 				A      (   self							:						Tuple			)							->    Optional[Any]:

									'''simple docstring'''







									if not self.test_rust_tokenizer:
															return

									UpperCamelCase__         = self.get_tokenizer()
									UpperCamelCase__         = self.get_rust_tokenizer()

									UpperCamelCase__         = '''UNwant\u00E9d,running'''

									UpperCamelCase__         = tokenizer.tokenize(_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.tokenize(_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

									UpperCamelCase__         = tokenizer.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

									UpperCamelCase__         = self.get_rust_tokenizer()
									UpperCamelCase__         = tokenizer.encode(_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.encode(_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

									# With lower casing
									UpperCamelCase__         = self.get_tokenizer(do_lower_case=_UpperCAmelCase			)
									UpperCamelCase__         = self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase			)

									UpperCamelCase__         = '''UNwant\u00E9d,running'''

									UpperCamelCase__         = tokenizer.tokenize(_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.tokenize(_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

									UpperCamelCase__         = tokenizer.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

									UpperCamelCase__         = self.get_rust_tokenizer()
									UpperCamelCase__         = tokenizer.encode(_UpperCAmelCase			)
									UpperCamelCase__         = rust_tokenizer.encode(_UpperCAmelCase			)
									self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)






			def 				A      (   self							:						Any			)							->    Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			) ,      ["""ah""", """\u535A""", """\u63A8""", """zz"""]			)






			def 				A      (   self							:						Optional[Any]			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			) ,      ["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			) ,      ["""hello"""]			)






			def 				A      (   self							:						Optional[Any]			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase ,      strip_accents=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			) ,      ["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			) ,      ["""h\u00E9llo"""]			)






			def 				A      (   self							:						Union[str, Any]			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase ,      strip_accents=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			) ,      ["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			) ,      ["""hello"""]			)






			def 				A      (   self							:						Dict			)							->    Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			) ,      ["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			) ,      ["""hello"""]			)






			def 				A      (   self							:						Any			)							->    List[Any]:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			) ,      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 				A      (   self							:						Tuple			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase ,      strip_accents=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			) ,      ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 				A      (   self							:						str			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase ,      strip_accents=_UpperCAmelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			) ,      ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 				A      (   self							:						Dict			)							->    Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer(do_lower_case=_UpperCAmelCase ,      never_split=["""[UNK]"""]			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			) ,      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)






			def 				A      (   self							:						Dict			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = BasicTokenizer()
									UpperCamelCase__         = '''a\n\'ll !!to?\'d of, can\'t.'''
									UpperCamelCase__         = ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
									self.assertListEqual(tokenizer.tokenize(_UpperCAmelCase			) ,      _UpperCAmelCase			)






			def 				A      (   self							:						Any			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									UpperCamelCase__         = {}
									for i, token in enumerate(_UpperCAmelCase			):
															UpperCamelCase__         = i
									UpperCamelCase__         = WordpieceTokenizer(vocab=_UpperCAmelCase ,      unk_token="""[UNK]"""			)

									self.assertListEqual(tokenizer.tokenize(""""""			) ,      []			)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""			) ,      ["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			) ,      ["""[UNK]""", """runn""", """##ing"""]			)






			def 				A      (   self							:						Any			)							->    List[Any]:

									'''simple docstring'''







									self.assertTrue(_is_whitespace(""" """			)			)
									self.assertTrue(_is_whitespace("""\t"""			)			)
									self.assertTrue(_is_whitespace("""\r"""			)			)
									self.assertTrue(_is_whitespace("""\n"""			)			)
									self.assertTrue(_is_whitespace("""\u00A0"""			)			)

									self.assertFalse(_is_whitespace("""A"""			)			)
									self.assertFalse(_is_whitespace("""-"""			)			)






			def 				A      (   self							:						Union[str, Any]			)							->    List[str]:

									'''simple docstring'''







									self.assertTrue(_is_control("""\u0005"""			)			)

									self.assertFalse(_is_control("""A"""			)			)
									self.assertFalse(_is_control(""" """			)			)
									self.assertFalse(_is_control("""\t"""			)			)
									self.assertFalse(_is_control("""\r"""			)			)






			def 				A      (   self							:						List[str]			)							->    Optional[int]:

									'''simple docstring'''







									self.assertTrue(_is_punctuation("""-"""			)			)
									self.assertTrue(_is_punctuation("""$"""			)			)
									self.assertTrue(_is_punctuation("""`"""			)			)
									self.assertTrue(_is_punctuation("""."""			)			)

									self.assertFalse(_is_punctuation("""A"""			)			)
									self.assertFalse(_is_punctuation(""" """			)			)






			def 				A      (   self							:						Any			)							->    Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.get_tokenizer()
									UpperCamelCase__         = self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase			) for t in ["""Test""", """\xad""", """test"""]] ,      [["""[UNK]"""], [], ["""[UNK]"""]]			)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(_UpperCAmelCase			) for t in ["""Test""", """\xad""", """test"""]] ,      [["""[UNK]"""], [], ["""[UNK]"""]]			)






			@slow
			def 				A      (   self							:						Dict			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = self.tokenizer_class.from_pretrained("""bert-base-uncased"""			)

									UpperCamelCase__         = tokenizer.encode("""sequence builders""" ,      add_special_tokens=_UpperCAmelCase			)
									UpperCamelCase__         = tokenizer.encode("""multi-sequence build""" ,      add_special_tokens=_UpperCAmelCase			)

									UpperCamelCase__         = tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase			)
									UpperCamelCase__         = tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase ,      _UpperCAmelCase			)

									assert encoded_sentence == [1_0_1] + text + [1_0_2]
									assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






			def 				A      (   self							:						Optional[int]			)							->    Union[str, Any]:

									'''simple docstring'''







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
																					UpperCamelCase__         = self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase ,      **_UpperCAmelCase			)

																					UpperCamelCase__         = f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																					UpperCamelCase__         = tokenizer_r.encode_plus(
																					    _UpperCAmelCase ,      return_attention_mask=_UpperCAmelCase ,      return_token_type_ids=_UpperCAmelCase ,      return_offsets_mapping=_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase ,      )

																					UpperCamelCase__         = tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase ,      """do_lower_case"""			) else False
																					UpperCamelCase__         = (
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), '''A'''),
																					        ((1, 2), ''','''),
																					        ((3, 5), '''na'''),
																					        ((5, 6), '''##ï'''),
																					        ((6, 8), '''##ve'''),
																					        ((9, 1_5), tokenizer_r.mask_token),
																					        ((1_6, 2_1), '''Allen'''),
																					        ((2_1, 2_3), '''##NL'''),
																					        ((2_3, 2_4), '''##P'''),
																					        ((2_5, 3_3), '''sentence'''),
																					        ((3_3, 3_4), '''.'''),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), '''a'''),
																					        ((1, 2), ''','''),
																					        ((3, 8), '''naive'''),
																					        ((9, 1_5), tokenizer_r.mask_token),
																					        ((1_6, 2_1), '''allen'''),
																					        ((2_1, 2_3), '''##nl'''),
																					        ((2_3, 2_4), '''##p'''),
																					        ((2_5, 3_3), '''sentence'''),
																					        ((3_3, 3_4), '''.'''),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results] ,      tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]			)			)
																					self.assertEqual([e[0] for e in expected_results] ,      tokens["""offset_mapping"""]			)







			def 				A      (   self							:						int			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = ['''的''', '''人''', '''有''']
									UpperCamelCase__         = ''''''.join(_UpperCAmelCase			)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
																					UpperCamelCase__         = True
																					UpperCamelCase__         = self.tokenizer_class.from_pretrained(_UpperCAmelCase ,      **_UpperCAmelCase			)
																					UpperCamelCase__         = self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase ,      **_UpperCAmelCase			)

																					UpperCamelCase__         = tokenizer_p.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
																					UpperCamelCase__         = tokenizer_r.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)

																					UpperCamelCase__         = tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase			)
																					UpperCamelCase__         = tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase			)

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)
																					self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)

																					UpperCamelCase__         = False
																					UpperCamelCase__         = self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase ,      **_UpperCAmelCase			)
																					UpperCamelCase__         = self.tokenizer_class.from_pretrained(_UpperCAmelCase ,      **_UpperCAmelCase			)

																					UpperCamelCase__         = tokenizer_r.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)
																					UpperCamelCase__         = tokenizer_p.encode(_UpperCAmelCase ,      add_special_tokens=_UpperCAmelCase			)

																					UpperCamelCase__         = tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase			)
																					UpperCamelCase__         = tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase			)

																					# it is expected that only the first Chinese character is not preceded by "##".
																					UpperCamelCase__         = [
																					    f"##{token}" if idx != 0 else token for idx, token in enumerate(_UpperCAmelCase			)
																					]
																					self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)
																					self.assertListEqual(_UpperCAmelCase ,      _UpperCAmelCase			)








'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase_				: Tuple    	=							{
    '''configuration_layoutlmv3''': [
        '''LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LayoutLMv3Config''',
        '''LayoutLMv3OnnxConfig''',
    ],
    '''processing_layoutlmv3''': ['''LayoutLMv3Processor'''],
    '''tokenization_layoutlmv3''': ['''LayoutLMv3Tokenizer'''],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCamelCase_				: Tuple    	=							['''LayoutLMv3TokenizerFast''']

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCamelCase_				: Union[str, Any]    	=							[
					    '''LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''LayoutLMv3ForQuestionAnswering''',
					    '''LayoutLMv3ForSequenceClassification''',
					    '''LayoutLMv3ForTokenClassification''',
					    '''LayoutLMv3Model''',
					    '''LayoutLMv3PreTrainedModel''',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCamelCase_				: Optional[Any]    	=							[
					    '''TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''TFLayoutLMv3ForQuestionAnswering''',
					    '''TFLayoutLMv3ForSequenceClassification''',
					    '''TFLayoutLMv3ForTokenClassification''',
					    '''TFLayoutLMv3Model''',
					    '''TFLayoutLMv3PreTrainedModel''',
					]

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCamelCase_				: Optional[Any]    	=							['''LayoutLMv3FeatureExtractor''']
					lowerCamelCase_				: Dict    	=							['''LayoutLMv3ImageProcessor''']


if TYPE_CHECKING:
					from .configuration_layoutlmva import (
					    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    LayoutLMvaConfig,
					    LayoutLMvaOnnxConfig,
					)
					from .processing_layoutlmva import LayoutLMvaProcessor
					from .tokenization_layoutlmva import LayoutLMvaTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_layoutlmva import (
										    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
										    LayoutLMvaForQuestionAnswering,
										    LayoutLMvaForSequenceClassification,
										    LayoutLMvaForTokenClassification,
										    LayoutLMvaModel,
										    LayoutLMvaPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_layoutlmva import (
										    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFLayoutLMvaForQuestionAnswering,
										    TFLayoutLMvaForSequenceClassification,
										    TFLayoutLMvaForTokenClassification,
										    TFLayoutLMvaModel,
										    TFLayoutLMvaPreTrainedModel,
										)

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
										from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
					import sys

					lowerCamelCase_				: Optional[Any]    	=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
