






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 						UpperCAmelCase__ (							__snake_case     ,				unittest.TestCase							):
		__snake_case      :       Optional[int]     =   BertTokenizer
		__snake_case      :       str     =   BertTokenizerFast
		__snake_case      :       Optional[int]     =   True
		__snake_case      :       Dict     =   True
		__snake_case      :       Tuple     =   filter_non_english







		def  A__      (					self		):
						super().setUp()

						_A	:				Optional[int]   =      [
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						_A	:				Union[str, Any]   =      os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''vocab_file''']		)
						with open(self.vocab_file       ,'''w'''       ,encoding='''utf-8'''		) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)







		def  A__      (					self       ,A__		):
						_A	:				str   =      '''UNwant\u00E9d,running'''
						_A	:				List[str]   =      '''unwanted, running'''
						return input_text, output_text







		def  A__      (					self		):
						_A	:				int   =      self.tokenizer_class(self.vocab_file		)

						_A	:				str   =      tokenizer.tokenize('''UNwant\u00E9d,running'''		)
						self.assertListEqual(A__       ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__		)       ,[9, 6, 7, 12, 10, 11]		)







		def  A__      (					self		):
						if not self.test_rust_tokenizer:
										return

						_A	:				Optional[int]   =      self.get_tokenizer()
						_A	:				Optional[Any]   =      self.get_rust_tokenizer()

						_A	:				Optional[int]   =      '''UNwant\u00E9d,running'''

						_A	:				List[Any]   =      tokenizer.tokenize(A__		)
						_A	:				Tuple   =      rust_tokenizer.tokenize(A__		)
						self.assertListEqual(A__       ,A__		)

						_A	:				Dict   =      tokenizer.encode(A__       ,add_special_tokens=A__		)
						_A	:				Tuple   =      rust_tokenizer.encode(A__       ,add_special_tokens=A__		)
						self.assertListEqual(A__       ,A__		)

						_A	:				List[str]   =      self.get_rust_tokenizer()
						_A	:				Dict   =      tokenizer.encode(A__		)
						_A	:				Optional[int]   =      rust_tokenizer.encode(A__		)
						self.assertListEqual(A__       ,A__		)

						# With lower casing
						_A	:				str   =      self.get_tokenizer(do_lower_case=A__		)
						_A	:				int   =      self.get_rust_tokenizer(do_lower_case=A__		)

						_A	:				List[str]   =      '''UNwant\u00E9d,running'''

						_A	:				Optional[Any]   =      tokenizer.tokenize(A__		)
						_A	:				int   =      rust_tokenizer.tokenize(A__		)
						self.assertListEqual(A__       ,A__		)

						_A	:				int   =      tokenizer.encode(A__       ,add_special_tokens=A__		)
						_A	:				List[Any]   =      rust_tokenizer.encode(A__       ,add_special_tokens=A__		)
						self.assertListEqual(A__       ,A__		)

						_A	:				Tuple   =      self.get_rust_tokenizer()
						_A	:				Dict   =      tokenizer.encode(A__		)
						_A	:				Tuple   =      rust_tokenizer.encode(A__		)
						self.assertListEqual(A__       ,A__		)







		def  A__      (					self		):
						_A	:				int   =      BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''		)       ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']		)







		def  A__      (					self		):
						_A	:				List[str]   =      BasicTokenizer(do_lower_case=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)       ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)       ,['''hello''']		)







		def  A__      (					self		):
						_A	:				int   =      BasicTokenizer(do_lower_case=A__       ,strip_accents=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)       ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)       ,['''h\u00E9llo''']		)







		def  A__      (					self		):
						_A	:				Dict   =      BasicTokenizer(do_lower_case=A__       ,strip_accents=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)       ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)       ,['''hello''']		)







		def  A__      (					self		):
						_A	:				int   =      BasicTokenizer(do_lower_case=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)       ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)       ,['''hello''']		)







		def  A__      (					self		):
						_A	:				int   =      BasicTokenizer(do_lower_case=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)       ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







		def  A__      (					self		):
						_A	:				str   =      BasicTokenizer(do_lower_case=A__       ,strip_accents=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)       ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







		def  A__      (					self		):
						_A	:				str   =      BasicTokenizer(do_lower_case=A__       ,strip_accents=A__		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)       ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







		def  A__      (					self		):
						_A	:				Dict   =      BasicTokenizer(do_lower_case=A__       ,never_split=['''[UNK]''']		)

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''		)       ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']		)







		def  A__      (					self		):
						_A	:				str   =      BasicTokenizer()
						_A	:				int   =      '''a\n\'ll !!to?\'d of, can\'t.'''
						_A	:				Any   =      ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
						self.assertListEqual(tokenizer.tokenize(A__		)       ,A__		)







		def  A__      (					self		):
						_A	:				Tuple   =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						_A	:				List[Any]   =      {}
						for i, token in enumerate(A__		):
										_A	:				Optional[Any]   =      i
						_A	:				Tuple   =      WordpieceTokenizer(vocab=A__       ,unk_token='''[UNK]'''		)

						self.assertListEqual(tokenizer.tokenize(''''''		)       ,[]		)

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''		)       ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']		)

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''		)       ,['''[UNK]''', '''runn''', '''##ing''']		)







		def  A__      (					self		):
						self.assertTrue(_is_whitespace(''' '''		)		)
						self.assertTrue(_is_whitespace('''\t'''		)		)
						self.assertTrue(_is_whitespace('''\r'''		)		)
						self.assertTrue(_is_whitespace('''\n'''		)		)
						self.assertTrue(_is_whitespace('''\u00A0'''		)		)

						self.assertFalse(_is_whitespace('''A'''		)		)
						self.assertFalse(_is_whitespace('''-'''		)		)







		def  A__      (					self		):
						self.assertTrue(_is_control('''\u0005'''		)		)

						self.assertFalse(_is_control('''A'''		)		)
						self.assertFalse(_is_control(''' '''		)		)
						self.assertFalse(_is_control('''\t'''		)		)
						self.assertFalse(_is_control('''\r'''		)		)







		def  A__      (					self		):
						self.assertTrue(_is_punctuation('''-'''		)		)
						self.assertTrue(_is_punctuation('''$'''		)		)
						self.assertTrue(_is_punctuation('''`'''		)		)
						self.assertTrue(_is_punctuation('''.'''		)		)

						self.assertFalse(_is_punctuation('''A'''		)		)
						self.assertFalse(_is_punctuation(''' '''		)		)







		def  A__      (					self		):
						_A	:				Tuple   =      self.get_tokenizer()
						_A	:				List[Any]   =      self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__		) for t in ['''Test''', '''\xad''', '''test''']]       ,[['''[UNK]'''], [], ['''[UNK]''']]		)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(A__		) for t in ['''Test''', '''\xad''', '''test''']]       ,[['''[UNK]'''], [], ['''[UNK]''']]		)







		@slow
		def  A__      (					self		):
						_A	:				List[str]   =      self.tokenizer_class.from_pretrained('''bert-base-uncased'''		)

						_A	:				Optional[int]   =      tokenizer.encode('''sequence builders'''       ,add_special_tokens=A__		)
						_A	:				Tuple   =      tokenizer.encode('''multi-sequence build'''       ,add_special_tokens=A__		)

						_A	:				int   =      tokenizer.build_inputs_with_special_tokens(A__		)
						_A	:				List[Any]   =      tokenizer.build_inputs_with_special_tokens(A__       ,A__		)

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]







		def  A__      (					self		):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[Any]   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				Dict   =      f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														_A	:				Tuple   =      tokenizer_r.encode_plus(
														    A__       ,return_attention_mask=A__       ,return_token_type_ids=A__       ,return_offsets_mapping=A__       ,add_special_tokens=A__       ,)

														_A	:				List[Any]   =      tokenizer_r.do_lower_case if hasattr(A__       ,'''do_lower_case'''		) else False
														_A	:				Tuple   =      (
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''Allen'''),
														        ((21, 23), '''##NL'''),
														        ((23, 24), '''##P'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''allen'''),
														        ((21, 23), '''##nl'''),
														        ((23, 24), '''##p'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]       ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']		)		)
														self.assertEqual([e[0] for e in expected_results]       ,tokens['''offset_mapping''']		)







		def  A__      (					self		):
						_A	:				Dict   =      ['''的''', '''人''', '''有''']
						_A	:				int   =      ''''''.join(A__		)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				str   =      True
														_A	:				Union[str, Any]   =      self.tokenizer_class.from_pretrained(A__       ,**A__		)
														_A	:				int   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				List[Any]   =      tokenizer_p.encode(A__       ,add_special_tokens=A__		)
														_A	:				str   =      tokenizer_r.encode(A__       ,add_special_tokens=A__		)

														_A	:				Union[str, Any]   =      tokenizer_r.convert_ids_to_tokens(A__		)
														_A	:				Optional[int]   =      tokenizer_p.convert_ids_to_tokens(A__		)

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(A__       ,A__		)
														self.assertListEqual(A__       ,A__		)

														_A	:				Optional[Any]   =      False
														_A	:				List[Any]   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)
														_A	:				List[str]   =      self.tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				Union[str, Any]   =      tokenizer_r.encode(A__       ,add_special_tokens=A__		)
														_A	:				Tuple   =      tokenizer_p.encode(A__       ,add_special_tokens=A__		)

														_A	:				str   =      tokenizer_r.convert_ids_to_tokens(A__		)
														_A	:				Any   =      tokenizer_p.convert_ids_to_tokens(A__		)

														# it is expected that only the first Chinese character is not preceded by "##".
														_A	:				Any   =      [
														    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__		)
														]
														self.assertListEqual(A__       ,A__		)
														self.assertListEqual(A__       ,A__		)











from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       Optional[Any]     =   ["image_processor", "tokenizer"]
		__snake_case      :       Tuple     =   "BridgeTowerImageProcessor"
		__snake_case      :       List[str]     =   ("RobertaTokenizer", "RobertaTokenizerFast")







		def __init__(					self       ,A__       ,A__		):
						super().__init__(A__       ,A__		)







		def __call__(					self       ,A__       ,A__ = None       ,A__ = True       ,A__ = False       ,A__ = None       ,A__ = None       ,A__ = 0       ,A__ = None       ,A__ = None       ,A__ = None       ,A__ = False       ,A__ = False       ,A__ = False       ,A__ = False       ,A__ = True       ,A__ = None       ,**A__       ,):
						_A	:				List[Any]   =      self.tokenizer(
						    text=A__       ,add_special_tokens=A__       ,padding=A__       ,truncation=A__       ,max_length=A__       ,stride=A__       ,pad_to_multiple_of=A__       ,return_token_type_ids=A__       ,return_attention_mask=A__       ,return_overflowing_tokens=A__       ,return_special_tokens_mask=A__       ,return_offsets_mapping=A__       ,return_length=A__       ,verbose=A__       ,return_tensors=A__       ,**A__       ,)
						# add pixel_values + pixel_mask
						_A	:				Optional[Any]   =      self.image_processor(
						    A__       ,return_tensors=A__       ,do_normalize=A__       ,do_center_crop=A__       ,**A__		)
						encoding.update(A__		)

						return encoding







		def  A__      (					self       ,*A__       ,**A__		):
						return self.tokenizer.batch_decode(*A__       ,**A__		)







		def  A__      (					self       ,*A__       ,**A__		):
						return self.tokenizer.decode(*A__       ,**A__		)







		@property
		def  A__      (					self		):
						_A	:				Union[str, Any]   =      self.tokenizer.model_input_names
						_A	:				str   =      self.image_processor.model_input_names
						return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names		)		)



