
"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class        _UpperCAmelCase(       lowerCamelCase ,	unittest.TestCase      ):
		lowercase__         = MobileBertTokenizer
		lowercase__         = MobileBertTokenizerFast
		lowercase__         = True
		lowercase__         = True
		lowercase__         = filter_non_english
		lowercase__         = 'google/mobilebert-uncased'



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							super().setUp()

							_UpperCamelCase									=    [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							_UpperCamelCase									=    os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file'''])
							with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8''') as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))

							_UpperCamelCase									=    [
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							_UpperCamelCase									=    '''UNwant\u00E9d,running'''
							_UpperCamelCase									=    '''unwanted, running'''
							return input_text, output_text



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.tokenizer_class(self.vocab_file)

							_UpperCamelCase									=    tokenizer.tokenize('''UNwant\u00E9d,running''')
							self.assertListEqual(__a							,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)							,						[9, 6, 7, 12, 10, 11])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							if not self.test_rust_tokenizer:
												return

							_UpperCamelCase									=    self.get_tokenizer()
							_UpperCamelCase									=    self.get_rust_tokenizer()

							_UpperCamelCase									=    '''UNwant\u00E9d,running'''

							_UpperCamelCase									=    tokenizer.tokenize(__a)
							_UpperCamelCase									=    rust_tokenizer.tokenize(__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    tokenizer.encode(__a							,						add_special_tokens=__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a							,						add_special_tokens=__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    self.get_rust_tokenizer()
							_UpperCamelCase									=    tokenizer.encode(__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a)
							self.assertListEqual(__a							,						__a)

							# With lower casing
							_UpperCamelCase									=    self.get_tokenizer(do_lower_case=__a)
							_UpperCamelCase									=    self.get_rust_tokenizer(do_lower_case=__a)

							_UpperCamelCase									=    '''UNwant\u00E9d,running'''

							_UpperCamelCase									=    tokenizer.tokenize(__a)
							_UpperCamelCase									=    rust_tokenizer.tokenize(__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    tokenizer.encode(__a							,						add_special_tokens=__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a							,						add_special_tokens=__a)
							self.assertListEqual(__a							,						__a)

							_UpperCamelCase									=    self.get_rust_tokenizer()
							_UpperCamelCase									=    tokenizer.encode(__a)
							_UpperCamelCase									=    rust_tokenizer.encode(__a)
							self.assertListEqual(__a							,						__a)



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')							,						['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''h\u00E9llo'''])



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,						['''hello'''])



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						strip_accents=__a)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    BasicTokenizer(do_lower_case=__a							,						never_split=['''[UNK]'''])

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							_UpperCamelCase									=    {}
							for i, token in enumerate(__a):
												_UpperCamelCase									=    i
							_UpperCamelCase									=    WordpieceTokenizer(vocab=__a							,						unk_token='''[UNK]''')

							self.assertListEqual(tokenizer.tokenize('''''')							,						[])

							self.assertListEqual(tokenizer.tokenize('''unwanted running''')							,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

							self.assertListEqual(tokenizer.tokenize('''unwantedX running''')							,						['''[UNK]''', '''runn''', '''##ing'''])



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							self.assertTrue(_is_whitespace(''' '''))
							self.assertTrue(_is_whitespace('''\t'''))
							self.assertTrue(_is_whitespace('''\r'''))
							self.assertTrue(_is_whitespace('''\n'''))
							self.assertTrue(_is_whitespace('''\u00A0'''))

							self.assertFalse(_is_whitespace('''A'''))
							self.assertFalse(_is_whitespace('''-'''))



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							self.assertTrue(_is_control('''\u0005'''))

							self.assertFalse(_is_control('''A'''))
							self.assertFalse(_is_control(''' '''))
							self.assertFalse(_is_control('''\t'''))
							self.assertFalse(_is_control('''\r'''))



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							self.assertTrue(_is_punctuation('''-'''))
							self.assertTrue(_is_punctuation('''$'''))
							self.assertTrue(_is_punctuation('''`'''))
							self.assertTrue(_is_punctuation('''.'''))

							self.assertFalse(_is_punctuation('''A'''))
							self.assertFalse(_is_punctuation(''' '''))



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.get_tokenizer()
							_UpperCamelCase									=    self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__a) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']])

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__a) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']])



		@slow
		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''')

							_UpperCamelCase									=    tokenizer.encode('''sequence builders'''							,						add_special_tokens=__a)
							_UpperCamelCase									=    tokenizer.encode('''multi-sequence build'''							,						add_special_tokens=__a)

							_UpperCamelCase									=    tokenizer.build_inputs_with_special_tokens(__a)
							_UpperCamelCase									=    tokenizer.build_inputs_with_special_tokens(__a							,						__a)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	_UpperCamelCase									=    tokenizer_r.encode_plus(
																	    __a							,						return_attention_mask=__a							,						return_token_type_ids=__a							,						return_offsets_mapping=__a							,						add_special_tokens=__a							,						)

																	_UpperCamelCase									=    tokenizer_r.do_lower_case if hasattr(__a							,						'''do_lower_case''') else False
																	_UpperCamelCase									=    (
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]							,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
																	self.assertEqual([e[0] for e in expected_results]							,						tokens['''offset_mapping'''])




		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    ['''的''', '''人''', '''有''']
							_UpperCamelCase									=    ''''''.join(__a)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
																	_UpperCamelCase									=    True
																	_UpperCamelCase									=    self.tokenizer_class.from_pretrained(__a							,						**__a)
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    tokenizer_p.encode(__a							,						add_special_tokens=__a)
																	_UpperCamelCase									=    tokenizer_r.encode(__a							,						add_special_tokens=__a)

																	_UpperCamelCase									=    tokenizer_r.convert_ids_to_tokens(__a)
																	_UpperCamelCase									=    tokenizer_p.convert_ids_to_tokens(__a)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(__a							,						__a)
																	self.assertListEqual(__a							,						__a)

																	_UpperCamelCase									=    False
																	_UpperCamelCase									=    self.rust_tokenizer_class.from_pretrained(__a							,						**__a)
																	_UpperCamelCase									=    self.tokenizer_class.from_pretrained(__a							,						**__a)

																	_UpperCamelCase									=    tokenizer_r.encode(__a							,						add_special_tokens=__a)
																	_UpperCamelCase									=    tokenizer_p.encode(__a							,						add_special_tokens=__a)

																	_UpperCamelCase									=    tokenizer_r.convert_ids_to_tokens(__a)
																	_UpperCamelCase									=    tokenizer_p.convert_ids_to_tokens(__a)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	_UpperCamelCase									=    [
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a)
																	]
																	self.assertListEqual(__a							,						__a)
																	self.assertListEqual(__a							,						__a)








"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_a								=						logging.get_logger(__name__)






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = ['pixel_values']



		def __init__(   self							,						__a = True							,						__a = None							,						__a = PILImageResampling.BICUBIC							,						__a = True							,						__a = True							,						__a = 1 / 2_55							,						__a = None							,						__a = True							,						__a = None							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							super().__init__(**__a)
							_UpperCamelCase									=    size if size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a)
							_UpperCamelCase									=    crop_size if crop_size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a							,						default_to_square=__a							,						param_name='''crop_size''')

							_UpperCamelCase									=    do_resize
							_UpperCamelCase									=    do_rescale
							_UpperCamelCase									=    do_normalize
							_UpperCamelCase									=    do_center_crop
							_UpperCamelCase									=    crop_size
							_UpperCamelCase									=    size
							_UpperCamelCase									=    resample
							_UpperCamelCase									=    rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
							_UpperCamelCase									=    image_std if image_std is not None else IMAGENET_DEFAULT_STD



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = PILImageResampling.BILINEAR							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)

							if "shortest_edge" in size:
												_UpperCamelCase									=    get_resize_output_image_size(__a							,						size=size['''shortest_edge''']							,						default_to_square=__a)
												# size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
							elif "height" in size and "width" in size:
												_UpperCamelCase									=    (size['''height'''], size['''width'''])
							else:
												raise ValueError(F'''Size must contain \'height\' and \'width\' keys or \'shortest_edge\' key. Got {size.keys()}''')
							return resize(__a							,						size=__a							,						resample=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)
							if "height" not in size or "width" not in size:
												raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}''')
							return center_crop(__a							,						size=(size['''height'''], size['''width'''])							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a)			->			np.ndarray:






							'''simple docstring'''
							return rescale(__a							,						scale=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							return normalize(__a							,						mean=__a							,						std=__a							,						data_format=__a							,						**__a)




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = ChannelDimension.FIRST							,						**__a							,						)			->			BatchFeature:






							'''simple docstring'''
							_UpperCamelCase									=    do_resize if do_resize is not None else self.do_resize
							_UpperCamelCase									=    do_rescale if do_rescale is not None else self.do_rescale
							_UpperCamelCase									=    do_normalize if do_normalize is not None else self.do_normalize
							_UpperCamelCase									=    do_center_crop if do_center_crop is not None else self.do_center_crop
							_UpperCamelCase									=    crop_size if crop_size is not None else self.crop_size
							_UpperCamelCase									=    get_size_dict(__a							,						param_name='''crop_size'''							,						default_to_square=__a)
							_UpperCamelCase									=    resample if resample is not None else self.resample
							_UpperCamelCase									=    rescale_factor if rescale_factor is not None else self.rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else self.image_mean
							_UpperCamelCase									=    image_std if image_std is not None else self.image_std

							_UpperCamelCase									=    size if size is not None else self.size
							_UpperCamelCase									=    get_size_dict(__a)

							if not is_batched(__a):
												_UpperCamelCase									=    [images]

							if not valid_images(__a):
												raise ValueError(
												    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
												    '''torch.Tensor, tf.Tensor or jax.ndarray.''')

							if do_resize and size is None:
												raise ValueError('''Size must be specified if do_resize is True.''')

							if do_center_crop and crop_size is None:
												raise ValueError('''Crop size must be specified if do_center_crop is True.''')

							if do_rescale and rescale_factor is None:
												raise ValueError('''Rescale factor must be specified if do_rescale is True.''')

							# All transformations expect numpy arrays.
							_UpperCamelCase									=    [to_numpy_array(__a) for image in images]

							if do_resize:
												_UpperCamelCase									=    [self.resize(image=__a							,						size=__a							,						resample=__a) for image in images]

							if do_center_crop:
												_UpperCamelCase									=    [self.center_crop(image=__a							,						size=__a) for image in images]

							if do_rescale:
												_UpperCamelCase									=    [self.rescale(image=__a							,						scale=__a) for image in images]

							if do_normalize:
												_UpperCamelCase									=    [self.normalize(image=__a							,						mean=__a							,						std=__a) for image in images]

							_UpperCamelCase									=    [to_channel_dimension_format(__a							,						__a) for image in images]

							_UpperCamelCase									=    {'''pixel_values''': images}
							return BatchFeature(data=__a							,						tensor_type=__a)






