





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


UpperCAmelCase_ 							=       logging.get_logger(__name__)


if is_vision_available():
					import PIL
class      UpperCamelCase_						(					_lowerCamelCase   ):
					lowerCAmelCase_							=		['''pixel_values''']




					def __init__(						self			,  lowerCAmelCase_ = True			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = PILImageResampling.BICUBIC			,  lowerCAmelCase_ = True			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = True			,  lowerCAmelCase_ = 1 / 255			,  lowerCAmelCase_ = True			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = True			,  **lowerCAmelCase_			,  )						->							None:
										super().__init__(**lowerCAmelCase_					)
										_snake_case													=			size if size is not None else {'shortest_edge': 224}
										_snake_case													=			get_size_dict(lowerCAmelCase_			,  default_to_square=lowerCAmelCase_					)
										_snake_case													=			crop_size if crop_size is not None else {'height': 224, 'width': 224}
										_snake_case													=			get_size_dict(lowerCAmelCase_			,  default_to_square=lowerCAmelCase_			,  param_name='crop_size'					)

										_snake_case													=			do_resize
										_snake_case													=			size
										_snake_case													=			resample
										_snake_case													=			do_center_crop
										_snake_case													=			crop_size
										_snake_case													=			do_rescale
										_snake_case													=			rescale_factor
										_snake_case													=			do_normalize
										_snake_case													=			image_mean if image_mean is not None else OPENAI_CLIP_MEAN
										_snake_case													=			image_std if image_std is not None else OPENAI_CLIP_STD
										_snake_case													=			do_convert_rgb




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_ = PILImageResampling.BICUBIC			,  lowerCAmelCase_ = None			,  **lowerCAmelCase_			,  )						->							np.ndarray:
										_snake_case													=			get_size_dict(lowerCAmelCase_			,  default_to_square=lowerCAmelCase_					)
										if "shortest_edge" not in size:
															raise ValueError(F'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''					)
										_snake_case													=			get_resize_output_image_size(lowerCAmelCase_			,  size=size['shortest_edge']			,  default_to_square=lowerCAmelCase_					)
										return resize(lowerCAmelCase_			,  size=lowerCAmelCase_			,  resample=lowerCAmelCase_			,  data_format=lowerCAmelCase_			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_ = None			,  **lowerCAmelCase_			,  )						->							np.ndarray:
										_snake_case													=			get_size_dict(lowerCAmelCase_					)
										if "height" not in size or "width" not in size:
															raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}'''					)
										return center_crop(lowerCAmelCase_			,  size=(size['height'], size['width'])			,  data_format=lowerCAmelCase_			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_ = None			,  **lowerCAmelCase_			,  )						->							Dict:
										return rescale(lowerCAmelCase_			,  scale=lowerCAmelCase_			,  data_format=lowerCAmelCase_			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_ = None			,  **lowerCAmelCase_			,  )						->							np.ndarray:
										return normalize(lowerCAmelCase_			,  mean=lowerCAmelCase_			,  std=lowerCAmelCase_			,  data_format=lowerCAmelCase_			,  **lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = ChannelDimension.FIRST			,  **lowerCAmelCase_			,  )						->							PIL.Image.Image:
										_snake_case													=			do_resize if do_resize is not None else self.do_resize
										_snake_case													=			size if size is not None else self.size
										_snake_case													=			get_size_dict(lowerCAmelCase_			,  param_name='size'			,  default_to_square=lowerCAmelCase_					)
										_snake_case													=			resample if resample is not None else self.resample
										_snake_case													=			do_center_crop if do_center_crop is not None else self.do_center_crop
										_snake_case													=			crop_size if crop_size is not None else self.crop_size
										_snake_case													=			get_size_dict(lowerCAmelCase_			,  param_name='crop_size'			,  default_to_square=lowerCAmelCase_					)
										_snake_case													=			do_rescale if do_rescale is not None else self.do_rescale
										_snake_case													=			rescale_factor if rescale_factor is not None else self.rescale_factor
										_snake_case													=			do_normalize if do_normalize is not None else self.do_normalize
										_snake_case													=			image_mean if image_mean is not None else self.image_mean
										_snake_case													=			image_std if image_std is not None else self.image_std
										_snake_case													=			do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

										_snake_case													=			make_list_of_images(lowerCAmelCase_					)

										if not valid_images(lowerCAmelCase_					):
															raise ValueError(
															    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
															    'torch.Tensor, tf.Tensor or jax.ndarray.'					)

										if do_resize and size is None:
															raise ValueError('Size must be specified if do_resize is True.'					)

										if do_center_crop and crop_size is None:
															raise ValueError('Crop size must be specified if do_center_crop is True.'					)

										if do_rescale and rescale_factor is None:
															raise ValueError('Rescale factor must be specified if do_rescale is True.'					)

										if do_normalize and (image_mean is None or image_std is None):
															raise ValueError('Image mean and std must be specified if do_normalize is True.'					)

										# PIL RGBA images are converted to RGB
										if do_convert_rgb:
															_snake_case													=			[convert_to_rgb(lowerCAmelCase_					) for image in images]

										# All transformations expect numpy arrays.
										_snake_case													=			[to_numpy_array(lowerCAmelCase_					) for image in images]

										if do_resize:
															_snake_case													=			[self.resize(image=lowerCAmelCase_			,  size=lowerCAmelCase_			,  resample=lowerCAmelCase_					) for image in images]

										if do_center_crop:
															_snake_case													=			[self.center_crop(image=lowerCAmelCase_			,  size=lowerCAmelCase_					) for image in images]

										if do_rescale:
															_snake_case													=			[self.rescale(image=lowerCAmelCase_			,  scale=lowerCAmelCase_					) for image in images]

										if do_normalize:
															_snake_case													=			[self.normalize(image=lowerCAmelCase_			,  mean=lowerCAmelCase_			,  std=lowerCAmelCase_					) for image in images]

										_snake_case													=			[to_channel_dimension_format(lowerCAmelCase_			,  lowerCAmelCase_					) for image in images]

										_snake_case													=			{'pixel_values': images}
										return BatchFeature(data=lowerCAmelCase_			,  tensor_type=lowerCAmelCase_					)










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		BertTokenizer
					lowerCAmelCase_							=		BertTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										super().setUp()

										_snake_case													=			[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			'UNwant\u00E9d,running'
										_snake_case													=			'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			self.tokenizer_class(self.vocab_file					)

										_snake_case													=			tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCAmelCase_			,  ['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  [9, 6, 7, 12, 10, 11]					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										if not self.test_rust_tokenizer:
															return

										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										# With lower casing
										_snake_case													=			self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
										_snake_case													=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)			,  ['ah', '\u535A', '\u63A8', 'zz']					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['h\u00E9llo']					)




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer()
										_snake_case													=			'a\n\'ll !!to?\'d of, can\'t.'
										_snake_case													=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													=			{}
										for i, token in enumerate(lowerCAmelCase_					):
															_snake_case													=			i
										_snake_case													=			WordpieceTokenizer(vocab=lowerCAmelCase_			,  unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					)			,  []					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					)			,  ['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					)			,  ['[UNK]', 'runn', '##ing']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)




					def 		lowerCAmelCase     (						self					)						->							int:
										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.tokenizer_class.from_pretrained('bert-base-uncased'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													=			tokenizer_r.encode_plus(
																				    lowerCAmelCase_			,  return_attention_mask=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_			,  return_offsets_mapping=lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  )

																				_snake_case													=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,  'do_lower_case'					) else False
																				_snake_case													=			(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
																				self.assertEqual([e[0] for e in expected_results]			,  tokens['offset_mapping']					)






					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			['的', '人', '有']
										_snake_case													=			''.join(lowerCAmelCase_					)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			True
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

																				_snake_case													=			False
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case													=			[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
																				]
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)



