
"""simple docstring"""


import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

						from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
						A_    							=					False

if is_vision_available():
						from PIL import Image

						from transformers import PixaStructImageProcessor




class  __lowerCamelCase  ( unittest.TestCase			):




				def __init__(				self							,						UpperCAmelCase							,						UpperCAmelCase=7							,						UpperCAmelCase=3							,						UpperCAmelCase=18							,						UpperCAmelCase=30							,						UpperCAmelCase=400							,						UpperCAmelCase=None							,						UpperCAmelCase=True							,						UpperCAmelCase=True							,						UpperCAmelCase=None							,						):
							lowerCamelCase_					=  size if size is not None else {'''height''': 20, '''width''': 20}
							lowerCamelCase_					=  parent
							lowerCamelCase_					=  batch_size
							lowerCamelCase_					=  num_channels
							lowerCamelCase_					=  image_size
							lowerCamelCase_					=  min_resolution
							lowerCamelCase_					=  max_resolution
							lowerCamelCase_					=  size
							lowerCamelCase_					=  do_normalize
							lowerCamelCase_					=  do_convert_rgb
							lowerCamelCase_					=  [512, 1024, 2048, 4096]
							lowerCamelCase_					=  patch_size if patch_size is not None else {'''height''': 16, '''width''': 16}




				def        UpperCAmelCase__						(				self   ):
							return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  '''https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg'''
							lowerCamelCase_					=  Image.open(requests.get(UpperCAmelCase							,						stream=UpperCAmelCase   ).raw   ).convert('''RGB'''   )
							return raw_image




@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11						,	reason='`Pix2StructImageProcessor` requires `torch>=1.11.0`.'						,	)
@require_torch
@require_vision
class  __lowerCamelCase  ( lowerCAmelCase						,	unittest.TestCase			):
				a__:      Union[str, Any]          =    PixaStructImageProcessor if is_vision_available() else None




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  PixaStructImageProcessingTester(self   )




				@property
				def        UpperCAmelCase__						(				self   ):
							return self.image_processor_tester.prepare_image_processor_dict()




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							self.assertTrue(hasattr(UpperCAmelCase							,						'''do_normalize'''   )   )
							self.assertTrue(hasattr(UpperCAmelCase							,						'''do_convert_rgb'''   )   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.image_processor_tester.prepare_dummy_image()

							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							lowerCamelCase_					=  2048

							lowerCamelCase_					=  image_processor(UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   )
							self.assertTrue(torch.allclose(inputs.flattened_patches.mean()							,						torch.tensor(0.0_6_0_6   )							,						atol=1e-3							,						rtol=1e-3   )   )




				def        UpperCAmelCase__						(				self   ):
							# Initialize image_processor
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							# create random PIL images
							lowerCamelCase_					=  prepare_image_inputs(self.image_processor_tester							,						equal_resolution=UpperCAmelCase   )
							for image in image_inputs:
										self.assertIsInstance(UpperCAmelCase							,						Image.Image   )

							# Test not batched input
							lowerCamelCase_					=  (
							    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
							    * self.image_processor_tester.num_channels
							) + 2

							for max_patch in self.image_processor_tester.max_patches:
										# Test not batched input
										lowerCamelCase_					=  image_processor(
										    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(1, max_patch, expected_hidden_dim)							,						)

										# Test batched
										lowerCamelCase_					=  image_processor(
										    UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,						)




				def        UpperCAmelCase__						(				self   ):
							# Initialize image_processor
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							# create random PIL images
							lowerCamelCase_					=  prepare_image_inputs(self.image_processor_tester							,						equal_resolution=UpperCAmelCase   )
							for image in image_inputs:
										self.assertIsInstance(UpperCAmelCase							,						Image.Image   )

							# Test not batched input
							lowerCamelCase_					=  (
							    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
							    * self.image_processor_tester.num_channels
							) + 2

							lowerCamelCase_					=  True

							for max_patch in self.image_processor_tester.max_patches:
										# Test not batched input
										with self.assertRaises(UpperCAmelCase   ):
													lowerCamelCase_					=  image_processor(
													    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches

										lowerCamelCase_					=  '''Hello'''

										lowerCamelCase_					=  image_processor(
										    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase							,						header_text=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(1, max_patch, expected_hidden_dim)							,						)

										# Test batched
										lowerCamelCase_					=  image_processor(
										    UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase							,						header_text=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,						)




				def        UpperCAmelCase__						(				self   ):
							# Initialize image_processor
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							# create random numpy tensors
							lowerCamelCase_					=  prepare_image_inputs(self.image_processor_tester							,						equal_resolution=UpperCAmelCase							,						numpify=UpperCAmelCase   )
							for image in image_inputs:
										self.assertIsInstance(UpperCAmelCase							,						np.ndarray   )

							lowerCamelCase_					=  (
							    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
							    * self.image_processor_tester.num_channels
							) + 2

							for max_patch in self.image_processor_tester.max_patches:
										# Test not batched input
										lowerCamelCase_					=  image_processor(
										    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(1, max_patch, expected_hidden_dim)							,						)

										# Test batched
										lowerCamelCase_					=  image_processor(
										    UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,						)





				def        UpperCAmelCase__						(				self   ):
							# Initialize image_processor
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							# create random PyTorch tensors
							lowerCamelCase_					=  prepare_image_inputs(self.image_processor_tester							,						equal_resolution=UpperCAmelCase							,						torchify=UpperCAmelCase   )
							for image in image_inputs:
										self.assertIsInstance(UpperCAmelCase							,						torch.Tensor   )

							# Test not batched input
							lowerCamelCase_					=  (
							    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
							    * self.image_processor_tester.num_channels
							) + 2

							for max_patch in self.image_processor_tester.max_patches:
										# Test not batched input
										lowerCamelCase_					=  image_processor(
										    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(1, max_patch, expected_hidden_dim)							,						)

										# Test batched
										lowerCamelCase_					=  image_processor(
										    UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,						)




@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11						,	reason='`Pix2StructImageProcessor` requires `torch>=1.11.0`.'						,	)
@require_torch
@require_vision
class  __lowerCamelCase  ( lowerCAmelCase						,	unittest.TestCase			):
				a__:      Any          =    PixaStructImageProcessor if is_vision_available() else None




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  PixaStructImageProcessingTester(self							,						num_channels=4   )
							lowerCamelCase_					=  3




				@property
				def        UpperCAmelCase__						(				self   ):
							return self.image_processor_tester.prepare_image_processor_dict()




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							self.assertTrue(hasattr(UpperCAmelCase							,						'''do_normalize'''   )   )
							self.assertTrue(hasattr(UpperCAmelCase							,						'''do_convert_rgb'''   )   )





				def        UpperCAmelCase__						(				self   ):
							# Initialize image_processor
							lowerCamelCase_					=  self.image_processing_class(**self.image_processor_dict   )
							# create random PIL images
							lowerCamelCase_					=  prepare_image_inputs(self.image_processor_tester							,						equal_resolution=UpperCAmelCase   )
							for image in image_inputs:
										self.assertIsInstance(UpperCAmelCase							,						Image.Image   )

							# Test not batched input
							lowerCamelCase_					=  (
							    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
							    * (self.image_processor_tester.num_channels - 1)
							) + 2

							for max_patch in self.image_processor_tester.max_patches:
										# Test not batched input
										lowerCamelCase_					=  image_processor(
										    image_inputs[0]							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(1, max_patch, expected_hidden_dim)							,						)

										# Test batched
										lowerCamelCase_					=  image_processor(
										    UpperCAmelCase							,						return_tensors='''pt'''							,						max_patches=UpperCAmelCase   ).flattened_patches
										self.assertEqual(
										    encoded_images.shape							,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,						)





"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class  __lowerCamelCase  ( lowerCAmelCase						,	unittest.TestCase			):
				a__:      Tuple          =    RoCBertTokenizer
				a__:      int          =    None
				a__:      Optional[Any]          =    False
				a__:      Optional[int]          =    True
				a__:      Tuple          =    filter_non_english




				def        UpperCAmelCase__						(				self   ):
							super().setUp()

							lowerCamelCase_					=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							lowerCamelCase_					=  {}
							lowerCamelCase_					=  {}
							for i, value in enumerate(UpperCAmelCase   ):
										lowerCamelCase_					=  i
										lowerCamelCase_					=  i
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file''']   )
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''word_shape_file''']   )
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''word_pronunciation_file''']   )
							with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8'''   ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
							with open(self.word_shape_file							,						'''w'''							,						encoding='''utf-8'''   ) as word_shape_writer:
										json.dump(UpperCAmelCase							,						UpperCAmelCase							,						ensure_ascii=UpperCAmelCase   )
							with open(self.word_pronunciation_file							,						'''w'''							,						encoding='''utf-8'''   ) as word_pronunciation_writer:
										json.dump(UpperCAmelCase							,						UpperCAmelCase							,						ensure_ascii=UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class(self.vocab_file							,						self.word_shape_file							,						self.word_pronunciation_file   )

							lowerCamelCase_					=  tokenizer.tokenize('''你好[SEP]你是谁'''   )
							self.assertListEqual(UpperCAmelCase							,						['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )							,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''h\u00E9llo''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						never_split=['''[UNK]''']   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							lowerCamelCase_					=  {}
							for i, token in enumerate(UpperCAmelCase   ):
										lowerCamelCase_					=  i
							lowerCamelCase_					=  RoCBertWordpieceTokenizer(vocab=UpperCAmelCase							,						unk_token='''[UNK]'''   )

							self.assertListEqual(tokenizer.tokenize(''''''   )							,						[]   )

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )							,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )							,						['''[UNK]''', '''runn''', '''##ing''']   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_whitespace(''' '''   )   )
							self.assertTrue(_is_whitespace('''\t'''   )   )
							self.assertTrue(_is_whitespace('''\r'''   )   )
							self.assertTrue(_is_whitespace('''\n'''   )   )
							self.assertTrue(_is_whitespace('''\u00A0'''   )   )

							self.assertFalse(_is_whitespace('''A'''   )   )
							self.assertFalse(_is_whitespace('''-'''   )   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_control('''\u0005'''   )   )

							self.assertFalse(_is_control('''A'''   )   )
							self.assertFalse(_is_control(''' '''   )   )
							self.assertFalse(_is_control('''\t'''   )   )
							self.assertFalse(_is_control('''\r'''   )   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_punctuation('''-'''   )   )
							self.assertTrue(_is_punctuation('''$'''   )   )
							self.assertTrue(_is_punctuation('''`'''   )   )
							self.assertTrue(_is_punctuation('''.'''   )   )

							self.assertFalse(_is_punctuation('''A'''   )   )
							self.assertFalse(_is_punctuation(''' '''   )   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCAmelCase   ) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']]   )

							if self.test_rust_tokenizer:
										lowerCamelCase_					=  self.get_rust_tokenizer()
										self.assertListEqual(
										    [rust_tokenizer.tokenize(UpperCAmelCase   ) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']]   )




				def        UpperCAmelCase__						(				self   ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
													lowerCamelCase_					=  tokenizer_r.encode_plus(
													    UpperCAmelCase							,						return_attention_mask=UpperCAmelCase							,						return_token_type_ids=UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						)

													lowerCamelCase_					=  tokenizer_r.do_lower_case if hasattr(UpperCAmelCase							,						'''do_lower_case'''   ) else False
													lowerCamelCase_					=  (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
													self.assertEqual([e[0] for e in expected_results]							,						tokens['''offset_mapping''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  ['''的''', '''人''', '''有''']
							lowerCamelCase_					=  ''''''.join(UpperCAmelCase   )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  True
													lowerCamelCase_					=  self.tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_p.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase   )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )

													lowerCamelCase_					=  False
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  self.tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase   )

													# it is expected that only the first Chinese character is not preceded by "##".
													lowerCamelCase_					=  [
													    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase   )
													]
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )




				@slow
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class(self.vocab_file							,						self.word_shape_file							,						self.word_pronunciation_file   )

							lowerCamelCase_					=  tokenizer.encode('''你好'''							,						add_special_tokens=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.encode('''你是谁'''							,						add_special_tokens=UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase							,						UpperCAmelCase   )

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizers(do_lower_case=UpperCAmelCase   )
							for tokenizer in tokenizers:
										with self.subTest(f"{tokenizer.__class__.__name__}"   ):
													lowerCamelCase_					=  '''你好，你是谁'''
													lowerCamelCase_					=  tokenizer.tokenize(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.prepare_for_model(
													    UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer.encode_plus(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )



