


'''simple docstring'''
from collections import defaultdict
from typing import Optional

from ..image_utils import load_image
from ..utils import (
    add_end_docstrings,
    is_torch_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ChunkPipeline


if is_torch_available():
		import torch

		from ..models.auto.modeling_auto import MODEL_FOR_MASK_GENERATION_MAPPING

__snake_case							   =  logging.get_logger(__name__)
@add_end_docstrings(A__      )
class      lowercase			(							A__      ):

		"""simple docstring"""

		def __init__(	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							super().__init__(**UpperCamelCase_							)
							requires_backends(self					,     '''vision'''							)
							requires_backends(self					,     '''torch'''							)

							if self.framework != "pt":
												raise ValueError(F'''The {self.__class__} is only available in PyTorch.'''							)

							self.check_model_type(UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[int]							      =				{}
							UpperCamelCase__     :Dict							      =				{}
							UpperCamelCase__     :Optional[int]							      =				{}
							# preprocess args
							if "points_per_batch" in kwargs:
												UpperCamelCase__     :int							      =				kwargs['''points_per_batch''']
							if "points_per_crop" in kwargs:
												UpperCamelCase__     :Any							      =				kwargs['''points_per_crop''']
							if "crops_n_layers" in kwargs:
												UpperCamelCase__     :Dict							      =				kwargs['''crops_n_layers''']
							if "crop_overlap_ratio" in kwargs:
												UpperCamelCase__     :Union[str, Any]							      =				kwargs['''crop_overlap_ratio''']
							if "crop_n_points_downscale_factor" in kwargs:
												UpperCamelCase__     :Optional[Any]							      =				kwargs['''crop_n_points_downscale_factor''']
							# postprocess args
							if "pred_iou_thresh" in kwargs:
												UpperCamelCase__     :int							      =				kwargs['''pred_iou_thresh''']
							if "stability_score_offset" in kwargs:
												UpperCamelCase__     :List[str]							      =				kwargs['''stability_score_offset''']
							if "mask_threshold" in kwargs:
												UpperCamelCase__     :Optional[int]							      =				kwargs['''mask_threshold''']
							if "stability_score_thresh" in kwargs:
												UpperCamelCase__     :Dict							      =				kwargs['''stability_score_thresh''']
							if "crops_nms_thresh" in kwargs:
												UpperCamelCase__     :Dict							      =				kwargs['''crops_nms_thresh''']
							if "output_rle_mask" in kwargs:
												UpperCamelCase__     :Dict							      =				kwargs['''output_rle_mask''']
							if "output_bboxes_mask" in kwargs:
												UpperCamelCase__     :List[str]							      =				kwargs['''output_bboxes_mask''']
							return preprocess_kwargs, forward_params, postprocess_kwargs




		def __call__(	self					,     UpperCamelCase_					,     *UpperCamelCase_					,     UpperCamelCase_=None					,     UpperCamelCase_=None					,     **UpperCamelCase_							):

							'''simple docstring'''


							return super().__call__(UpperCamelCase_					,     *UpperCamelCase_					,     num_workers=UpperCamelCase_					,     batch_size=UpperCamelCase_					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_=64					,     UpperCamelCase_ = 0					,     UpperCamelCase_ = 512 / 1500					,     UpperCamelCase_ = 32					,     UpperCamelCase_ = 1					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Optional[int]							      =				load_image(UpperCamelCase_							)
							UpperCamelCase__     :Optional[Any]							      =				self.image_processor.size['''longest_edge''']
							UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :int							      =				self.image_processor.generate_crop_boxes(
							    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)
							UpperCamelCase__     :Union[str, Any]							      =				self.image_processor(images=UpperCamelCase_					,     return_tensors='''pt'''							)

							with self.device_placement():
												if self.framework == "pt":
																	UpperCamelCase__     :List[str]							      =				self.get_inference_context()
																	with inference_context():
																						UpperCamelCase__     :str							      =				self._ensure_tensor_on_device(UpperCamelCase_					,     device=self.device							)
																						UpperCamelCase__     :Tuple							      =				self.model.get_image_embeddings(model_inputs.pop('''pixel_values'''							)							)
																						UpperCamelCase__     :Optional[int]							      =				image_embeddings

							UpperCamelCase__     :Optional[Any]							      =				grid_points.shape[1]
							UpperCamelCase__     :int							      =				points_per_batch if points_per_batch is not None else n_points

							if points_per_batch <= 0:
												raise ValueError(
												    '''Cannot have points_per_batch<=0. Must be >=1 to returned batched outputs. '''
												    '''To return all points at once, set points_per_batch to None'''							)

							for i in range(0					,     UpperCamelCase_					,     UpperCamelCase_							):
												UpperCamelCase__     :Optional[Any]							      =				grid_points[:, i : i + points_per_batch, :, :]
												UpperCamelCase__     :Dict							      =				input_labels[:, i : i + points_per_batch]
												UpperCamelCase__     :Optional[Any]							      =				i == n_points - points_per_batch
												yield {
												    "input_points": batched_points,
												    "input_labels": labels,
												    "input_boxes": crop_boxes,
												    "is_last": is_last,
												    **model_inputs,
												}




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_=0.88					,     UpperCamelCase_=0.95					,     UpperCamelCase_=0					,     UpperCamelCase_=1					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Optional[int]							      =				model_inputs.pop('''input_boxes'''							)
							UpperCamelCase__     :Optional[int]							      =				model_inputs.pop('''is_last'''							)
							UpperCamelCase__     :int							      =				model_inputs.pop('''original_sizes'''							).tolist()
							UpperCamelCase__     :str							      =				model_inputs.pop('''reshaped_input_sizes'''							).tolist()

							UpperCamelCase__     :Dict							      =				self.model(**UpperCamelCase_							)

							# post processing happens here in order to avoid CPU GPU copies of ALL the masks
							UpperCamelCase__     :Any							      =				model_outputs['''pred_masks''']
							UpperCamelCase__     :Union[str, Any]							      =				self.image_processor.post_process_masks(
							    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     binarize=UpperCamelCase_							)
							UpperCamelCase__     :Optional[int]							      =				model_outputs['''iou_scores''']
							UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Tuple							      =				self.image_processor.filter_masks(
							    masks[0]					,     iou_scores[0]					,     original_sizes[0]					,     input_boxes[0]					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     )
							return {
							    "masks": masks,
							    "is_last": is_last,
							    "boxes": boxes,
							    "iou_scores": iou_scores,
							}




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_=False					,     UpperCamelCase_=False					,     UpperCamelCase_=0.7					,     ):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				[]
							UpperCamelCase__     :List[str]							      =				[]
							UpperCamelCase__     :Dict							      =				[]
							for model_output in model_outputs:
												all_scores.append(model_output.pop('''iou_scores'''							)							)
												all_masks.extend(model_output.pop('''masks'''							)							)
												all_boxes.append(model_output.pop('''boxes'''							)							)

							UpperCamelCase__     :Optional[Any]							      =				torch.cat(UpperCamelCase_							)
							UpperCamelCase__     :Optional[Any]							      =				torch.cat(UpperCamelCase_							)
							UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :List[Any]							      =				self.image_processor.post_process_for_mask_generation(
							    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)

							UpperCamelCase__     :int							      =				defaultdict(UpperCamelCase_							)
							for output in model_outputs:
												for k, v in output.items():
																	extra[k].append(UpperCamelCase_							)

							UpperCamelCase__     :List[str]							      =				{}
							if output_rle_mask:
												UpperCamelCase__     :Tuple							      =				rle_mask

							if output_bboxes_mask:
												UpperCamelCase__     :Optional[Any]							      =				bounding_boxes

							return {"masks": output_masks, "scores": iou_scores, **optional, **extra}



'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       RoCBertTokenizer
		_a									=       None
		_a									=       False
		_a									=       True
		_a									=       filter_non_english




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase__     :Optional[Any]							      =				{}
							UpperCamelCase__     :str							      =				{}
							for i, value in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :List[Any]							      =				i
												UpperCamelCase__     :List[str]							      =				i
							UpperCamelCase__     :Any							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__     :Optional[Any]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_shape_file''']							)
							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_pronunciation_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
							with open(self.word_shape_file					,     '''w'''					,     encoding='''utf-8'''							) as word_shape_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)
							with open(self.word_pronunciation_file					,     '''w'''					,     encoding='''utf-8'''							) as word_pronunciation_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize('''你好[SEP]你是谁'''							)
							self.assertListEqual(UpperCamelCase_					,     ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)					,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''h\u00E9llo''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     never_split=['''[UNK]''']							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__     :List[str]							      =				{}
							for i, token in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :Tuple							      =				i
							UpperCamelCase__     :Any							      =				RoCBertWordpieceTokenizer(vocab=UpperCamelCase_					,     unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							)					,     []							)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)					,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)					,     ['''[UNK]''', '''runn''', '''##ing''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''							)							)
							self.assertTrue(_is_whitespace('''\t'''							)							)
							self.assertTrue(_is_whitespace('''\r'''							)							)
							self.assertTrue(_is_whitespace('''\n'''							)							)
							self.assertTrue(_is_whitespace('''\u00A0'''							)							)

							self.assertFalse(_is_whitespace('''A'''							)							)
							self.assertFalse(_is_whitespace('''-'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''							)							)

							self.assertFalse(_is_control('''A'''							)							)
							self.assertFalse(_is_control(''' '''							)							)
							self.assertFalse(_is_control('''\t'''							)							)
							self.assertFalse(_is_control('''\r'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''							)							)
							self.assertTrue(_is_punctuation('''$'''							)							)
							self.assertTrue(_is_punctuation('''`'''							)							)
							self.assertTrue(_is_punctuation('''.'''							)							)

							self.assertFalse(_is_punctuation('''A'''							)							)
							self.assertFalse(_is_punctuation(''' '''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)

							if self.test_rust_tokenizer:
												UpperCamelCase__     :List[str]							      =				self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Optional[Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__     :List[str]							      =				tokenizer_r.encode_plus(
																	    UpperCamelCase_					,     return_attention_mask=UpperCamelCase_					,     return_token_type_ids=UpperCamelCase_					,     return_offsets_mapping=UpperCamelCase_					,     add_special_tokens=UpperCamelCase_					,     )

																	UpperCamelCase__     :int							      =				tokenizer_r.do_lower_case if hasattr(UpperCamelCase_					,     '''do_lower_case'''							) else False
																	UpperCamelCase__     :str							      =				(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																	self.assertEqual([e[0] for e in expected_results]					,     tokens['''offset_mapping''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				['''的''', '''人''', '''有''']
							UpperCamelCase__     :Tuple							      =				''''''.join(UpperCamelCase_							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Union[str, Any]							      =				True
																	UpperCamelCase__     :str							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :Optional[Any]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :str							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				False
																	UpperCamelCase__     :Union[str, Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Dict							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__     :Optional[Any]							      =				[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_							)
																	]
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)




		@slow
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode('''你好'''					,     add_special_tokens=UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				tokenizer.encode('''你是谁'''					,     add_special_tokens=UpperCamelCase_							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
							UpperCamelCase__     :Optional[int]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_					,     UpperCamelCase_							)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[int]							      =				'''你好，你是谁'''
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(UpperCamelCase_							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.prepare_for_model(
																	    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer.encode_plus(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	self.assertEqual(UpperCamelCase_					,     UpperCamelCase_							)