





import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextConfig, SegformerImageProcessor, UperNetConfig, UperNetForSemanticSegmentation





def   lowerCamelCase__       (       UpperCamelCase__     :    Tuple  )     ->	Dict:



					'''simple docstring'''




					_snake_case													=			384
					if "tiny" in model_name:
										_snake_case													=			[3, 3, 9, 3]
										_snake_case													=			[96, 192, 384, 768]
					if "small" in model_name:
										_snake_case													=			[3, 3, 27, 3]
										_snake_case													=			[96, 192, 384, 768]
					if "base" in model_name:
										_snake_case													=			[3, 3, 27, 3]
										_snake_case													=			[128, 256, 512, 1_024]
										_snake_case													=			512
					if "large" in model_name:
										_snake_case													=			[3, 3, 27, 3]
										_snake_case													=			[192, 384, 768, 1_536]
										_snake_case													=			768
					if "xlarge" in model_name:
										_snake_case													=			[3, 3, 27, 3]
										_snake_case													=			[256, 512, 1_024, 2_048]
										_snake_case													=			1_024

					# set label information
					_snake_case													=			150
					_snake_case													=			'huggingface/label-files'
					_snake_case													=			'ade20k-id2label.json'
					_snake_case													=			json.load(open(hf_hub_download(UpperCamelCase__    ,      UpperCamelCase__    ,      repo_type='dataset'  )    ,      'r'  )  )
					_snake_case													=			{int(UpperCamelCase__  ): v for k, v in idalabel.items()}
					_snake_case													=			{v: k for k, v in idalabel.items()}

					_snake_case													=			ConvNextConfig(
					    depths=UpperCamelCase__    ,      hidden_sizes=UpperCamelCase__    ,      out_features=['stage1', 'stage2', 'stage3', 'stage4']  )
					_snake_case													=			UperNetConfig(
					    backbone_config=UpperCamelCase__    ,      auxiliary_in_channels=UpperCamelCase__    ,      num_labels=UpperCamelCase__    ,      idalabel=UpperCamelCase__    ,      labelaid=UpperCamelCase__    ,      )

					return config





def   lowerCamelCase__       (       UpperCamelCase__     :    Optional[int]  )     ->	str:



					'''simple docstring'''




					_snake_case													=			[]

					# fmt: off
					# stem
					rename_keys.append(('backbone.downsample_layers.0.0.weight', 'backbone.embeddings.patch_embeddings.weight')  )
					rename_keys.append(('backbone.downsample_layers.0.0.bias', 'backbone.embeddings.patch_embeddings.bias')  )
					rename_keys.append(('backbone.downsample_layers.0.1.weight', 'backbone.embeddings.layernorm.weight')  )
					rename_keys.append(('backbone.downsample_layers.0.1.bias', 'backbone.embeddings.layernorm.bias')  )
					# stages
					for i in range(len(config.backbone_config.depths  )  ):
										for j in range(config.backbone_config.depths[i]  ):
															rename_keys.append((F'''backbone.stages.{i}.{j}.gamma''', F'''backbone.encoder.stages.{i}.layers.{j}.layer_scale_parameter''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.depthwise_conv.weight''', F'''backbone.encoder.stages.{i}.layers.{j}.dwconv.weight''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.depthwise_conv.bias''', F'''backbone.encoder.stages.{i}.layers.{j}.dwconv.bias''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.norm.weight''', F'''backbone.encoder.stages.{i}.layers.{j}.layernorm.weight''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.norm.bias''', F'''backbone.encoder.stages.{i}.layers.{j}.layernorm.bias''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.pointwise_conv1.weight''', F'''backbone.encoder.stages.{i}.layers.{j}.pwconv1.weight''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.pointwise_conv1.bias''', F'''backbone.encoder.stages.{i}.layers.{j}.pwconv1.bias''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.pointwise_conv2.weight''', F'''backbone.encoder.stages.{i}.layers.{j}.pwconv2.weight''')  )
															rename_keys.append((F'''backbone.stages.{i}.{j}.pointwise_conv2.bias''', F'''backbone.encoder.stages.{i}.layers.{j}.pwconv2.bias''')  )
										if i > 0:
															rename_keys.append((F'''backbone.downsample_layers.{i}.0.weight''', F'''backbone.encoder.stages.{i}.downsampling_layer.0.weight''')  )
															rename_keys.append((F'''backbone.downsample_layers.{i}.0.bias''', F'''backbone.encoder.stages.{i}.downsampling_layer.0.bias''')  )
															rename_keys.append((F'''backbone.downsample_layers.{i}.1.weight''', F'''backbone.encoder.stages.{i}.downsampling_layer.1.weight''')  )
															rename_keys.append((F'''backbone.downsample_layers.{i}.1.bias''', F'''backbone.encoder.stages.{i}.downsampling_layer.1.bias''')  )

										rename_keys.append((F'''backbone.norm{i}.weight''', F'''backbone.hidden_states_norms.stage{i+1}.weight''')  )
										rename_keys.append((F'''backbone.norm{i}.bias''', F'''backbone.hidden_states_norms.stage{i+1}.bias''')  )

					# decode head
					rename_keys.extend(
					    [
					        ('decode_head.conv_seg.weight', 'decode_head.classifier.weight'),
					        ('decode_head.conv_seg.bias', 'decode_head.classifier.bias'),
					        ('auxiliary_head.conv_seg.weight', 'auxiliary_head.classifier.weight'),
					        ('auxiliary_head.conv_seg.bias', 'auxiliary_head.classifier.bias'),
					    ]  )
					# fmt: on

					return rename_keys





def   lowerCamelCase__       (       UpperCamelCase__     :    Optional[int]    ,      UpperCamelCase__     :    Union[str, Any]    ,      UpperCamelCase__     :    Union[str, Any]  )     ->	Optional[Any]:



					'''simple docstring'''




					_snake_case													=			dct.pop(UpperCamelCase__  )
					_snake_case													=			val





def   lowerCamelCase__       (       UpperCamelCase__     :    Optional[Any]    ,      UpperCamelCase__     :    str    ,      UpperCamelCase__     :    Optional[Any]  )     ->	Dict:



					'''simple docstring'''




					_snake_case													=			{
					    'upernet-convnext-tiny': 'https://download.openmmlab.com/mmsegmentation/v0.5/convnext/upernet_convnext_tiny_fp16_512x512_160k_ade20k/upernet_convnext_tiny_fp16_512x512_160k_ade20k_20220227_124553-cad485de.pth',
					    'upernet-convnext-small': 'https://download.openmmlab.com/mmsegmentation/v0.5/convnext/upernet_convnext_small_fp16_512x512_160k_ade20k/upernet_convnext_small_fp16_512x512_160k_ade20k_20220227_131208-1b1e394f.pth',
					    'upernet-convnext-base': 'https://download.openmmlab.com/mmsegmentation/v0.5/convnext/upernet_convnext_base_fp16_512x512_160k_ade20k/upernet_convnext_base_fp16_512x512_160k_ade20k_20220227_181227-02a24fc6.pth',
					    'upernet-convnext-large': 'https://download.openmmlab.com/mmsegmentation/v0.5/convnext/upernet_convnext_large_fp16_640x640_160k_ade20k/upernet_convnext_large_fp16_640x640_160k_ade20k_20220226_040532-e57aa54d.pth',
					    'upernet-convnext-xlarge': 'https://download.openmmlab.com/mmsegmentation/v0.5/convnext/upernet_convnext_xlarge_fp16_640x640_160k_ade20k/upernet_convnext_xlarge_fp16_640x640_160k_ade20k_20220226_080344-95fc38c2.pth',
					}
					_snake_case													=			model_name_to_url[model_name]
					_snake_case													=			torch.hub.load_state_dict_from_url(UpperCamelCase__    ,      map_location='cpu'  )['state_dict']

					_snake_case													=			get_upernet_config(UpperCamelCase__  )
					_snake_case													=			UperNetForSemanticSegmentation(UpperCamelCase__  )
					model.eval()

					# replace "bn" => "batch_norm"
					for key in state_dict.copy().keys():
										_snake_case													=			state_dict.pop(UpperCamelCase__  )
										if "bn" in key:
															_snake_case													=			key.replace('bn'    ,      'batch_norm'  )
										_snake_case													=			val

					# rename keys
					_snake_case													=			create_rename_keys(UpperCamelCase__  )
					for src, dest in rename_keys:
										rename_key(UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__  )

					model.load_state_dict(UpperCamelCase__  )

					# verify on image
					_snake_case													=			'https://huggingface.co/datasets/hf-internal-testing/fixtures_ade20k/resolve/main/ADE_val_00000001.jpg'
					_snake_case													=			Image.open(requests.get(UpperCamelCase__    ,      stream=UpperCamelCase__  ).raw  ).convert('RGB'  )

					_snake_case													=			SegformerImageProcessor()
					_snake_case													=			processor(UpperCamelCase__    ,      return_tensors='pt'  ).pixel_values

					with torch.no_grad():
										_snake_case													=			model(UpperCamelCase__  )

					if model_name == "upernet-convnext-tiny":
										_snake_case													=			torch.tensor(
										    [[-8.8110, -8.8110, -8.6521], [-8.8110, -8.8110, -8.6521], [-8.7746, -8.7746, -8.6130]]  )
					elif model_name == "upernet-convnext-small":
										_snake_case													=			torch.tensor(
										    [[-8.8236, -8.8236, -8.6771], [-8.8236, -8.8236, -8.6771], [-8.7638, -8.7638, -8.6240]]  )
					elif model_name == "upernet-convnext-base":
										_snake_case													=			torch.tensor(
										    [[-8.8558, -8.8558, -8.6905], [-8.8558, -8.8558, -8.6905], [-8.7669, -8.7669, -8.6021]]  )
					elif model_name == "upernet-convnext-large":
										_snake_case													=			torch.tensor(
										    [[-8.6660, -8.6660, -8.6210], [-8.6660, -8.6660, -8.6210], [-8.6310, -8.6310, -8.5964]]  )
					elif model_name == "upernet-convnext-xlarge":
										_snake_case													=			torch.tensor(
										    [[-8.4980, -8.4980, -8.3977], [-8.4980, -8.4980, -8.3977], [-8.4379, -8.4379, -8.3412]]  )
					print('Logits:'    ,      outputs.logits[0, 0, :3, :3]  )
					assert torch.allclose(outputs.logits[0, 0, :3, :3]    ,      UpperCamelCase__    ,      atol=1e-4  )
					print('Looks ok!'  )

					if pytorch_dump_folder_path is not None:
										print(F'''Saving model {model_name} to {pytorch_dump_folder_path}'''  )
										model.save_pretrained(UpperCamelCase__  )
										print(F'''Saving processor to {pytorch_dump_folder_path}'''  )
										processor.save_pretrained(UpperCamelCase__  )

					if push_to_hub:
										print(F'''Pushing model and processor for {model_name} to hub'''  )
										model.push_to_hub(F'''openmmlab/{model_name}'''  )
										processor.push_to_hub(F'''openmmlab/{model_name}'''  )


if __name__ == "__main__":
					UpperCAmelCase_ 							=       argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""upernet-convnext-tiny""",
					    type=str,
					    choices=[F"upernet-convnext-{size}" for size in ["""tiny""", """small""", """base""", """large""", """xlarge"""]],
					    help="""Name of the ConvNext UperNet model you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
					)

					UpperCAmelCase_ 							=       parser.parse_args()
					convert_upernet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		BertTokenizer
					lowerCAmelCase_							=		BertTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										super().setUp()

										_snake_case													=			[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			'UNwant\u00E9d,running'
										_snake_case													=			'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			self.tokenizer_class(self.vocab_file					)

										_snake_case													=			tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCAmelCase_			,  ['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  [9, 6, 7, 12, 10, 11]					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										if not self.test_rust_tokenizer:
															return

										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										# With lower casing
										_snake_case													=			self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
										_snake_case													=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)			,  ['ah', '\u535A', '\u63A8', 'zz']					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['h\u00E9llo']					)




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer()
										_snake_case													=			'a\n\'ll !!to?\'d of, can\'t.'
										_snake_case													=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													=			{}
										for i, token in enumerate(lowerCAmelCase_					):
															_snake_case													=			i
										_snake_case													=			WordpieceTokenizer(vocab=lowerCAmelCase_			,  unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					)			,  []					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					)			,  ['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					)			,  ['[UNK]', 'runn', '##ing']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)




					def 		lowerCAmelCase     (						self					)						->							int:
										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.tokenizer_class.from_pretrained('bert-base-uncased'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													=			tokenizer_r.encode_plus(
																				    lowerCAmelCase_			,  return_attention_mask=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_			,  return_offsets_mapping=lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  )

																				_snake_case													=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,  'do_lower_case'					) else False
																				_snake_case													=			(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
																				self.assertEqual([e[0] for e in expected_results]			,  tokens['offset_mapping']					)






					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			['的', '人', '有']
										_snake_case													=			''.join(lowerCAmelCase_					)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			True
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

																				_snake_case													=			False
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case													=			[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
																				]
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)



