





import argparse

import fairseq
import torch

from transformers import UniSpeechSatConfig, UniSpeechSatForCTC, UniSpeechSatForPreTraining, logging


logging.set_verbosity_info()
UpperCAmelCase_ 							=       logging.get_logger(__name__)

UpperCAmelCase_ 							=       {
    'post_extract_proj': 'feature_projection.projection',
    'encoder.pos_conv.0': 'encoder.pos_conv_embed.conv',
    'self_attn.k_proj': 'encoder.layers.*.attention.k_proj',
    'self_attn.v_proj': 'encoder.layers.*.attention.v_proj',
    'self_attn.q_proj': 'encoder.layers.*.attention.q_proj',
    'self_attn.out_proj': 'encoder.layers.*.attention.out_proj',
    'self_attn_layer_norm': 'encoder.layers.*.layer_norm',
    'fc1': 'encoder.layers.*.feed_forward.intermediate_dense',
    'fc2': 'encoder.layers.*.feed_forward.output_dense',
    'final_layer_norm': 'encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'encoder.layer_norm',
    'encoder.layer_norm_for_extract': 'layer_norm_for_extract',
    'w2v_model.layer_norm': 'feature_projection.layer_norm',
    'quantizer.weight_proj': 'quantizer.weight_proj',
    'quantizer.vars': 'quantizer.codevectors',
    'project_q': 'project_q',
    'final_proj': 'project_hid',
    'w2v_encoder.proj': 'lm_head',
    'label_embs_concat': 'label_embeddings_concat',
    'mask_emb': 'masked_spec_embed',
    'spk_proj': 'speaker_proj',
}
UpperCAmelCase_ 							=       [
    'lm_head',
    'quantizer.weight_proj',
    'quantizer.codevectors',
    'project_q',
    'project_hid',
    'label_embeddings_concat',
    'speaker_proj',
    'layer_norm_for_extract',
]





def   lowerCamelCase__       (       UpperCamelCase__     :    str    ,      UpperCamelCase__     :    List[Any]    ,      UpperCamelCase__     :    Optional[Any]    ,      UpperCamelCase__     :    List[str]    ,      UpperCamelCase__     :    List[str]  )     ->	List[str]:



					'''simple docstring'''




					for attribute in key.split('.'  ):
										_snake_case													=			getattr(UpperCamelCase__    ,      UpperCamelCase__  )

					if weight_type is not None:
										_snake_case													=			getattr(UpperCamelCase__    ,      UpperCamelCase__  ).shape
					else:
										_snake_case													=			hf_pointer.shape

					if hf_shape != value.shape:
										raise ValueError(
										    F'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
										    F''' {value.shape} for {full_name}'''  )

					if weight_type == "weight":
										_snake_case													=			value
					elif weight_type == "weight_g":
										_snake_case													=			value
					elif weight_type == "weight_v":
										_snake_case													=			value
					elif weight_type == "bias":
										_snake_case													=			value
					else:
										_snake_case													=			value

					logger.info(F'''{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'''  )





def   lowerCamelCase__       (       UpperCamelCase__     :    str    ,      UpperCamelCase__     :    Union[str, Any]  )     ->	Dict:



					'''simple docstring'''




					_snake_case													=			[]
					_snake_case													=			fairseq_model.state_dict()

					_snake_case													=			hf_model.unispeech_sat.feature_extractor

					for name, value in fairseq_dict.items():
										_snake_case													=			False
										if "conv_layers" in name:
															load_conv_layer(
															    UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__    ,      hf_model.config.feat_extract_norm == 'group'    ,      )
															_snake_case													=			True
										else:
															for key, mapped_key in MAPPING.items():
																				_snake_case													=			'unispeech_sat.' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
																				if key in name or key.split('w2v_model.'  )[-1] == name.split('.'  )[0]:
																									if "layer_norm_for_extract" in name and (".".join(name.split('.'  )[:-1]  ) != key):
																														# special case since naming is very similar
																														continue
																									_snake_case													=			True
																									if "*" in mapped_key:
																														_snake_case													=			name.split(UpperCamelCase__  )[0].split('.'  )[-2]
																														_snake_case													=			mapped_key.replace('*'    ,      UpperCamelCase__  )
																									if "weight_g" in name:
																														_snake_case													=			'weight_g'
																									elif "weight_v" in name:
																														_snake_case													=			'weight_v'
																									elif "bias" in name:
																														_snake_case													=			'bias'
																									elif "weight" in name:
																														# TODO: don't match quantizer.weight_proj
																														_snake_case													=			'weight'
																									else:
																														_snake_case													=			None
																									set_recursively(UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__    ,      UpperCamelCase__  )
																				continue
										if not is_used:
															unused_weights.append(UpperCamelCase__  )

					logger.warning(F'''Unused weights: {unused_weights}'''  )





def   lowerCamelCase__       (       UpperCamelCase__     :    Optional[int]    ,      UpperCamelCase__     :    Tuple    ,      UpperCamelCase__     :    Optional[Any]    ,      UpperCamelCase__     :    Tuple    ,      UpperCamelCase__     :    int  )     ->	Optional[int]:



					'''simple docstring'''




					_snake_case													=			full_name.split('conv_layers.'  )[-1]
					_snake_case													=			name.split('.'  )
					_snake_case													=			int(items[0]  )
					_snake_case													=			int(items[1]  )

					if type_id == 0:
										if "bias" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''  )
															_snake_case													=			value
															logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''  )
										elif "weight" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''  )
															_snake_case													=			value
															logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''  )
					elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
										if "bias" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor[layer_id].layer_norm.bias.data.shape} was found.'''  )
															_snake_case													=			value
															logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''  )
										elif "weight" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''  )
															_snake_case													=			value
															logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''  )
					else:
										unused_weights.append(UpperCamelCase__  )





@torch.no_grad()
def   lowerCamelCase__       (       UpperCamelCase__     :    Dict    ,      UpperCamelCase__     :    str    ,      UpperCamelCase__     :    Union[str, Any]=None    ,      UpperCamelCase__     :    Tuple=None    ,      UpperCamelCase__     :    int=True  )     ->	Any:



					'''simple docstring'''




					if config_path is not None:
										_snake_case													=			UniSpeechSatConfig.from_pretrained(UpperCamelCase__  )
					else:
										_snake_case													=			UniSpeechSatConfig()

					_snake_case													=			''

					if is_finetuned:
										_snake_case													=			UniSpeechSatForCTC(UpperCamelCase__  )
					else:
										_snake_case													=			UniSpeechSatForPreTraining(UpperCamelCase__  )

					_snake_case							,       _snake_case							,       _snake_case													=			fairseq.checkpoint_utils.load_model_ensemble_and_task(
					    [checkpoint_path]    ,      arg_overrides={'data': '/'.join(dict_path.split('/'  )[:-1]  )}  )
					_snake_case													=			model[0].eval()

					recursively_load_weights(UpperCamelCase__    ,      UpperCamelCase__  )

					hf_wavavec.save_pretrained(UpperCamelCase__  )


if __name__ == "__main__":
					UpperCAmelCase_ 							=       argparse.ArgumentParser()
					parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
					parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
					parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
					parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
					parser.add_argument(
					    """--not_finetuned""", action="""store_true""", help="""Whether the model to convert is a fine-tuned model or not"""
					)
					UpperCAmelCase_ 							=       parser.parse_args()
					convert_unispeech_sat_checkpoint(
					    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
					)










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		BertTokenizer
					lowerCAmelCase_							=		BertTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										super().setUp()

										_snake_case													=			[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			'UNwant\u00E9d,running'
										_snake_case													=			'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			self.tokenizer_class(self.vocab_file					)

										_snake_case													=			tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCAmelCase_			,  ['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  [9, 6, 7, 12, 10, 11]					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										if not self.test_rust_tokenizer:
															return

										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										# With lower casing
										_snake_case													=			self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
										_snake_case													=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)			,  ['ah', '\u535A', '\u63A8', 'zz']					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['h\u00E9llo']					)




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer()
										_snake_case													=			'a\n\'ll !!to?\'d of, can\'t.'
										_snake_case													=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													=			{}
										for i, token in enumerate(lowerCAmelCase_					):
															_snake_case													=			i
										_snake_case													=			WordpieceTokenizer(vocab=lowerCAmelCase_			,  unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					)			,  []					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					)			,  ['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					)			,  ['[UNK]', 'runn', '##ing']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)




					def 		lowerCAmelCase     (						self					)						->							int:
										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.tokenizer_class.from_pretrained('bert-base-uncased'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													=			tokenizer_r.encode_plus(
																				    lowerCAmelCase_			,  return_attention_mask=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_			,  return_offsets_mapping=lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  )

																				_snake_case													=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,  'do_lower_case'					) else False
																				_snake_case													=			(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
																				self.assertEqual([e[0] for e in expected_results]			,  tokens['offset_mapping']					)






					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			['的', '人', '有']
										_snake_case													=			''.join(lowerCAmelCase_					)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			True
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

																				_snake_case													=			False
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case													=			[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
																				]
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)



