


'''simple docstring'''
import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    WavaVecaConformerConfig,
    WavaVecaConformerForCTC,
    WavaVecaConformerForPreTraining,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.linear_k": "encoder.layers.*.self_attn.linear_k",
    "self_attn.linear_v": "encoder.layers.*.self_attn.linear_v",
    "self_attn.linear_q": "encoder.layers.*.self_attn.linear_q",
    "self_attn.pos_bias_u": "encoder.layers.*.self_attn.pos_bias_u",
    "self_attn.pos_bias_v": "encoder.layers.*.self_attn.pos_bias_v",
    "self_attn.linear_out": "encoder.layers.*.self_attn.linear_out",
    "self_attn.linear_pos": "encoder.layers.*.self_attn.linear_pos",
    "self_attn.rotary_emb": "encoder.embed_positions",
    "self_attn_layer_norm": "encoder.layers.*.self_attn_layer_norm",
    "conv_module.pointwise_conv1": "encoder.layers.*.conv_module.pointwise_conv1",
    "conv_module.pointwise_conv2": "encoder.layers.*.conv_module.pointwise_conv2",
    "conv_module.depthwise_conv": "encoder.layers.*.conv_module.depthwise_conv",
    "conv_module.batch_norm": "encoder.layers.*.conv_module.batch_norm",
    "conv_module.layer_norm": "encoder.layers.*.conv_module.layer_norm",
    "ffn1.w_1": "encoder.layers.*.ffn1.intermediate_dense",
    "ffn1.w_2": "encoder.layers.*.ffn1.output_dense",
    "ffn1.layer_norm": "encoder.layers.*.ffn1_layer_norm",
    "ffn2.w_1": "encoder.layers.*.ffn2.intermediate_dense",
    "ffn2.w_2": "encoder.layers.*.ffn2.output_dense",
    "ffn2.layer_norm": "encoder.layers.*.ffn2_layer_norm",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "quantizer.weight_proj": "quantizer.weight_proj",
    "quantizer.vars": "quantizer.codevectors",
    "project_q": "project_q",
    "final_proj": "project_hid",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}
__snake_case							   =  [
    "lm_head",
    "quantizer.weight_proj",
    "quantizer.codevectors",
    "project_q",
    "project_hid",
]


def     a   (    __a				,      __a				,      __a				,      __a				,      __a     )						->			Dict:



     '''simple docstring'''






     for attribute in key.split('''.'''     ):
          UpperCamelCase__     :List[str]							      =				getattr(__a				,      __a     )

     if weight_type is not None:
          UpperCamelCase__     :int							      =				getattr(__a				,      __a     ).shape
     else:
          UpperCamelCase__     :Union[str, Any]							      =				hf_pointer.shape

     if hf_shape != value.shape:
          raise ValueError(
              f'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
              f''' {value.shape} for {full_name}'''     )

     if weight_type == "weight":
          UpperCamelCase__     :Any							      =				value
     elif weight_type == "weight_g":
          UpperCamelCase__     :Tuple							      =				value
     elif weight_type == "weight_v":
          UpperCamelCase__     :Union[str, Any]							      =				value
     elif weight_type == "bias":
          UpperCamelCase__     :Any							      =				value
     elif weight_type == "running_mean":
          UpperCamelCase__     :List[str]							      =				value
     elif weight_type == "running_var":
          UpperCamelCase__     :int							      =				value
     elif weight_type == "num_batches_tracked":
          UpperCamelCase__     :Optional[Any]							      =				value
     elif weight_type == "inv_freq":
          UpperCamelCase__     :Union[str, Any]							      =				value
     else:
          UpperCamelCase__     :Optional[int]							      =				value

     logger.info(f'''{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'''     )


def     a   (    __a				,      __a				,      __a     )						->			Any:



     '''simple docstring'''






     UpperCamelCase__     :Tuple							      =				[]
     UpperCamelCase__     :List[str]							      =				fairseq_model.state_dict()

     UpperCamelCase__     :Tuple							      =				hf_model.wavaveca_conformer.feature_extractor

     for name, value in fairseq_dict.items():
          UpperCamelCase__     :str							      =				False
          if "conv_layers" in name:
               load_conv_layer(
                   __a				,      __a				,      __a				,      __a				,      hf_model.config.feat_extract_norm == '''group'''				,      )
               UpperCamelCase__     :List[str]							      =				True
          else:
               for key, mapped_key in MAPPING.items():
                    UpperCamelCase__     :str							      =				'''wav2vec2_conformer.''' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
                    if key in name or key.split('''w2v_model.'''     )[-1] == name.split('''.'''     )[0]:
                         UpperCamelCase__     :str							      =				True
                         if "*" in mapped_key:
                              UpperCamelCase__     :Dict							      =				name.split(__a     )[0].split('''.'''     )[-2]
                              UpperCamelCase__     :Optional[int]							      =				mapped_key.replace('''*'''				,      __a     )
                         if "pos_bias_u" in name:
                              UpperCamelCase__     :Any							      =				None
                         elif "pos_bias_v" in name:
                              UpperCamelCase__     :Union[str, Any]							      =				None
                         elif "weight_g" in name:
                              UpperCamelCase__     :Union[str, Any]							      =				'''weight_g'''
                         elif "weight_v" in name:
                              UpperCamelCase__     :List[Any]							      =				'''weight_v'''
                         elif "bias" in name:
                              UpperCamelCase__     :List[Any]							      =				'''bias'''
                         elif "weight" in name:
                              # TODO: don't match quantizer.weight_proj
                              UpperCamelCase__     :Optional[int]							      =				'''weight'''
                         elif "running_mean" in name:
                              UpperCamelCase__     :Dict							      =				'''running_mean'''
                         elif "inv_freq" in name:
                              UpperCamelCase__     :List[Any]							      =				'''inv_freq'''
                         elif "running_var" in name:
                              UpperCamelCase__     :Optional[int]							      =				'''running_var'''
                         elif "num_batches_tracked" in name:
                              UpperCamelCase__     :int							      =				'''num_batches_tracked'''
                         else:
                              UpperCamelCase__     :Optional[Any]							      =				None
                         set_recursively(__a				,      __a				,      __a				,      __a				,      __a     )
                    continue
          if not is_used:
               unused_weights.append(__a     )

     logger.warning(f'''Unused weights: {unused_weights}'''     )


def     a   (    __a				,      __a				,      __a				,      __a				,      __a     )						->			Optional[int]:



     '''simple docstring'''






     UpperCamelCase__     :Optional[int]							      =				full_name.split('''conv_layers.'''     )[-1]
     UpperCamelCase__     :Union[str, Any]							      =				name.split('''.'''     )
     UpperCamelCase__     :Union[str, Any]							      =				int(items[0]     )
     UpperCamelCase__     :List[Any]							      =				int(items[1]     )

     if type_id == 0:
          if "bias" in name:
               if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
                    raise ValueError(
                        f'''{full_name} has size {value.shape}, but'''
                        f''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''     )
               UpperCamelCase__     :Optional[Any]							      =				value
               logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''     )
          elif "weight" in name:
               if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
                    raise ValueError(
                        f'''{full_name} has size {value.shape}, but'''
                        f''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''     )
               UpperCamelCase__     :Any							      =				value
               logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''     )
     elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
          if "bias" in name:
               if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
                    raise ValueError(
                        f'''{full_name} has size {value.shape}, but'''
                        f''' {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.'''     )
               UpperCamelCase__     :Tuple							      =				value
               logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''     )
          elif "weight" in name:
               if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
                    raise ValueError(
                        f'''{full_name} has size {value.shape}, but'''
                        f''' {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.'''     )
               UpperCamelCase__     :Union[str, Any]							      =				value
               logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''     )
     else:
          unused_weights.append(__a     )


@torch.no_grad()
def     a   (    __a				,      __a				,      __a=None				,      __a=None				,      __a=True     )						->			int:



     '''simple docstring'''






     if config_path is not None:
          UpperCamelCase__     :Optional[Any]							      =				WavaVecaConformerConfig.from_pretrained(__a				,      hidden_act='''swish'''     )
     else:
          UpperCamelCase__     :int							      =				WavaVecaConformerConfig()

     if "rope" in checkpoint_path:
          UpperCamelCase__     :List[Any]							      =				'''rotary'''

     if is_finetuned:
          if dict_path:
               UpperCamelCase__     :Union[str, Any]							      =				Dictionary.load(__a     )

               # important change bos & pad token id since CTC symbol is <pad> and
               # not <s> as in fairseq
               UpperCamelCase__     :Union[str, Any]							      =				target_dict.pad_index
               UpperCamelCase__     :Optional[Any]							      =				target_dict.bos_index
               UpperCamelCase__     :Tuple							      =				target_dict.eos_index
               UpperCamelCase__     :Tuple							      =				len(target_dict.symbols     )
               UpperCamelCase__     :Optional[int]							      =				os.path.join(__a				,      '''vocab.json'''     )
               if not os.path.isdir(__a     ):
                    logger.error('''--pytorch_dump_folder_path ({}) should be a directory'''.format(__a     )     )
                    return
               os.makedirs(__a				,      exist_ok=__a     )
               UpperCamelCase__     :Any							      =				target_dict.indices

               # fairseq has the <pad> and <s> switched
               UpperCamelCase__     :Dict							      =				0
               UpperCamelCase__     :str							      =				1
               with open(__a				,      '''w'''				,      encoding='''utf-8'''     ) as vocab_handle:
                    json.dump(__a				,      __a     )
               UpperCamelCase__     :str							      =				WavaVecaCTCTokenizer(
                   __a				,      unk_token=target_dict.unk_word				,      pad_token=target_dict.pad_word				,      bos_token=target_dict.bos_word				,      eos_token=target_dict.eos_word				,      word_delimiter_token='''|'''				,      do_lower_case=__a				,      )
               UpperCamelCase__     :List[Any]							      =				True if config.feat_extract_norm == '''layer''' else False
               UpperCamelCase__     :Optional[int]							      =				WavaVecaFeatureExtractor(
                   feature_size=1				,      sampling_rate=16000				,      padding_value=0				,      do_normalize=__a				,      return_attention_mask=__a				,      )
               UpperCamelCase__     :Optional[Any]							      =				WavaVecaProcessor(feature_extractor=__a				,      tokenizer=__a     )
               processor.save_pretrained(__a     )

          UpperCamelCase__     :List[str]							      =				WavaVecaConformerForCTC(__a     )
     else:
          UpperCamelCase__     :Tuple							      =				WavaVecaConformerForPreTraining(__a     )

     if is_finetuned:
          UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :List[str]							      =				fairseq.checkpoint_utils.load_model_ensemble_and_task(
              [checkpoint_path]				,      arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''     )[:-1]     )}     )
     else:
          UpperCamelCase__     :str							      =				argparse.Namespace(task='''audio_pretraining'''     )
          UpperCamelCase__     :int							      =				fairseq.tasks.setup_task(__a     )

          UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :List[str]							      =				fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]				,      task=__a     )

     UpperCamelCase__     :List[str]							      =				model[0].eval()

     recursively_load_weights(__a				,      __a				,      not is_finetuned     )

     hf_wavavec.save_pretrained(__a     )


if __name__ == "__main__":
  __snake_case							   =  argparse.ArgumentParser()
  parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
  parser.add_argument('''--checkpoint_path''', default=None, type=str, help='''Path to fairseq checkpoint''')
  parser.add_argument('''--dict_path''', default=None, type=str, help='''Path to dict of fine-tuned model''')
  parser.add_argument('''--config_path''', default=None, type=str, help='''Path to hf config.json of model to convert''')
  parser.add_argument(
      '''--not_finetuned''', action='''store_true''', help='''Whether the model to convert is a fine-tuned model or not'''
  )
  __snake_case							   =  parser.parse_args()
  convert_wavaveca_conformer_checkpoint(
      args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
  )



'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       RoCBertTokenizer
		_a									=       None
		_a									=       False
		_a									=       True
		_a									=       filter_non_english




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase__     :Optional[Any]							      =				{}
							UpperCamelCase__     :str							      =				{}
							for i, value in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :List[Any]							      =				i
												UpperCamelCase__     :List[str]							      =				i
							UpperCamelCase__     :Any							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__     :Optional[Any]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_shape_file''']							)
							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_pronunciation_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
							with open(self.word_shape_file					,     '''w'''					,     encoding='''utf-8'''							) as word_shape_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)
							with open(self.word_pronunciation_file					,     '''w'''					,     encoding='''utf-8'''							) as word_pronunciation_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize('''你好[SEP]你是谁'''							)
							self.assertListEqual(UpperCamelCase_					,     ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)					,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''h\u00E9llo''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     never_split=['''[UNK]''']							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__     :List[str]							      =				{}
							for i, token in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :Tuple							      =				i
							UpperCamelCase__     :Any							      =				RoCBertWordpieceTokenizer(vocab=UpperCamelCase_					,     unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							)					,     []							)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)					,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)					,     ['''[UNK]''', '''runn''', '''##ing''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''							)							)
							self.assertTrue(_is_whitespace('''\t'''							)							)
							self.assertTrue(_is_whitespace('''\r'''							)							)
							self.assertTrue(_is_whitespace('''\n'''							)							)
							self.assertTrue(_is_whitespace('''\u00A0'''							)							)

							self.assertFalse(_is_whitespace('''A'''							)							)
							self.assertFalse(_is_whitespace('''-'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''							)							)

							self.assertFalse(_is_control('''A'''							)							)
							self.assertFalse(_is_control(''' '''							)							)
							self.assertFalse(_is_control('''\t'''							)							)
							self.assertFalse(_is_control('''\r'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''							)							)
							self.assertTrue(_is_punctuation('''$'''							)							)
							self.assertTrue(_is_punctuation('''`'''							)							)
							self.assertTrue(_is_punctuation('''.'''							)							)

							self.assertFalse(_is_punctuation('''A'''							)							)
							self.assertFalse(_is_punctuation(''' '''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)

							if self.test_rust_tokenizer:
												UpperCamelCase__     :List[str]							      =				self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Optional[Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__     :List[str]							      =				tokenizer_r.encode_plus(
																	    UpperCamelCase_					,     return_attention_mask=UpperCamelCase_					,     return_token_type_ids=UpperCamelCase_					,     return_offsets_mapping=UpperCamelCase_					,     add_special_tokens=UpperCamelCase_					,     )

																	UpperCamelCase__     :int							      =				tokenizer_r.do_lower_case if hasattr(UpperCamelCase_					,     '''do_lower_case'''							) else False
																	UpperCamelCase__     :str							      =				(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																	self.assertEqual([e[0] for e in expected_results]					,     tokens['''offset_mapping''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				['''的''', '''人''', '''有''']
							UpperCamelCase__     :Tuple							      =				''''''.join(UpperCamelCase_							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Union[str, Any]							      =				True
																	UpperCamelCase__     :str							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :Optional[Any]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :str							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				False
																	UpperCamelCase__     :Union[str, Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Dict							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__     :Optional[Any]							      =				[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_							)
																	]
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)




		@slow
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode('''你好'''					,     add_special_tokens=UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				tokenizer.encode('''你是谁'''					,     add_special_tokens=UpperCamelCase_							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
							UpperCamelCase__     :Optional[int]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_					,     UpperCamelCase_							)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[int]							      =				'''你好，你是谁'''
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(UpperCamelCase_							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.prepare_for_model(
																	    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer.encode_plus(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	self.assertEqual(UpperCamelCase_					,     UpperCamelCase_							)