


'''simple docstring'''




import importlib
import math
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Tuple, Union

import flax
import jax.numpy as jnp

from ..utils import BaseOutput


_a   :   Optional[int]												=   "scheduler_config.json"




class 			_lowercase (	__lowercase							):
   _SCREAMING_SNAKE_CASE  :  List[str]          =	1
   _SCREAMING_SNAKE_CASE  :  Any          =	2
   _SCREAMING_SNAKE_CASE  :  int          =	3
   _SCREAMING_SNAKE_CASE  :  str          =	4
   _SCREAMING_SNAKE_CASE  :  int          =	5




@dataclass
class 			_lowercase (	__lowercase							):
   _SCREAMING_SNAKE_CASE  :  jnp.ndarray




class 			_lowercase :
   _SCREAMING_SNAKE_CASE  :  Optional[Any]          =	SCHEDULER_CONFIG_NAME
   _SCREAMING_SNAKE_CASE  :  Any          =	["dtype"]
   _SCREAMING_SNAKE_CASE  :  Tuple          =	[]
   _SCREAMING_SNAKE_CASE  :  Any          =	True
   @classmethod
   def a  (						cls       :      List[Any]       ,   SCREAMING_SNAKE_CASE_       :      Dict[str, Any] = None       ,   SCREAMING_SNAKE_CASE_       :      Optional[str] = None       ,   SCREAMING_SNAKE_CASE_       :      List[Any]=False       ,   **SCREAMING_SNAKE_CASE_       :      Dict       ,   ) ->							Dict:
        __snake_case      ,       __snake_case           =						cls.load_config(
            pretrained_model_name_or_path=SCREAMING_SNAKE_CASE_       ,   subfolder=SCREAMING_SNAKE_CASE_       ,   return_unused_kwargs=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_       ,   )
        __snake_case      ,       __snake_case           =						cls.from_config(SCREAMING_SNAKE_CASE_       ,   return_unused_kwargs=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )

        if hasattr(SCREAMING_SNAKE_CASE_       ,   'create_state'      ) and getattr(SCREAMING_SNAKE_CASE_       ,   'has_state'       ,   SCREAMING_SNAKE_CASE_      ):
             __snake_case           =						scheduler.create_state()

        if return_unused_kwargs:
             return scheduler, state, unused_kwargs

        return scheduler, state
   def a  (						self       :      str       ,   SCREAMING_SNAKE_CASE_       :      Union[str, os.PathLike]       ,   SCREAMING_SNAKE_CASE_       :      bool = False       ,   **SCREAMING_SNAKE_CASE_       :      Union[str, Any]      ) ->							Union[str, Any]:
        self.save_config(save_directory=SCREAMING_SNAKE_CASE_       ,   push_to_hub=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )
   @property
   def a  (						self       :      Optional[Any]      ) ->							Optional[int]:
        return self._get_compatibles()





   @classmethod
   def a  (						cls       :      Any      ) ->							int:
        __snake_case           =						list(set([cls.__name__] + cls._compatibles      )      )
        __snake_case           =						importlib.import_module(__name__.split('.'      )[0]      )
        __snake_case           =						[
            getattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      ) for c in compatible_classes_str if hasattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
        ]
        return compatible_classes





def   _a		(lowercase__			:   jnp.ndarray							,   lowercase__			:   Tuple[int]						)			->			jnp.ndarray:






     """simple docstring"""
     assert len(lowercase__						) >= x.ndim
     return jnp.broadcast_to(x.reshape(x.shape + (1,) * (len(lowercase__						) - x.ndim)						)							,   lowercase__						)





def   _a		(lowercase__			:   int							,   lowercase__			:   Optional[Any]=0.9_99							,   lowercase__			:   Dict=jnp.floataa						)			->			jnp.ndarray:






     """simple docstring"""

     def alpha_bar(lowercase__			:   Union[str, Any]						):
          return math.cos((time_step + 0.0_08) / 1.0_08 * math.pi / 2						) ** 2

     __snake_case           =						[]
     for i in range(lowercase__						):
          __snake_case           =						i / num_diffusion_timesteps
          __snake_case           =						(i + 1) / num_diffusion_timesteps
          betas.append(min(1 - alpha_bar(lowercase__						) / alpha_bar(lowercase__						)							,   lowercase__						)						)
     return jnp.array(lowercase__							,   dtype=lowercase__						)




@flax.struct.dataclass
class 			_lowercase :
   _SCREAMING_SNAKE_CASE  :  jnp.ndarray
   _SCREAMING_SNAKE_CASE  :  jnp.ndarray
   _SCREAMING_SNAKE_CASE  :  jnp.ndarray
   @classmethod
   def a  (						cls       :      List[str]       ,   SCREAMING_SNAKE_CASE_       :      Union[str, Any]      ) ->							Dict:
        __snake_case           =						scheduler.config

        if config.trained_betas is not None:
             __snake_case           =						jnp.asarray(config.trained_betas       ,   dtype=scheduler.dtype      )
        elif config.beta_schedule == "linear":
             __snake_case           =						jnp.linspace(config.beta_start       ,   config.beta_end       ,   config.num_train_timesteps       ,   dtype=scheduler.dtype      )
        elif config.beta_schedule == "scaled_linear":
             # this schedule is very specific to the latent diffusion model.
             __snake_case           =						(
                 jnp.linspace(
                     config.beta_start**0.5       ,   config.beta_end**0.5       ,   config.num_train_timesteps       ,   dtype=scheduler.dtype      )
                 ** 2
             )
        elif config.beta_schedule == "squaredcos_cap_v2":
             # Glide cosine schedule
             __snake_case           =						betas_for_alpha_bar(config.num_train_timesteps       ,   dtype=scheduler.dtype      )
        else:
             raise NotImplementedError(
                 f'beta_schedule {config.beta_schedule} is not implemented for scheduler {scheduler.__class__.__name__}'      )

        __snake_case           =						1.0 - betas

        __snake_case           =						jnp.cumprod(SCREAMING_SNAKE_CASE_       ,   axis=0      )

        return cls(
            alphas=SCREAMING_SNAKE_CASE_       ,   betas=SCREAMING_SNAKE_CASE_       ,   alphas_cumprod=SCREAMING_SNAKE_CASE_       ,   )





def   _a		(lowercase__			:   CommonSchedulerState							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray						)			->			Optional[int]:






     """simple docstring"""
     __snake_case           =						state.alphas_cumprod

     __snake_case           =						alphas_cumprod[timesteps] ** 0.5
     __snake_case           =						sqrt_alpha_prod.flatten()
     __snake_case           =						broadcast_to_shape_from_left(lowercase__							,   original_samples.shape						)

     __snake_case           =						(1 - alphas_cumprod[timesteps]) ** 0.5
     __snake_case           =						sqrt_one_minus_alpha_prod.flatten()
     __snake_case           =						broadcast_to_shape_from_left(lowercase__							,   original_samples.shape						)

     return sqrt_alpha_prod, sqrt_one_minus_alpha_prod





def   _a		(lowercase__			:   CommonSchedulerState							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray						)			->			int:






     """simple docstring"""
     __snake_case      ,       __snake_case           =						get_sqrt_alpha_prod(lowercase__							,   lowercase__							,   lowercase__							,   lowercase__						)
     __snake_case           =						sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
     return noisy_samples





def   _a		(lowercase__			:   CommonSchedulerState							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray							,   lowercase__			:   jnp.ndarray						)			->			List[Any]:






     """simple docstring"""
     __snake_case      ,       __snake_case           =						get_sqrt_alpha_prod(lowercase__							,   lowercase__							,   lowercase__							,   lowercase__						)
     __snake_case           =						sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
     return velocity






"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class  __lowerCamelCase  ( lowerCAmelCase						,	unittest.TestCase			):
				a__:      Tuple          =    RoCBertTokenizer
				a__:      int          =    None
				a__:      Optional[Any]          =    False
				a__:      Optional[int]          =    True
				a__:      Tuple          =    filter_non_english




				def        UpperCAmelCase__						(				self   ):
							super().setUp()

							lowerCamelCase_					=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							lowerCamelCase_					=  {}
							lowerCamelCase_					=  {}
							for i, value in enumerate(UpperCAmelCase   ):
										lowerCamelCase_					=  i
										lowerCamelCase_					=  i
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file''']   )
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''word_shape_file''']   )
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''word_pronunciation_file''']   )
							with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8'''   ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
							with open(self.word_shape_file							,						'''w'''							,						encoding='''utf-8'''   ) as word_shape_writer:
										json.dump(UpperCAmelCase							,						UpperCAmelCase							,						ensure_ascii=UpperCAmelCase   )
							with open(self.word_pronunciation_file							,						'''w'''							,						encoding='''utf-8'''   ) as word_pronunciation_writer:
										json.dump(UpperCAmelCase							,						UpperCAmelCase							,						ensure_ascii=UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class(self.vocab_file							,						self.word_shape_file							,						self.word_pronunciation_file   )

							lowerCamelCase_					=  tokenizer.tokenize('''你好[SEP]你是谁'''   )
							self.assertListEqual(UpperCAmelCase							,						['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase   )							,						[5, 6, 2, 5, 7, 8]   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )							,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''h\u00E9llo''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )							,						['''hello''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						strip_accents=UpperCAmelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )							,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase							,						never_split=['''[UNK]''']   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )							,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							lowerCamelCase_					=  {}
							for i, token in enumerate(UpperCAmelCase   ):
										lowerCamelCase_					=  i
							lowerCamelCase_					=  RoCBertWordpieceTokenizer(vocab=UpperCAmelCase							,						unk_token='''[UNK]'''   )

							self.assertListEqual(tokenizer.tokenize(''''''   )							,						[]   )

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )							,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )							,						['''[UNK]''', '''runn''', '''##ing''']   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_whitespace(''' '''   )   )
							self.assertTrue(_is_whitespace('''\t'''   )   )
							self.assertTrue(_is_whitespace('''\r'''   )   )
							self.assertTrue(_is_whitespace('''\n'''   )   )
							self.assertTrue(_is_whitespace('''\u00A0'''   )   )

							self.assertFalse(_is_whitespace('''A'''   )   )
							self.assertFalse(_is_whitespace('''-'''   )   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_control('''\u0005'''   )   )

							self.assertFalse(_is_control('''A'''   )   )
							self.assertFalse(_is_control(''' '''   )   )
							self.assertFalse(_is_control('''\t'''   )   )
							self.assertFalse(_is_control('''\r'''   )   )




				def        UpperCAmelCase__						(				self   ):
							self.assertTrue(_is_punctuation('''-'''   )   )
							self.assertTrue(_is_punctuation('''$'''   )   )
							self.assertTrue(_is_punctuation('''`'''   )   )
							self.assertTrue(_is_punctuation('''.'''   )   )

							self.assertFalse(_is_punctuation('''A'''   )   )
							self.assertFalse(_is_punctuation(''' '''   )   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCAmelCase   ) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']]   )

							if self.test_rust_tokenizer:
										lowerCamelCase_					=  self.get_rust_tokenizer()
										self.assertListEqual(
										    [rust_tokenizer.tokenize(UpperCAmelCase   ) for t in ['''Test''', '''\xad''', '''test''']]							,						[['''[UNK]'''], [], ['''[UNK]''']]   )




				def        UpperCAmelCase__						(				self   ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
													lowerCamelCase_					=  tokenizer_r.encode_plus(
													    UpperCAmelCase							,						return_attention_mask=UpperCAmelCase							,						return_token_type_ids=UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						)

													lowerCamelCase_					=  tokenizer_r.do_lower_case if hasattr(UpperCAmelCase							,						'''do_lower_case'''   ) else False
													lowerCamelCase_					=  (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
													self.assertEqual([e[0] for e in expected_results]							,						tokens['''offset_mapping''']   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  ['''的''', '''人''', '''有''']
							lowerCamelCase_					=  ''''''.join(UpperCAmelCase   )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  True
													lowerCamelCase_					=  self.tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_p.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase   )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )

													lowerCamelCase_					=  False
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  self.tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase   )

													# it is expected that only the first Chinese character is not preceded by "##".
													lowerCamelCase_					=  [
													    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase   )
													]
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )
													self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )




				@slow
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class(self.vocab_file							,						self.word_shape_file							,						self.word_pronunciation_file   )

							lowerCamelCase_					=  tokenizer.encode('''你好'''							,						add_special_tokens=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.encode('''你是谁'''							,						add_special_tokens=UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase							,						UpperCAmelCase   )

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizers(do_lower_case=UpperCAmelCase   )
							for tokenizer in tokenizers:
										with self.subTest(f"{tokenizer.__class__.__name__}"   ):
													lowerCamelCase_					=  '''你好，你是谁'''
													lowerCamelCase_					=  tokenizer.tokenize(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer.prepare_for_model(
													    UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													lowerCamelCase_					=  tokenizer.encode_plus(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )

													self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )



