

import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np
import torch

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


A_			:					Optional[Any]    			=     logging.get_logger(__name__)



class        _a		(__magic_name__							):


					'''simple docstring'''






					UpperCAmelCase__:  Optional[int]    				=			['''input_features''', '''is_longer''']






					def __init__(					self       ,			A__=64       ,			A__=4_8000       ,			A__=480       ,			A__=10       ,			A__=1024       ,			A__=0.0       ,			A__=False       ,			A__ = 0       ,			A__ = 1_4000       ,			A__ = None       ,			A__ = "fusion"       ,			A__ = "repeatpad"       ,			**A__       ,			):
						super().__init__(
						    feature_size=A__       ,			sampling_rate=A__       ,			padding_value=A__       ,			return_attention_mask=A__       ,			**A__       ,			)
						A__				:       List[Any]	  =						top_db
						A__				:       Optional[Any]	  =						truncation
						A__				:       List[str]	  =						padding
						A__				:       Dict	  =						fft_window_size
						A__				:       str	  =						(fft_window_size >> 1) + 1
						A__				:       str	  =						hop_length
						A__				:       Dict	  =						max_length_s
						A__				:       List[str]	  =						max_length_s * sampling_rate
						A__				:       List[str]	  =						sampling_rate
						A__				:       List[Any]	  =						frequency_min
						A__				:       Union[str, Any]	  =						frequency_max
						A__				:       str	  =						mel_filter_bank(
						    num_frequency_bins=self.nb_frequency_bins       ,			num_mel_filters=A__       ,			min_frequency=A__       ,			max_frequency=A__       ,			sampling_rate=A__       ,			norm=A__       ,			mel_scale="""htk"""       ,			)
						A__				:       List[str]	  =						mel_filter_bank(
						    num_frequency_bins=self.nb_frequency_bins       ,			num_mel_filters=A__       ,			min_frequency=A__       ,			max_frequency=A__       ,			sampling_rate=A__       ,			norm="""slaney"""       ,			mel_scale="""slaney"""       ,			)






					def     __A     (					self     ):
						A__				:       Tuple	  =						copy.deepcopy(self.__dict__     )
						A__				:       Union[str, Any]	  =						self.__class__.__name__
						if "mel_filters" in output:
							del output["mel_filters"]
						if "mel_filters_slaney" in output:
							del output["mel_filters_slaney"]
						return output






					def     __A     (					self       ,			A__       ,			A__ = None     ):
						A__				:       List[Any]	  =						spectrogram(
						    A__       ,			window_function(self.fft_window_size       ,			"""hann"""     )       ,			frame_length=self.fft_window_size       ,			hop_length=self.hop_length       ,			power=2.0       ,			mel_filters=A__       ,			log_mel="""dB"""       ,			)
						return log_mel_spectrogram.T






					def     __A     (					self       ,			A__       ,			A__       ,			A__     ):
						A__				:       List[Any]	  =						np.array_split(list(range(0       ,			total_frames - chunk_frames + 1     )     )       ,			3     )
						if len(ranges[1]     ) == 0:
							# if the audio is too short, we just use the first chunk
							A__				:       Optional[Any]	  =						[0]
						if len(ranges[2]     ) == 0:
							# if the audio is too short, we just use the first chunk
							A__				:       Dict	  =						[0]
						# randomly choose index for each part
						A__				:       Tuple	  =						np.random.choice(ranges[0]     )
						A__				:       Optional[int]	  =						np.random.choice(ranges[1]     )
						A__				:       Optional[Any]	  =						np.random.choice(ranges[2]     )

						A__				:       Tuple	  =						mel[idx_front : idx_front + chunk_frames, :]
						A__				:       int	  =						mel[idx_middle : idx_middle + chunk_frames, :]
						A__				:       Optional[int]	  =						mel[idx_back : idx_back + chunk_frames, :]

						A__				:       Tuple	  =						torch.tensor(mel[None, None, :]     )
						A__				:       List[str]	  =						torch.nn.functional.interpolate(
						    A__       ,			size=[chunk_frames, 64]       ,			mode="""bilinear"""       ,			align_corners=A__     )
						A__				:       Optional[int]	  =						mel_shrink[0][0].numpy()
						A__				:       str	  =						np.stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back]       ,			axis=0     )
						return mel_fusion






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__     ):
						if waveform.shape[0] > max_length:
							if truncation == "rand_trunc":
								A__				:       Tuple	  =						True
								# random crop to max_length (for compatibility) -> this should be handled by self.pad
								A__				:       Optional[Any]	  =						len(A__     ) - max_length
								A__				:       Tuple	  =						np.random.randint(0       ,			overflow + 1     )
								A__				:       Union[str, Any]	  =						waveform[idx : idx + max_length]
								A__				:       Optional[Any]	  =						self._np_extract_fbank_features(A__       ,			self.mel_filters_slaney     )[None, :]
							elif truncation == "fusion":
								A__				:       Optional[int]	  =						self._np_extract_fbank_features(A__       ,			self.mel_filters     )
								A__				:       Tuple	  =						max_length // self.hop_length + 1  # the +1 related to how the spectrogram is computed
								A__				:       List[Any]	  =						mel.shape[0]
								if chunk_frames == total_frames:
									# there is a corner case where the audio length is larger than max_length but smaller than max_length+hop_length.
									# In this case, we just use the whole audio.
									A__				:       str	  =						np.stack([mel, mel, mel, mel]       ,			axis=0     )
									A__				:       Optional[Any]	  =						False
								else:
									A__				:       List[str]	  =						self._random_mel_fusion(A__       ,			A__       ,			A__     )
									A__				:       List[Any]	  =						True
							else:
								raise NotImplementedError(F"""data_truncating {truncation} not implemented"""     )

						else:
							A__				:       str	  =						False
							# only use repeat as a new possible value for padding. you repeat the audio before applying the usual max_length padding
							if waveform.shape[0] < max_length:
								if padding == "repeat":
									A__				:       Union[str, Any]	  =						int(max_length / len(A__     )     )
									A__				:       Optional[Any]	  =						np.stack(np.tile(A__       ,			n_repeat + 1     )     )[:max_length]
								if padding == "repeatpad":
									A__				:       Optional[int]	  =						int(max_length / len(A__     )     )
									A__				:       List[str]	  =						np.stack(np.tile(A__       ,			A__     )     )
								A__				:       List[Any]	  =						np.pad(A__       ,			(0, max_length - waveform.shape[0])       ,			mode="""constant"""       ,			constant_values=0     )

							if truncation == "fusion":
								A__				:       List[Any]	  =						self._np_extract_fbank_features(A__       ,			self.mel_filters     )
								A__				:       Optional[int]	  =						np.stack([input_mel, input_mel, input_mel, input_mel]       ,			axis=0     )
							else:
								A__				:       List[Any]	  =						self._np_extract_fbank_features(A__       ,			self.mel_filters_slaney     )[None, :]

						return input_mel, longer






					def __call__(					self       ,			A__       ,			A__ = None       ,			A__ = None       ,			A__ = None       ,			A__ = None       ,			A__ = None       ,			**A__       ,			):
						A__				:       Optional[int]	  =						truncation if truncation is not None else self.truncation
						A__				:       Dict	  =						padding if padding else self.padding

						if sampling_rate is not None:
							if sampling_rate != self.sampling_rate:
								raise ValueError(
								    F"""The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"""
								    F""" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"""
								    F""" was sampled with {self.sampling_rate} and not {sampling_rate}."""     )
						else:
							logger.warning(
							    """It is strongly recommended to pass the `sampling_rate` argument to this function. """
							    """Failing to do so can result in silent errors that might be hard to debug."""     )

						A__				:       List[Any]	  =						isinstance(A__       ,			np.ndarray     ) and len(raw_speech.shape     ) > 1
						if is_batched_numpy and len(raw_speech.shape     ) > 2:
							raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""     )
						A__				:       Dict	  =						is_batched_numpy or (
						    isinstance(A__       ,			(list, tuple)     ) and (isinstance(raw_speech[0]       ,			(np.ndarray, tuple, list)     ))
						)

						if is_batched:
							A__				:       Any	  =						[np.asarray(A__       ,			dtype=np.floataa     ) for speech in raw_speech]
						elif not is_batched and not isinstance(A__       ,			np.ndarray     ):
							A__				:       List[str]	  =						np.asarray(A__       ,			dtype=np.floataa     )
						elif isinstance(A__       ,			np.ndarray     ) and raw_speech.dtype is np.dtype(np.floataa     ):
							A__				:       Any	  =						raw_speech.astype(np.floataa     )

						# always return batch
						if not is_batched:
							A__				:       int	  =						[np.asarray(A__     )]

						# convert to mel spectrogram, truncate and pad if needed.
						A__				:       List[Any]	  =						[
						    self._get_input_mel(A__       ,			max_length if max_length else self.nb_max_samples       ,			A__       ,			A__     )
						    for waveform in raw_speech
						]

						A__				:       int	  =						[]
						A__				:       List[str]	  =						[]
						for mel, longer in padded_inputs:
							input_mel.append(A__     )
							is_longer.append(A__     )

						if truncation == "fusion" and sum(A__     ) == 0:
							# if no audio is longer than 10s, then randomly select one audio to be longer
							A__				:       Dict	  =						np.random.randint(0       ,			len(A__     )     )
							A__				:       str	  =						True

						if isinstance(input_mel[0]       ,			A__     ):
							A__				:       Tuple	  =						[np.asarray(A__       ,			dtype=np.floataa     ) for feature in input_mel]

						# is_longer is a list of bool
						A__				:       str	  =						[[longer] for longer in is_longer]

						A__				:       Union[str, Any]	  =						{"""input_features""": input_mel, """is_longer""": is_longer}
						A__				:       int	  =						BatchFeature(A__     )

						if return_tensors is not None:
							A__				:       List[Any]	  =						input_features.convert_to_tensors(A__     )

						return input_features






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class        _a		(__magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  List[str]    				=			RoCBertTokenizer
					UpperCAmelCase__:  Dict    				=			None
					UpperCAmelCase__:  Optional[Any]    				=			False
					UpperCAmelCase__:  Union[str, Any]    				=			True
					UpperCAmelCase__:  Union[str, Any]    				=			filter_non_english






					def     __A     (					self     ):
						super().setUp()

						A__				:       Tuple	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
						A__				:       Union[str, Any]	  =						{}
						A__				:       Dict	  =						{}
						for i, value in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
							A__				:       Optional[int]	  =						i
						A__				:       Dict	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""vocab_file"""]     )
						A__				:       Optional[Any]	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_shape_file"""]     )
						A__				:       int	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
						with open(self.vocab_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
						with open(self.word_shape_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_shape_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )
						with open(self.word_pronunciation_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_pronunciation_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )






					def     __A     (					self     ):
						A__				:       Dict	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.tokenize("""你好[SEP]你是谁"""     )
						self.assertListEqual(A__       ,			["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )       ,			["""ah""", """\u535A""", """\u63A8""", """zz"""]     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       int	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""h\u00E9llo"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			never_split=["""[UNK]"""]     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )






					def     __A     (					self     ):
						A__				:       List[Any]	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__				:       Any	  =						{}
						for i, token in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
						A__				:       Optional[Any]	  =						RoCBertWordpieceTokenizer(vocab=A__       ,			unk_token="""[UNK]"""     )

						self.assertListEqual(tokenizer.tokenize(""""""     )       ,			[]     )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )       ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )       ,			["""[UNK]""", """runn""", """##ing"""]     )






					def     __A     (					self     ):
						self.assertTrue(_is_whitespace(""" """     )     )
						self.assertTrue(_is_whitespace("""\t"""     )     )
						self.assertTrue(_is_whitespace("""\r"""     )     )
						self.assertTrue(_is_whitespace("""\n"""     )     )
						self.assertTrue(_is_whitespace("""\u00A0"""     )     )

						self.assertFalse(_is_whitespace("""A"""     )     )
						self.assertFalse(_is_whitespace("""-"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_control("""\u0005"""     )     )

						self.assertFalse(_is_control("""A"""     )     )
						self.assertFalse(_is_control(""" """     )     )
						self.assertFalse(_is_control("""\t"""     )     )
						self.assertFalse(_is_control("""\r"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_punctuation("""-"""     )     )
						self.assertTrue(_is_punctuation("""$"""     )     )
						self.assertTrue(_is_punctuation("""`"""     )     )
						self.assertTrue(_is_punctuation("""."""     )     )

						self.assertFalse(_is_punctuation("""A"""     )     )
						self.assertFalse(_is_punctuation(""" """     )     )






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )

						if self.test_rust_tokenizer:
							A__				:       Tuple	  =						self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )






					def     __A     (					self     ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       str	  =						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__				:       Union[str, Any]	  =						tokenizer_r.encode_plus(
								    A__       ,			return_attention_mask=A__       ,			return_token_type_ids=A__       ,			return_offsets_mapping=A__       ,			add_special_tokens=A__       ,			)

								A__				:       Any	  =						tokenizer_r.do_lower_case if hasattr(A__       ,			"""do_lower_case"""     ) else False
								A__				:       List[str]	  =						(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]       ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
								self.assertEqual([e[0] for e in expected_results]       ,			tokens["""offset_mapping"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						["""的""", """人""", """有"""]
						A__				:       List[str]	  =						"""""".join(A__     )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Any	  =						True
								A__				:       int	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[Any]	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Tuple	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       int	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )

								A__				:       Optional[int]	  =						False
								A__				:       Union[str, Any]	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       str	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       Tuple	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Union[str, Any]	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       Tuple	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__				:       str	  =						[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__     )
								]
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )






					@slow
					def     __A     (					self     ):
						A__				:       str	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.encode("""你好"""       ,			add_special_tokens=A__     )
						A__				:       List[Any]	  =						tokenizer.encode("""你是谁"""       ,			add_special_tokens=A__     )

						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__     )
						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__       ,			A__     )

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]






					def     __A     (					self     ):
						A__				:       List[str]	  =						self.get_tokenizers(do_lower_case=A__     )
						for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""     ):
								A__				:       Optional[int]	  =						"""你好，你是谁"""
								A__				:       List[str]	  =						tokenizer.tokenize(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.convert_tokens_to_ids(A__     )
								A__				:       str	  =						tokenizer.convert_tokens_to_shape_ids(A__     )
								A__				:       Optional[int]	  =						tokenizer.convert_tokens_to_pronunciation_ids(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.prepare_for_model(
								    A__       ,			A__       ,			A__       ,			add_special_tokens=A__     )

								A__				:       int	  =						tokenizer.encode_plus(A__       ,			add_special_tokens=A__     )

								self.assertEqual(A__       ,			A__     )



