






import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
							from transformers import WhisperFeatureExtractor

if is_torch_available():
							import torch

lowercase  :							Tuple			  =    random.Random()

def      lowerCAmelCase__	(    _a       :			Optional[int]  ,      _a       :			Union[str, Any]=1.0  ,      _a       :			Any=None  ,      _a       :			Optional[Any]=None    ):
		if rng is None:
				snake_case_				:							Union[str, Any]						 =       global_rng

		snake_case_				:							str						 =       []
		for batch_idx in range(shape[0]    ):
				values.append([]    )
				for _ in range(shape[1]    ):
						values[-1].append(rng.random() * scale    )

		return values




@require_torch
@require_torchaudio
class 		UpperCAmelCase_						(	unittest.TestCase					):


						'''simple docstring'''




						def __init__(	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=7		,				_SCREAMING_SNAKE_CASE=400		,				_SCREAMING_SNAKE_CASE=2000		,				_SCREAMING_SNAKE_CASE=10		,				_SCREAMING_SNAKE_CASE=160		,				_SCREAMING_SNAKE_CASE=8		,				_SCREAMING_SNAKE_CASE=0.0		,				_SCREAMING_SNAKE_CASE=4000		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=True		,				) ->				str:
								snake_case_				:							Union[str, Any]						 =       parent
								snake_case_				:							Any						 =       batch_size
								snake_case_				:							str						 =       min_seq_length
								snake_case_				:							Tuple						 =       max_seq_length
								snake_case_				:							List[str]						 =       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
								snake_case_				:							List[str]						 =       padding_value
								snake_case_				:							List[Any]						 =       sampling_rate
								snake_case_				:							List[Any]						 =       return_attention_mask
								snake_case_				:							Optional[int]						 =       do_normalize
								snake_case_				:							List[Any]						 =       feature_size
								snake_case_				:							List[str]						 =       chunk_length
								snake_case_				:							Dict						 =       hop_length






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								return {
								    "feature_size": self.feature_size,
								    "hop_length": self.hop_length,
								    "chunk_length": self.chunk_length,
								    "padding_value": self.padding_value,
								    "sampling_rate": self.sampling_rate,
								    "return_attention_mask": self.return_attention_mask,
								    "do_normalize": self.do_normalize,
								}






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=False				) ->				List[str]:
								def _flatten(_SCREAMING_SNAKE_CASE				):
										return list(itertools.chain(*_SCREAMING_SNAKE_CASE				)				)

								if equal_length:
										snake_case_				:							Optional[Any]						 =       [floats_list((self.max_seq_length, self.feature_size)				) for _ in range(self.batch_size				)]
								else:
										# make sure that inputs increase in size
										snake_case_				:							List[str]						 =       [
										    floats_list((x, self.feature_size)				)
										    for x in range(self.min_seq_length		,				self.max_seq_length		,				self.seq_length_diff				)
										]
								if numpify:
										snake_case_				:							Tuple						 =       [np.asarray(_SCREAMING_SNAKE_CASE				) for x in speech_inputs]
								return speech_inputs




@require_torch
@require_torchaudio
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   Optional[Any]						=					WhisperFeatureExtractor if is_speech_available() else None






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							Tuple						 =       WhisperFeatureExtractionTester(self				)






						def       _lowerCAmelCase       (	self				) ->				Any:
								snake_case_				:							List[Any]						 =       self.feature_extraction_class(**self.feat_extract_dict				)

								with tempfile.TemporaryDirectory() as tmpdirname:
										snake_case_				:							int						 =       feat_extract_first.save_pretrained(_SCREAMING_SNAKE_CASE				)[0]
										check_json_file_has_correct_format(_SCREAMING_SNAKE_CASE				)
										snake_case_				:							List[Any]						 =       self.feature_extraction_class.from_pretrained(_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[Any]						 =       feat_extract_first.to_dict()
								snake_case_				:							Optional[int]						 =       feat_extract_second.to_dict()
								snake_case_				:							str						 =       feat_extract_first.mel_filters
								snake_case_				:							List[str]						 =       feat_extract_second.mel_filters
								self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)				)
								self.assertEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							Tuple						 =       self.feature_extraction_class(**self.feat_extract_dict				)

								with tempfile.TemporaryDirectory() as tmpdirname:
										snake_case_				:							Optional[int]						 =       os.path.join(_SCREAMING_SNAKE_CASE		,				"feat_extract.json"				)
										feat_extract_first.to_json_file(_SCREAMING_SNAKE_CASE				)
										snake_case_				:							Union[str, Any]						 =       self.feature_extraction_class.from_json_file(_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       feat_extract_first.to_dict()
								snake_case_				:							Any						 =       feat_extract_second.to_dict()
								snake_case_				:							Optional[int]						 =       feat_extract_first.mel_filters
								snake_case_				:							Any						 =       feat_extract_second.mel_filters
								self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)				)
								self.assertEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								# Tests that all call wrap to encode_plus and batch_encode_plus
								snake_case_				:							Dict						 =       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()				)
								# create three inputs of length 800, 1000, and 1200
								snake_case_				:							str						 =       [floats_list((1, x)				)[0] for x in range(800		,				1400		,				200				)]
								snake_case_				:							Any						 =       [np.asarray(_SCREAMING_SNAKE_CASE				) for speech_input in speech_inputs]

								# Test feature size
								snake_case_				:							Dict						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				padding="max_length"		,				return_tensors="np"				).input_features
								self.assertTrue(input_features.ndim == 3				)
								self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames				)
								self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size				)

								# Test not batched input
								snake_case_				:							Tuple						 =       feature_extractor(speech_inputs[0]		,				return_tensors="np"				).input_features
								snake_case_				:							Union[str, Any]						 =       feature_extractor(np_speech_inputs[0]		,				return_tensors="np"				).input_features
								self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				atol=1e-3				)				)

								# Test batched
								snake_case_				:							Dict						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								snake_case_				:							List[str]						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				):
										self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				atol=1e-3				)				)

								# Test 2-D numpy arrays are batched.
								snake_case_				:							int						 =       [floats_list((1, x)				)[0] for x in (800, 800, 800)]
								snake_case_				:							Optional[int]						 =       np.asarray(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								snake_case_				:							List[Any]						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				):
										self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				atol=1e-3				)				)

								# Test truncation required
								snake_case_				:							str						 =       [floats_list((1, x)				)[0] for x in range(200		,				(feature_extractor.n_samples + 500)		,				200				)]
								snake_case_				:							str						 =       [np.asarray(_SCREAMING_SNAKE_CASE				) for speech_input in speech_inputs]

								snake_case_				:							Tuple						 =       [x[: feature_extractor.n_samples] for x in speech_inputs]
								snake_case_				:							Any						 =       [np.asarray(_SCREAMING_SNAKE_CASE				) for speech_input in speech_inputs_truncated]

								snake_case_				:							Union[str, Any]						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								snake_case_				:							List[str]						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="np"				).input_features
								for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				):
										self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				atol=1e-3				)				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								import torch

								snake_case_				:							Optional[Any]						 =       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()				)
								snake_case_				:							str						 =       np.random.rand(100		,				32				).astype(np.floataa				)
								snake_case_				:							Union[str, Any]						 =       np_speech_inputs.tolist()

								for inputs in [py_speech_inputs, np_speech_inputs]:
										snake_case_				:							List[str]						 =       feature_extractor.pad([{"input_features": inputs}]		,				return_tensors="np"				)
										self.assertTrue(np_processed.input_features.dtype == np.floataa				)
										snake_case_				:							str						 =       feature_extractor.pad([{"input_features": inputs}]		,				return_tensors="pt"				)
										self.assertTrue(pt_processed.input_features.dtype == torch.floataa				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Union[str, Any]:
								snake_case_				:							Optional[Any]						 =       load_dataset("hf-internal-testing/librispeech_asr_dummy"		,				"clean"		,				split="validation"				)
								# automatic decoding with librispeech
								snake_case_				:							int						 =       ds.sort("id"				).select(range(_SCREAMING_SNAKE_CASE				)				)[:num_samples]["audio"]

								return [x["array"] for x in speech_samples]






						def       _lowerCAmelCase       (	self				) ->				Any:
								# fmt: off
								snake_case_				:							Any						 =       torch.tensor(
								    [
								        0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
								        0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
								        0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
								        -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
								    ]				)
								# fmt: on

								snake_case_				:							str						 =       self._load_datasamples(1				)
								snake_case_				:							List[Any]						 =       WhisperFeatureExtractor()
								snake_case_				:							Optional[Any]						 =       feature_extractor(_SCREAMING_SNAKE_CASE		,				return_tensors="pt"				).input_features
								self.assertEqual(input_features.shape		,				(1, 80, 3000)				)
								self.assertTrue(torch.allclose(input_features[0, 0, :30]		,				_SCREAMING_SNAKE_CASE		,				atol=1e-4				)				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							str						 =       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()				)
								snake_case_				:							List[Any]						 =       self._load_datasamples(1				)[0]
								snake_case_				:							Optional[Any]						 =       ((audio - audio.min()) / (audio.max() - audio.min())) * 6_5535  # Rescale to [0, 65535] to show issue
								snake_case_				:							int						 =       feat_extract.zero_mean_unit_var_norm([audio]		,				attention_mask=_SCREAMING_SNAKE_CASE				)[0]

								self.assertTrue(np.all(np.mean(_SCREAMING_SNAKE_CASE				) < 1e-3				)				)
								self.assertTrue(np.all(np.abs(np.var(_SCREAMING_SNAKE_CASE				) - 1				) < 1e-3				)				)








import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   str						=					BertTokenizer
						A     :   Union[str, Any]						=					BertTokenizerFast
						A     :   int						=					True
						A     :   Union[str, Any]						=					True
						A     :   Union[str, Any]						=					filter_non_english






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								super().setUp()

								snake_case_				:							Optional[int]						 =       [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "[PAD]",
								    "[MASK]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								snake_case_				:							str						 =       os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["vocab_file"]				)
								with open(self.vocab_file		,				"w"		,				encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Tuple:
								snake_case_				:							str						 =       "UNwant\u00E9d,running"
								snake_case_				:							Optional[int]						 =       "unwanted, running"
								return input_text, output_text






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							str						 =       self.tokenizer_class(self.vocab_file				)

								snake_case_				:							Tuple						 =       tokenizer.tokenize("UNwant\u00E9d,running"				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				["un", "##want", "##ed", ",", "runn", "##ing"]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)		,				[9, 6, 7, 12, 10, 11]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								if not self.test_rust_tokenizer:
										return

								snake_case_				:							Any						 =       self.get_tokenizer()
								snake_case_				:							str						 =       self.get_rust_tokenizer()

								snake_case_				:							List[Any]						 =       "UNwant\u00E9d,running"

								snake_case_				:							Union[str, Any]						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							str						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							int						 =       self.get_rust_tokenizer()
								snake_case_				:							Any						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								# With lower casing
								snake_case_				:							Tuple						 =       self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Tuple						 =       "UNwant\u00E9d,running"

								snake_case_				:							Tuple						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[Any]						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[int]						 =       self.get_rust_tokenizer()
								snake_case_				:							Dict						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								snake_case_				:							Union[str, Any]						 =       BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)		,				["ah", "\u535A", "\u63A8", "zz"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["hello", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hällo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["h\u00E9llo"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Dict						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["HeLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				int:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HäLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HaLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							Any						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				never_split=["[UNK]"]				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)		,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer()
								snake_case_				:							Optional[Any]						 =       "a\n'll !!to?'d of, can't."
								snake_case_				:							Tuple						 =       ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
								self.assertListEqual(tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[Any]						 =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								snake_case_				:							Optional[int]						 =       {}
								for i, token in enumerate(_SCREAMING_SNAKE_CASE				):
										snake_case_				:							Any						 =       i
								snake_case_				:							Any						 =       WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE		,				unk_token="[UNK]"				)

								self.assertListEqual(tokenizer.tokenize(""				)		,				[]				)

								self.assertListEqual(tokenizer.tokenize("unwanted running"				)		,				["un", "##want", "##ed", "runn", "##ing"]				)

								self.assertListEqual(tokenizer.tokenize("unwantedX running"				)		,				["[UNK]", "runn", "##ing"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_whitespace(" "				)				)
								self.assertTrue(_is_whitespace("\t"				)				)
								self.assertTrue(_is_whitespace("\r"				)				)
								self.assertTrue(_is_whitespace("\n"				)				)
								self.assertTrue(_is_whitespace("\u00A0"				)				)

								self.assertFalse(_is_whitespace("A"				)				)
								self.assertFalse(_is_whitespace("-"				)				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_control("\u0005"				)				)

								self.assertFalse(_is_control("A"				)				)
								self.assertFalse(_is_control(" "				)				)
								self.assertFalse(_is_control("\t"				)				)
								self.assertFalse(_is_control("\r"				)				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								self.assertTrue(_is_punctuation("-"				)				)
								self.assertTrue(_is_punctuation("$"				)				)
								self.assertTrue(_is_punctuation("`"				)				)
								self.assertTrue(_is_punctuation("."				)				)

								self.assertFalse(_is_punctuation("A"				)				)
								self.assertFalse(_is_punctuation(" "				)				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Dict						 =       self.get_tokenizer()
								snake_case_				:							Any						 =       self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)






						@slow
						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained("bert-base-uncased"				)

								snake_case_				:							Tuple						 =       tokenizer.encode("sequence builders"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							List[Any]						 =       tokenizer.encode("multi-sequence build"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]






						def       _lowerCAmelCase       (	self				) ->				int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							Union[str, Any]						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[Any]						 =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
												snake_case_				:							List[Any]						 =       tokenizer_r.encode_plus(
												    _SCREAMING_SNAKE_CASE		,				return_attention_mask=_SCREAMING_SNAKE_CASE		,				return_token_type_ids=_SCREAMING_SNAKE_CASE		,				return_offsets_mapping=_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE		,				)

												snake_case_				:							Optional[Any]						 =       tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE		,				"do_lower_case"				) else False
												snake_case_				:							Union[str, Any]						 =       (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "A"),
												        ((1, 2), ","),
												        ((3, 5), "na"),
												        ((5, 6), "##ï"),
												        ((6, 8), "##ve"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "Allen"),
												        ((21, 23), "##NL"),
												        ((23, 24), "##P"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "a"),
												        ((1, 2), ","),
												        ((3, 8), "naive"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "allen"),
												        ((21, 23), "##nl"),
												        ((23, 24), "##p"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]		,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
												self.assertEqual([e[0] for e in expected_results]		,				tokens["offset_mapping"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       ["的", "人", "有"]
								snake_case_				:							int						 =       "".join(_SCREAMING_SNAKE_CASE				)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							int						 =       True
												snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[str]						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Union[str, Any]						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Tuple						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Optional[int]						 =       False
												snake_case_				:							Dict						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							List[Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							str						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Any						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that only the first Chinese character is not preceded by "##".
												snake_case_				:							int						 =       [
												    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE				)
												]
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
