






import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch




class 		UpperCAmelCase_						(	unittest.TestCase					):


						'''simple docstring'''




						A     :   List[str]						=					JukeboxTokenizer
						A     :   List[str]						=					{
						    'artist': 'Zac Brown Band',
						    'genres': 'Country',
						    'lyrics': 'I met a traveller from an antique land,\n        Who said "Two vast and trunkless legs of stone\n        Stand in the desert. . . . Near them, on the sand,\n        Half sunk a shattered visage lies, whose frown,\n        And wrinkled lip, and sneer of cold command,\n        Tell that its sculptor well those passions read\n        Which yet survive, stamped on these lifeless things,\n        The hand that mocked them, and the heart that fed;\n        And on the pedestal, these words appear:\n        My name is Ozymandias, King of Kings;\n        Look on my Works, ye Mighty, and despair!\n        Nothing beside remains. Round the decay\n        Of that colossal Wreck, boundless and bare\n        The lone and level sands stretch far away\n        ',
						}






						@require_torch
						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								import torch

								snake_case_				:							int						 =       JukeboxTokenizer.from_pretrained("openai/jukebox-1b-lyrics"				)
								snake_case_				:							List[str]						 =       tokenizer(**self.metas				)["input_ids"]
								# fmt: off
								snake_case_				:							Tuple						 =       [
								    torch.tensor([[
								        0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
								        76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
								        44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
								        47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
								        76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
								        30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
								        27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
								        45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
								        41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
								        76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
								        76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
								        64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
								        30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
								        27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
								        34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
								        27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
								        41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
								        44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
								        76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
								        32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
								        40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
								        45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
								        31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
								        45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
								        34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
								        31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
								        40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
								        38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
								        76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
								        41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
								        27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
								        46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
								        76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
								        41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
								        46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
								        41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
								        40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
								        27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
								        76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
								        76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
								        41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
								        76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
								        27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
								        34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
								        44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
								        40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
								        78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
								        46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
								        38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
								        40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
								        27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
								        20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
								        76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
								        76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
								        76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
								        76, 76]]				),
								    torch.tensor([[0, 0, 0, 1069, 11]]				),
								    torch.tensor([[0, 0, 0, 1069, 11]]				),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]				)				)
								self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]				)				)
								self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]				)				)






						@require_torch
						def       _lowerCAmelCase       (	self				) ->				str:
								import torch

								snake_case_				:							Optional[Any]						 =       JukeboxTokenizer.from_pretrained("openai/jukebox-5b-lyrics"				)
								snake_case_				:							Union[str, Any]						 =       tokenizer(**self.metas				)["input_ids"]
								# fmt: off
								snake_case_				:							Optional[Any]						 =       [
								    torch.tensor([[
								        0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
								        31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
								        31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
								        40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
								        79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
								        77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
								        27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
								        37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
								        32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
								        77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
								        77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
								        46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
								        77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
								        77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
								        77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
								        77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
								        64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
								        40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
								        40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
								        38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
								        31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
								        41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
								        46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
								        41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
								        31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
								        31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
								        44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
								        31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
								        38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
								        40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
								        27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
								        31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
								        34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
								        31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
								        1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
								        31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
								        45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
								        31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
								        15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
								        11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
								        45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
								        41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
								        44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
								        46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
								        27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
								        35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
								        77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
								        31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
								        77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
								        41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
								        77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
								        40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
								        77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
								        77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
								        27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
								        77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
								        77, 77, 77, 77, 77, 77]]				),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]				),
								    torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]				),
								]
								# fmt: on
								self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]				)				)
								self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]				)				)
								self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]				)				)








import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   str						=					BertTokenizer
						A     :   Union[str, Any]						=					BertTokenizerFast
						A     :   int						=					True
						A     :   Union[str, Any]						=					True
						A     :   Union[str, Any]						=					filter_non_english






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								super().setUp()

								snake_case_				:							Optional[int]						 =       [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "[PAD]",
								    "[MASK]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								snake_case_				:							str						 =       os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["vocab_file"]				)
								with open(self.vocab_file		,				"w"		,				encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Tuple:
								snake_case_				:							str						 =       "UNwant\u00E9d,running"
								snake_case_				:							Optional[int]						 =       "unwanted, running"
								return input_text, output_text






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							str						 =       self.tokenizer_class(self.vocab_file				)

								snake_case_				:							Tuple						 =       tokenizer.tokenize("UNwant\u00E9d,running"				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				["un", "##want", "##ed", ",", "runn", "##ing"]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)		,				[9, 6, 7, 12, 10, 11]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								if not self.test_rust_tokenizer:
										return

								snake_case_				:							Any						 =       self.get_tokenizer()
								snake_case_				:							str						 =       self.get_rust_tokenizer()

								snake_case_				:							List[Any]						 =       "UNwant\u00E9d,running"

								snake_case_				:							Union[str, Any]						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							str						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							int						 =       self.get_rust_tokenizer()
								snake_case_				:							Any						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								# With lower casing
								snake_case_				:							Tuple						 =       self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Tuple						 =       "UNwant\u00E9d,running"

								snake_case_				:							Tuple						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[Any]						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[int]						 =       self.get_rust_tokenizer()
								snake_case_				:							Dict						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								snake_case_				:							Union[str, Any]						 =       BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)		,				["ah", "\u535A", "\u63A8", "zz"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["hello", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hällo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["h\u00E9llo"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Dict						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["HeLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				int:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HäLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HaLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							Any						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				never_split=["[UNK]"]				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)		,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer()
								snake_case_				:							Optional[Any]						 =       "a\n'll !!to?'d of, can't."
								snake_case_				:							Tuple						 =       ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
								self.assertListEqual(tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[Any]						 =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								snake_case_				:							Optional[int]						 =       {}
								for i, token in enumerate(_SCREAMING_SNAKE_CASE				):
										snake_case_				:							Any						 =       i
								snake_case_				:							Any						 =       WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE		,				unk_token="[UNK]"				)

								self.assertListEqual(tokenizer.tokenize(""				)		,				[]				)

								self.assertListEqual(tokenizer.tokenize("unwanted running"				)		,				["un", "##want", "##ed", "runn", "##ing"]				)

								self.assertListEqual(tokenizer.tokenize("unwantedX running"				)		,				["[UNK]", "runn", "##ing"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_whitespace(" "				)				)
								self.assertTrue(_is_whitespace("\t"				)				)
								self.assertTrue(_is_whitespace("\r"				)				)
								self.assertTrue(_is_whitespace("\n"				)				)
								self.assertTrue(_is_whitespace("\u00A0"				)				)

								self.assertFalse(_is_whitespace("A"				)				)
								self.assertFalse(_is_whitespace("-"				)				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_control("\u0005"				)				)

								self.assertFalse(_is_control("A"				)				)
								self.assertFalse(_is_control(" "				)				)
								self.assertFalse(_is_control("\t"				)				)
								self.assertFalse(_is_control("\r"				)				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								self.assertTrue(_is_punctuation("-"				)				)
								self.assertTrue(_is_punctuation("$"				)				)
								self.assertTrue(_is_punctuation("`"				)				)
								self.assertTrue(_is_punctuation("."				)				)

								self.assertFalse(_is_punctuation("A"				)				)
								self.assertFalse(_is_punctuation(" "				)				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Dict						 =       self.get_tokenizer()
								snake_case_				:							Any						 =       self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)






						@slow
						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained("bert-base-uncased"				)

								snake_case_				:							Tuple						 =       tokenizer.encode("sequence builders"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							List[Any]						 =       tokenizer.encode("multi-sequence build"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]






						def       _lowerCAmelCase       (	self				) ->				int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							Union[str, Any]						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[Any]						 =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
												snake_case_				:							List[Any]						 =       tokenizer_r.encode_plus(
												    _SCREAMING_SNAKE_CASE		,				return_attention_mask=_SCREAMING_SNAKE_CASE		,				return_token_type_ids=_SCREAMING_SNAKE_CASE		,				return_offsets_mapping=_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE		,				)

												snake_case_				:							Optional[Any]						 =       tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE		,				"do_lower_case"				) else False
												snake_case_				:							Union[str, Any]						 =       (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "A"),
												        ((1, 2), ","),
												        ((3, 5), "na"),
												        ((5, 6), "##ï"),
												        ((6, 8), "##ve"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "Allen"),
												        ((21, 23), "##NL"),
												        ((23, 24), "##P"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "a"),
												        ((1, 2), ","),
												        ((3, 8), "naive"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "allen"),
												        ((21, 23), "##nl"),
												        ((23, 24), "##p"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]		,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
												self.assertEqual([e[0] for e in expected_results]		,				tokens["offset_mapping"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       ["的", "人", "有"]
								snake_case_				:							int						 =       "".join(_SCREAMING_SNAKE_CASE				)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							int						 =       True
												snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[str]						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Union[str, Any]						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Tuple						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Optional[int]						 =       False
												snake_case_				:							Dict						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							List[Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							str						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Any						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that only the first Chinese character is not preceded by "##".
												snake_case_				:							int						 =       [
												    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE				)
												]
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
