






"""simple docstring"""
import argparse
import struct
import unittest







class      UpperCamelCase_			:





	def __init__( self   :					List[str]      ,      lowerCAmelCase_   :					bytes		)		->     None:
				UpperCAmelCase_	:					Optional[Any]     	=   data

				# Initialize hash values
				UpperCAmelCase_	:					Union[str, Any]     	=   [
				    0X6A09_E667,
				    0XBB67_AE85,
				    0X3C6E_F372,
				    0XA54F_F53A,
				    0X510E_527F,
				    0X9B05_688C,
				    0X1F83_D9AB,
				    0X5BE0_CD19,
				]

				# Initialize round constants
				UpperCAmelCase_	:					Any     	=   [
				    0X428A_2F98,
				    0X7137_4491,
				    0XB5C0_FBCF,
				    0XE9B5_DBA5,
				    0X3956_C25B,
				    0X59F1_11F1,
				    0X923F_82A4,
				    0XAB1C_5ED5,
				    0XD807_AA98,
				    0X1283_5B01,
				    0X2431_85BE,
				    0X550C_7DC3,
				    0X72BE_5D74,
				    0X80DE_B1FE,
				    0X9BDC_06A7,
				    0XC19B_F174,
				    0XE49B_69C1,
				    0XEFBE_4786,
				    0X0FC1_9DC6,
				    0X240C_A1CC,
				    0X2DE9_2C6F,
				    0X4A74_84AA,
				    0X5CB0_A9DC,
				    0X76F9_88DA,
				    0X983E_5152,
				    0XA831_C66D,
				    0XB003_27C8,
				    0XBF59_7FC7,
				    0XC6E0_0BF3,
				    0XD5A7_9147,
				    0X06CA_6351,
				    0X1429_2967,
				    0X27B7_0A85,
				    0X2E1B_2138,
				    0X4D2C_6DFC,
				    0X5338_0D13,
				    0X650A_7354,
				    0X766A_0ABB,
				    0X81C2_C92E,
				    0X9272_2C85,
				    0XA2BF_E8A1,
				    0XA81A_664B,
				    0XC24B_8B70,
				    0XC76C_51A3,
				    0XD192_E819,
				    0XD699_0624,
				    0XF40E_3585,
				    0X106A_A070,
				    0X19A4_C116,
				    0X1E37_6C08,
				    0X2748_774C,
				    0X34B0_BCB5,
				    0X391C_0CB3,
				    0X4ED8_AA4A,
				    0X5B9C_CA4F,
				    0X682E_6FF3,
				    0X748F_82EE,
				    0X78A5_636F,
				    0X84C8_7814,
				    0X8CC7_0208,
				    0X90BE_FFFA,
				    0XA450_6CEB,
				    0XBEF9_A3F7,
				    0XC671_78F2,
				]

				UpperCAmelCase_	:					List[str]     	=   self.preprocessing(self.data		)
				self.final_hash()





	@staticmethod
	def 	_SCREAMING_SNAKE_CASE ( lowerCAmelCase_   :					bytes		)		->     bytes:
				UpperCAmelCase_	:					List[Any]     	=   b"\x80" + (b"\x00" * (63 - (len(lowerCAmelCase_		) + 8) % 64))
				UpperCAmelCase_	:					int     	=   struct.pack(">Q"      ,      (len(lowerCAmelCase_		) * 8)		)
				return data + padding + big_endian_integer





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     None:
				# Convert into blocks of 64 bytes
				UpperCAmelCase_	:					str     	=   [
				    self.preprocessed_data[x : x + 64]
				    for x in range(0      ,      len(self.preprocessed_data		)      ,      64		)
				]

				for block in self.blocks:
							# Convert the given block into a list of 4 byte integers
							UpperCAmelCase_	:					Optional[Any]     	=   list(struct.unpack(">16L"      ,      lowerCAmelCase_		)		)
							# add 48 0-ed integers
							words += [0] * 48

							UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	:					Union[str, Any]     	=   self.hashes

							for index in range(0      ,      64		):
										if index > 15:
													# modify the zero-ed indexes at the end of the array
													UpperCAmelCase_	:					Any     	=   (
													    self.ror(words[index - 15]      ,      7		)
													    ^ self.ror(words[index - 15]      ,      18		)
													    ^ (words[index - 15] >> 3)
													)
													UpperCAmelCase_	:					Dict     	=   (
													    self.ror(words[index - 2]      ,      17		)
													    ^ self.ror(words[index - 2]      ,      19		)
													    ^ (words[index - 2] >> 10)
													)

													UpperCAmelCase_	:					int     	=   (
													    words[index - 16] + sa + words[index - 7] + sa
													) % 0X1_0000_0000

										# Compression
										UpperCAmelCase_	:					Any     	=   self.ror(lowerCAmelCase_      ,      6		) ^ self.ror(lowerCAmelCase_      ,      11		) ^ self.ror(lowerCAmelCase_      ,      25		)
										UpperCAmelCase_	:					Optional[int]     	=   (e & f) ^ ((~e & 0XFFFF_FFFF) & g)
										UpperCAmelCase_	:					str     	=   (
										    h + sa + ch + self.round_constants[index] + words[index]
										) % 0X1_0000_0000
										UpperCAmelCase_	:					str     	=   self.ror(lowerCAmelCase_      ,      2		) ^ self.ror(lowerCAmelCase_      ,      13		) ^ self.ror(lowerCAmelCase_      ,      22		)
										UpperCAmelCase_	:					List[Any]     	=   (a & b) ^ (a & c) ^ (b & c)
										UpperCAmelCase_	:					Optional[Any]     	=   (sa + maj) % 0X1_0000_0000

										UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	,      UpperCAmelCase_	:					Union[str, Any]     	=   (
										    g,
										    f,
										    e,
										    ((d + tempa) % 0X1_0000_0000),
										    c,
										    b,
										    a,
										    ((tempa + tempa) % 0X1_0000_0000),
										)

							UpperCAmelCase_	:					Optional[Any]     	=   [a, b, c, d, e, f, g, h]

							# Modify final values
							UpperCAmelCase_	:					Any     	=   [
							    ((element + mutated_hash_values[index]) % 0X1_0000_0000)
							    for index, element in enumerate(self.hashes		)
							]

				UpperCAmelCase_	:					Optional[Any]     	=   "".join([hex(lowerCAmelCase_		)[2:].zfill(8		) for value in self.hashes]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]      ,      lowerCAmelCase_   :					int      ,      lowerCAmelCase_   :					int		)		->     int:
				return 0XFFFF_FFFF & (value << (32 - rotations)) | (value >> rotations)







class      UpperCamelCase_			(unittest.TestCase	):





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[int]		)		->     None:
				import hashlib

				UpperCAmelCase_	:					Union[str, Any]     	=   bytes("Test String"      ,      "utf-8"		)
				self.assertEqual(SHAaaa(lowerCAmelCase_		).hash      ,      hashlib.shaaaa(lowerCAmelCase_		).hexdigest()		)





def 							snake_case (  ):
			import doctest

			doctest.testmod()

			UpperCAmelCase_	:					Any     	=   argparse.ArgumentParser()
			parser.add_argument(
			    "-s"      ,"--string"      ,dest="input_string"      ,default="Hello World!! Welcome to Cryptography"      ,help="Hash the string"      ,)
			parser.add_argument(
			    "-f"      ,"--file"      ,dest="input_file"      ,help="Hash contents of a file"		)

			UpperCAmelCase_	:					Union[str, Any]     	=   parser.parse_args()

			UpperCAmelCase_	:					Optional[int]     	=   args.input_string

			# hash input should be a bytestring
			if args.input_file:
						with open(args.input_file      ,"rb"		) as f:
									UpperCAmelCase_	:					Dict     	=   f.read()
			else:
						UpperCAmelCase_	:					Union[str, Any]     	=   bytes(A__      ,"utf-8"		)

			print(SHAaaa(A__		).hash		)


if __name__ == "__main__":
			main()











"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class      UpperCamelCase_			(__A							,		unittest.TestCase	):
	__magic_name__       =							BertTokenizer
	__magic_name__       =							BertTokenizerFast
	__magic_name__       =							True
	__magic_name__       =							True
	__magic_name__       =							filter_non_english





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Optional[Any]:
				super().setUp()

				UpperCAmelCase_	:					Optional[int]     	=   [
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_	:					Optional[Any]     	=   os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["vocab_file"]		)
				with open(self.vocab_file      ,      "w"      ,      encoding="utf-8"		) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					List[str]		)		->     List[str]:
				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"
				UpperCAmelCase_	:					Optional[int]     	=   "unwanted, running"
				return input_text, output_text





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				UpperCAmelCase_	:					Optional[int]     	=   self.tokenizer_class(self.vocab_file		)

				UpperCAmelCase_	:					int     	=   tokenizer.tokenize("UNwant\u00E9d,running"		)
				self.assertListEqual(lowerCAmelCase_      ,      ["un", "##want", "##ed", ",", "runn", "##ing"]		)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)      ,      [9, 6, 7, 12, 10, 11]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     List[Any]:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer()

				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Any     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					List[Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					List[str]     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Dict     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					int     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Optional[Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				# With lower casing
				UpperCAmelCase_	:					int     	=   self.get_tokenizer(do_lower_case=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_		)

				UpperCAmelCase_	:					Any     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Dict     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Optional[int]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					Any     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					Dict     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     List[str]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)      ,      ["ah", "\u535A", "\u63A8", "zz"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Optional[int]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["hello", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Optional[Any]:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hällo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["h\u00E9llo"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     Any:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[str]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Union[str, Any]:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     str:
				UpperCAmelCase_	:					Any     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Dict:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      never_split=["[UNK]"]		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Union[str, Any]:
				UpperCAmelCase_	:					Optional[int]     	=   BasicTokenizer()
				UpperCAmelCase_	:					int     	=   "a\n'll !!to?'d of, can't."
				UpperCAmelCase_	:					Union[str, Any]     	=   ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_		)      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Dict:
				UpperCAmelCase_	:					List[str]     	=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_	:					int     	=   {}
				for i, token in enumerate(lowerCAmelCase_		):
							UpperCAmelCase_	:					Tuple     	=   i
				UpperCAmelCase_	:					Dict     	=   WordpieceTokenizer(vocab=lowerCAmelCase_      ,      unk_token="[UNK]"		)

				self.assertListEqual(tokenizer.tokenize(""		)      ,      []		)

				self.assertListEqual(tokenizer.tokenize("unwanted running"		)      ,      ["un", "##want", "##ed", "runn", "##ing"]		)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"		)      ,      ["[UNK]", "runn", "##ing"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				self.assertTrue(_is_whitespace(" "		)		)
				self.assertTrue(_is_whitespace("\t"		)		)
				self.assertTrue(_is_whitespace("\r"		)		)
				self.assertTrue(_is_whitespace("\n"		)		)
				self.assertTrue(_is_whitespace("\u00A0"		)		)

				self.assertFalse(_is_whitespace("A"		)		)
				self.assertFalse(_is_whitespace("-"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Any:
				self.assertTrue(_is_control("\u0005"		)		)

				self.assertFalse(_is_control("A"		)		)
				self.assertFalse(_is_control(" "		)		)
				self.assertFalse(_is_control("\t"		)		)
				self.assertFalse(_is_control("\r"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     int:
				self.assertTrue(_is_punctuation("-"		)		)
				self.assertTrue(_is_punctuation("$"		)		)
				self.assertTrue(_is_punctuation("`"		)		)
				self.assertTrue(_is_punctuation("."		)		)

				self.assertFalse(_is_punctuation("A"		)		)
				self.assertFalse(_is_punctuation(" "		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					List[Any]     	=   self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)





	@slow
	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     List[Any]:
				UpperCAmelCase_	:					str     	=   self.tokenizer_class.from_pretrained("bert-base-uncased"		)

				UpperCAmelCase_	:					Optional[Any]     	=   tokenizer.encode("sequence builders"      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   tokenizer.encode("multi-sequence build"      ,      add_special_tokens=lowerCAmelCase_		)

				UpperCAmelCase_	:					str     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		)
				UpperCAmelCase_	:					Tuple     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_      ,      lowerCAmelCase_		)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[int]		)		->     List[str]:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Optional[int]     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_	:					Dict     	=   tokenizer_r.encode_plus(
										    lowerCAmelCase_      ,      return_attention_mask=lowerCAmelCase_      ,      return_token_type_ids=lowerCAmelCase_      ,      return_offsets_mapping=lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_      ,      )

										UpperCAmelCase_	:					int     	=   tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_      ,      "do_lower_case"		) else False
										UpperCAmelCase_	:					Dict     	=   (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]      ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
										self.assertEqual([e[0] for e in expected_results]      ,      tokens["offset_mapping"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					Dict     	=   ["的", "人", "有"]
				UpperCAmelCase_	:					Union[str, Any]     	=   "".join(lowerCAmelCase_		)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Tuple     	=   True
										UpperCAmelCase_	:					Optional[Any]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					Dict     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					str     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   False
										UpperCAmelCase_	:					str     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					int     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_	:					Dict     	=   [
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_		)
										]
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)



