
"""simple docstring"""






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 					__a     ( __a					,   unittest.TestCase       ):



							'''simple docstring'''







							_lowerCamelCase					:			str          =   RoCBertTokenizer
							_lowerCamelCase					:			Optional[int]          =   None
							_lowerCamelCase					:			int          =   False
							_lowerCamelCase					:			Optional[Any]          =   True
							_lowerCamelCase					:			List[str]          =   filter_non_english
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										super().setUp()

										__lowercase       =       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
										__lowercase       =       {}
										__lowercase       =       {}
										for i, value in enumerate(_lowerCamelCase       ):
													__lowercase       =       i
													__lowercase       =       i
										__lowercase       =       os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES["vocab_file"]       )
										__lowercase       =       os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES["word_shape_file"]       )
										__lowercase       =       os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES["word_pronunciation_file"]       )
										with open(self.vocab_file  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )
										with open(self.word_shape_file  ,    "w"  ,    encoding="utf-8"       ) as word_shape_writer:
													json.dump(_lowerCamelCase  ,    _lowerCamelCase  ,    ensure_ascii=_lowerCamelCase       )
										with open(self.word_pronunciation_file  ,    "w"  ,    encoding="utf-8"       ) as word_pronunciation_writer:
													json.dump(_lowerCamelCase  ,    _lowerCamelCase  ,    ensure_ascii=_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       self.tokenizer_class(self.vocab_file  ,    self.word_shape_file  ,    self.word_pronunciation_file       )

										__lowercase       =       tokenizer.tokenize("你好[SEP]你是谁"       )
										self.assertListEqual(_lowerCamelCase  ,    ["你", "好", "[SEP]", "你", "是", "谁"]       )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase       )  ,    [5, 6, 2, 5, 7, 8]       )
										self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase       )  ,    [5, 6, 2, 5, 7, 8]       )
										self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase       )  ,    [5, 6, 2, 5, 7, 8]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )  ,    ["ah", "\u535A", "\u63A8", "zz"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )  ,    ["hello", "!", "how", "are", "you", "?"]       )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )  ,    ["hello"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,    strip_accents=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )  ,    ["hällo", "!", "how", "are", "you", "?"]       )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )  ,    ["h\u00E9llo"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,    strip_accents=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )  ,    ["hallo", "!", "how", "are", "you", "?"]       )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )  ,    ["hello"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )  ,    ["hallo", "!", "how", "are", "you", "?"]       )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )  ,    ["hello"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )  ,    ["HeLLo", "!", "how", "Are", "yoU", "?"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,    strip_accents=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )  ,    ["HäLLo", "!", "how", "Are", "yoU", "?"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[Any]:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,    strip_accents=_lowerCamelCase       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )  ,    ["HaLLo", "!", "how", "Are", "yoU", "?"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,    never_split=["[UNK]"]       )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )  ,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										__lowercase       =       {}
										for i, token in enumerate(_lowerCamelCase       ):
													__lowercase       =       i
										__lowercase       =       RoCBertWordpieceTokenizer(vocab=_lowerCamelCase  ,    unk_token="[UNK]"       )

										self.assertListEqual(tokenizer.tokenize(""       )  ,    []       )

										self.assertListEqual(tokenizer.tokenize("unwanted running"       )  ,    ["un", "##want", "##ed", "runn", "##ing"]       )

										self.assertListEqual(tokenizer.tokenize("unwantedX running"       )  ,    ["[UNK]", "runn", "##ing"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										self.assertTrue(_is_whitespace(" "       )       )
										self.assertTrue(_is_whitespace("\t"       )       )
										self.assertTrue(_is_whitespace("\r"       )       )
										self.assertTrue(_is_whitespace("\n"       )       )
										self.assertTrue(_is_whitespace("\u00A0"       )       )

										self.assertFalse(_is_whitespace("A"       )       )
										self.assertFalse(_is_whitespace("-"       )       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										self.assertTrue(_is_control("\u0005"       )       )

										self.assertFalse(_is_control("A"       )       )
										self.assertFalse(_is_control(" "       )       )
										self.assertFalse(_is_control("\t"       )       )
										self.assertFalse(_is_control("\r"       )       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[Any]:




										'''simple docstring'''
										self.assertTrue(_is_punctuation("-"       )       )
										self.assertTrue(_is_punctuation("$"       )       )
										self.assertTrue(_is_punctuation("`"       )       )
										self.assertTrue(_is_punctuation("."       )       )

										self.assertFalse(_is_punctuation("A"       )       )
										self.assertFalse(_is_punctuation(" "       )       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(_lowerCamelCase       ) for t in ["Test", "\xad", "test"]]  ,    [["[UNK]"], [], ["[UNK]"]]       )

										if self.test_rust_tokenizer:
													__lowercase       =       self.get_rust_tokenizer()
													self.assertListEqual(
													    [rust_tokenizer.tokenize(_lowerCamelCase       ) for t in ["Test", "\xad", "test"]]  ,    [["[UNK]"], [], ["[UNK]"]]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																__lowercase       =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )

																__lowercase       =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																__lowercase       =       tokenizer_r.encode_plus(
																    _lowerCamelCase  ,    return_attention_mask=_lowerCamelCase  ,    return_token_type_ids=_lowerCamelCase  ,    return_offsets_mapping=_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase  ,    )

																__lowercase       =       tokenizer_r.do_lower_case if hasattr(_lowerCamelCase  ,    "do_lower_case"       ) else False
																__lowercase       =       (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "A"),
																        ((1, 2), ","),
																        ((3, 5), "na"),
																        ((5, 6), "##ï"),
																        ((6, 8), "##ve"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "Allen"),
																        ((21, 23), "##NL"),
																        ((23, 24), "##P"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "a"),
																        ((1, 2), ","),
																        ((3, 8), "naive"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "allen"),
																        ((21, 23), "##nl"),
																        ((23, 24), "##p"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]  ,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
																self.assertEqual([e[0] for e in expected_results]  ,    tokens["offset_mapping"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[str]:




										'''simple docstring'''
										__lowercase       =       ["的", "人", "有"]
										__lowercase       =       "".join(_lowerCamelCase       )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																__lowercase       =       True
																__lowercase       =       self.tokenizer_class.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )
																__lowercase       =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )

																__lowercase       =       tokenizer_p.encode(_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )
																__lowercase       =       tokenizer_r.encode(_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )

																__lowercase       =       tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
																__lowercase       =       tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(_lowerCamelCase  ,    _lowerCamelCase       )
																self.assertListEqual(_lowerCamelCase  ,    _lowerCamelCase       )

																__lowercase       =       False
																__lowercase       =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )
																__lowercase       =       self.tokenizer_class.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )

																__lowercase       =       tokenizer_r.encode(_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )
																__lowercase       =       tokenizer_p.encode(_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )

																__lowercase       =       tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
																__lowercase       =       tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

																# it is expected that only the first Chinese character is not preceded by "##".
																__lowercase       =       [
																    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase       )
																]
																self.assertListEqual(_lowerCamelCase  ,    _lowerCamelCase       )
																self.assertListEqual(_lowerCamelCase  ,    _lowerCamelCase       )
							@slow
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[Any]:




										'''simple docstring'''
										__lowercase       =       self.tokenizer_class(self.vocab_file  ,    self.word_shape_file  ,    self.word_pronunciation_file       )

										__lowercase       =       tokenizer.encode("你好"  ,    add_special_tokens=_lowerCamelCase       )
										__lowercase       =       tokenizer.encode("你是谁"  ,    add_special_tokens=_lowerCamelCase       )

										__lowercase       =       tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       )
										__lowercase       =       tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,    _lowerCamelCase       )

										assert encoded_sentence == [1] + text + [2]
										assert encoded_pair == [1] + text + [2] + text_a + [2]





							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[Any]:




										'''simple docstring'''
										__lowercase       =       self.get_tokenizers(do_lower_case=_lowerCamelCase       )
										for tokenizer in tokenizers:
													with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																__lowercase       =       "你好，你是谁"
																__lowercase       =       tokenizer.tokenize(_lowerCamelCase       )
																__lowercase       =       tokenizer.convert_tokens_to_ids(_lowerCamelCase       )
																__lowercase       =       tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase       )
																__lowercase       =       tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase       )
																__lowercase       =       tokenizer.prepare_for_model(
																    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )

																__lowercase       =       tokenizer.encode_plus(_lowerCamelCase  ,    add_special_tokens=_lowerCamelCase       )

																self.assertEqual(_lowerCamelCase  ,    _lowerCamelCase       )







"""simple docstring"""






def      lowerCAmelCase__			(		__magic_name__ = 1_0      )						->str:
			if not isinstance(__magic_name__			,   __magic_name__      ) or n < 0:
						raise ValueError("Invalid input"      )
			__lowercase       =       1_0**n
			__lowercase       =       2_8_4_3_3 * (pow(2			,   7_8_3_0_4_5_7			,   __magic_name__      )) + 1
			return str(number % modulus      )


if __name__ == "__main__":
					from doctest import testmod

					testmod()
					print(F"{solution(10) = }")





