






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class      lowercase__     (			_UpperCAmelCase,					unittest.TestCase	):
	a_         =RoCBertTokenizer
	a_         =None
	a_         =False
	a_         =True
	a_         =filter_non_english

	def 		UpperCAmelCase  (							self       )->					Tuple:







							'''simple docstring'''





							super().setUp()

							lowerCAmelCase__       =						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
							lowerCAmelCase__       =						{}
							lowerCAmelCase__       =						{}
							for i, value in enumerate(__UpperCAmelCase       ):
													lowerCAmelCase__       =						i
													lowerCAmelCase__       =						i
							lowerCAmelCase__       =						os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]       )
							lowerCAmelCase__       =						os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["word_shape_file"]       )
							lowerCAmelCase__       =						os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["word_pronunciation_file"]       )
							with open(self.vocab_file		,    "w"		,    encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )
							with open(self.word_shape_file		,    "w"		,    encoding="utf-8"       ) as word_shape_writer:
													json.dump(__UpperCAmelCase		,    __UpperCAmelCase		,    ensure_ascii=__UpperCAmelCase       )
							with open(self.word_pronunciation_file		,    "w"		,    encoding="utf-8"       ) as word_pronunciation_writer:
													json.dump(__UpperCAmelCase		,    __UpperCAmelCase		,    ensure_ascii=__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						self.tokenizer_class(self.vocab_file		,    self.word_shape_file		,    self.word_pronunciation_file       )

							lowerCAmelCase__       =						tokenizer.tokenize("你好[SEP]你是谁"       )
							self.assertListEqual(__UpperCAmelCase		,    ["你", "好", "[SEP]", "你", "是", "谁"]       )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase       )		,    [5, 6, 2, 5, 7, 8]       )
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__UpperCAmelCase       )		,    [5, 6, 2, 5, 7, 8]       )
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__UpperCAmelCase       )		,    [5, 6, 2, 5, 7, 8]       )

	def 		UpperCAmelCase  (							self       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )		,    ["ah", "\u535A", "\u63A8", "zz"]       )

	def 		UpperCAmelCase  (							self       )->					Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )		,    ["hello", "!", "how", "are", "you", "?"]       )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,    ["hello"]       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase		,    strip_accents=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,    ["hällo", "!", "how", "are", "you", "?"]       )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,    ["h\u00E9llo"]       )

	def 		UpperCAmelCase  (							self       )->					List[str]:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase		,    strip_accents=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,    ["hallo", "!", "how", "are", "you", "?"]       )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,    ["hello"]       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,    ["hallo", "!", "how", "are", "you", "?"]       )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,    ["hello"]       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]       )

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase		,    strip_accents=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]       )

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase		,    strip_accents=__UpperCAmelCase       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]       )

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						RoCBertBasicTokenizer(do_lower_case=__UpperCAmelCase		,    never_split=["[UNK]"]       )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )

	def 		UpperCAmelCase  (							self       )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							lowerCAmelCase__       =						{}
							for i, token in enumerate(__UpperCAmelCase       ):
													lowerCAmelCase__       =						i
							lowerCAmelCase__       =						RoCBertWordpieceTokenizer(vocab=__UpperCAmelCase		,    unk_token="[UNK]"       )

							self.assertListEqual(tokenizer.tokenize(""       )		,    []       )

							self.assertListEqual(tokenizer.tokenize("unwanted running"       )		,    ["un", "##want", "##ed", "runn", "##ing"]       )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"       )		,    ["[UNK]", "runn", "##ing"]       )

	def 		UpperCAmelCase  (							self       )->					Tuple:







							'''simple docstring'''





							self.assertTrue(_is_whitespace(" "       )       )
							self.assertTrue(_is_whitespace("\t"       )       )
							self.assertTrue(_is_whitespace("\r"       )       )
							self.assertTrue(_is_whitespace("\n"       )       )
							self.assertTrue(_is_whitespace("\u00A0"       )       )

							self.assertFalse(_is_whitespace("A"       )       )
							self.assertFalse(_is_whitespace("-"       )       )

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							self.assertTrue(_is_control("\u0005"       )       )

							self.assertFalse(_is_control("A"       )       )
							self.assertFalse(_is_control(" "       )       )
							self.assertFalse(_is_control("\t"       )       )
							self.assertFalse(_is_control("\r"       )       )

	def 		UpperCAmelCase  (							self       )->					List[str]:







							'''simple docstring'''





							self.assertTrue(_is_punctuation("-"       )       )
							self.assertTrue(_is_punctuation("$"       )       )
							self.assertTrue(_is_punctuation("`"       )       )
							self.assertTrue(_is_punctuation("."       )       )

							self.assertFalse(_is_punctuation("A"       )       )
							self.assertFalse(_is_punctuation(" "       )       )

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__UpperCAmelCase       ) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]       )

							if self.test_rust_tokenizer:
													lowerCAmelCase__       =						self.get_rust_tokenizer()
													self.assertListEqual(
													    [rust_tokenizer.tokenize(__UpperCAmelCase       ) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]       )

	def 		UpperCAmelCase  (							self       )->					Tuple:







							'''simple docstring'''





							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowerCAmelCase__       =						self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )

																			lowerCAmelCase__       =						F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																			lowerCAmelCase__       =						tokenizer_r.encode_plus(
																			    __UpperCAmelCase		,    return_attention_mask=__UpperCAmelCase		,    return_token_type_ids=__UpperCAmelCase		,    return_offsets_mapping=__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase		,    )

																			lowerCAmelCase__       =						tokenizer_r.do_lower_case if hasattr(__UpperCAmelCase		,    "do_lower_case"       ) else False
																			lowerCAmelCase__       =						(
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "A"),
																			        ((1, 2), ","),
																			        ((3, 5), "na"),
																			        ((5, 6), "##ï"),
																			        ((6, 8), "##ve"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "Allen"),
																			        ((21, 23), "##NL"),
																			        ((23, 24), "##P"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "a"),
																			        ((1, 2), ","),
																			        ((3, 8), "naive"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "allen"),
																			        ((21, 23), "##nl"),
																			        ((23, 24), "##p"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
																			self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]       )

	def 		UpperCAmelCase  (							self       )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						["的", "人", "有"]
							lowerCAmelCase__       =						"".join(__UpperCAmelCase       )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowerCAmelCase__       =						True
																			lowerCAmelCase__       =						self.tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )
																			lowerCAmelCase__       =						self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )

																			lowerCAmelCase__       =						tokenizer_p.encode(__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer_r.encode(__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )

																			lowerCAmelCase__       =						tokenizer_r.convert_ids_to_tokens(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer_p.convert_ids_to_tokens(__UpperCAmelCase       )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )
																			self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )

																			lowerCAmelCase__       =						False
																			lowerCAmelCase__       =						self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )
																			lowerCAmelCase__       =						self.tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )

																			lowerCAmelCase__       =						tokenizer_r.encode(__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer_p.encode(__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )

																			lowerCAmelCase__       =						tokenizer_r.convert_ids_to_tokens(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer_p.convert_ids_to_tokens(__UpperCAmelCase       )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			lowerCAmelCase__       =						[
																			    F"##{token}" if idx != 0 else token for idx, token in enumerate(__UpperCAmelCase       )
																			]
																			self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )
																			self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )

	@slow
	def 		UpperCAmelCase  (							self       )->					List[str]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.tokenizer_class(self.vocab_file		,    self.word_shape_file		,    self.word_pronunciation_file       )

							lowerCAmelCase__       =						tokenizer.encode("你好"		,    add_special_tokens=__UpperCAmelCase       )
							lowerCAmelCase__       =						tokenizer.encode("你是谁"		,    add_special_tokens=__UpperCAmelCase       )

							lowerCAmelCase__       =						tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase       )
							lowerCAmelCase__       =						tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase		,    __UpperCAmelCase       )

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]


	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						self.get_tokenizers(do_lower_case=__UpperCAmelCase       )
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"       ):
																			lowerCAmelCase__       =						"你好，你是谁"
																			lowerCAmelCase__       =						tokenizer.tokenize(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer.convert_tokens_to_ids(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer.convert_tokens_to_shape_ids(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer.convert_tokens_to_pronunciation_ids(__UpperCAmelCase       )
																			lowerCAmelCase__       =						tokenizer.prepare_for_model(
																			    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )

																			lowerCAmelCase__       =						tokenizer.encode_plus(__UpperCAmelCase		,    add_special_tokens=__UpperCAmelCase       )

																			self.assertEqual(__UpperCAmelCase		,    __UpperCAmelCase       )














# this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.:
#   python ./utils/get_modified_files.py utils src tests examples
#
# it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered
# since the output of this script is fed into Makefile commands it doesn't print a newline after the results

import re
import subprocess
import sys


a_							=    subprocess.check_output('''git merge-base main HEAD'''.split()).decode('''utf-8''')
a_							=    subprocess.check_output(F"git diff --name-only {fork_point_sha}".split()).decode('''utf-8''').split()

a_							=    '''|'''.join(sys.argv[1:])
a_							=    re.compile(rF"^({joined_dirs}).*?\.py$")

a_							=    [x for x in modified_files if regex.match(x)]
print(''' '''.join(relevant_modified_files), end='''''')






