





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class     UpperCamelCase__					(     lowerCamelCase__				, unittest.TestCase				):






		'''simple docstring'''






		__a       :       Tuple       =	RoCBertTokenizer
		__a       :       Optional[Any]       =	None
		__a       :       Optional[int]       =	False
		__a       :       Tuple       =	True
		__a       :       str       =	filter_non_english






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									super().setUp()

									lowercase_	:   List[Any]  				=     ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									lowercase_	:   Tuple  				=     {}
									lowercase_	:   Tuple  				=     {}
									for i, value in enumerate(snake_case__  ):
																lowercase_	:   Optional[Any]  				=     i
																lowercase_	:   int  				=     i
									lowercase_	:   Any  				=     os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["""vocab_file"""]  )
									lowercase_	:   Optional[int]  				=     os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["""word_shape_file"""]  )
									lowercase_	:   Optional[int]  				=     os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["""word_pronunciation_file"""]  )
									with open(self.vocab_file,   """w""",   encoding="""utf-8"""  ) as vocab_writer:
																vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )
									with open(self.word_shape_file,   """w""",   encoding="""utf-8"""  ) as word_shape_writer:
																json.dump(snake_case__,   snake_case__,   ensure_ascii=snake_case__  )
									with open(self.word_pronunciation_file,   """w""",   encoding="""utf-8"""  ) as word_pronunciation_writer:
																json.dump(snake_case__,   snake_case__,   ensure_ascii=snake_case__  )






		def 						snake_case__				(							self  )       -> Dict:






									"""simple docstring"""
									lowercase_	:   int  				=     self.tokenizer_class(self.vocab_file,   self.word_shape_file,   self.word_pronunciation_file  )

									lowercase_	:   Union[str, Any]  				=     tokenizer.tokenize("""你好[SEP]你是谁"""  )
									self.assertListEqual(snake_case__,   ["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]  )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__  ),   [5, 6, 2, 5, 7, 8]  )
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(snake_case__  ),   [5, 6, 2, 5, 7, 8]  )
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(snake_case__  ),   [5, 6, 2, 5, 7, 8]  )






		def 						snake_case__				(							self  )       -> Tuple:






									"""simple docstring"""
									lowercase_	:   Optional[int]  				=     RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""  ),   ["""ah""", """\u535A""", """\u63A8""", """zz"""]  )






		def 						snake_case__				(							self  )       -> Tuple:






									"""simple docstring"""
									lowercase_	:   Optional[Any]  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  ),   ["""hello""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ),   ["""hello"""]  )






		def 						snake_case__				(							self  )       -> List[str]:






									"""simple docstring"""
									lowercase_	:   str  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__,   strip_accents=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ),   ["""hällo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ),   ["""h\u00E9llo"""]  )






		def 						snake_case__				(							self  )       -> int:






									"""simple docstring"""
									lowercase_	:   int  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__,   strip_accents=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ),   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ),   ["""hello"""]  )






		def 						snake_case__				(							self  )       -> Optional[int]:






									"""simple docstring"""
									lowercase_	:   Optional[int]  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ),   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  ),   ["""hello"""]  )






		def 						snake_case__				(							self  )       -> Any:






									"""simple docstring"""
									lowercase_	:   str  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  ),   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






		def 						snake_case__				(							self  )       -> Tuple:






									"""simple docstring"""
									lowercase_	:   Any  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__,   strip_accents=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ),   ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									lowercase_	:   Tuple  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__,   strip_accents=snake_case__  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  ),   ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






		def 						snake_case__				(							self  )       -> Union[str, Any]:






									"""simple docstring"""
									lowercase_	:   str  				=     RoCBertBasicTokenizer(do_lower_case=snake_case__,   never_split=["""[UNK]"""]  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""  ),   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]  )






		def 						snake_case__				(							self  )       -> List[Any]:






									"""simple docstring"""
									lowercase_	:   Dict  				=     ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									lowercase_	:   Optional[int]  				=     {}
									for i, token in enumerate(snake_case__  ):
																lowercase_	:   Optional[Any]  				=     i
									lowercase_	:   Optional[int]  				=     RoCBertWordpieceTokenizer(vocab=snake_case__,   unk_token="""[UNK]"""  )

									self.assertListEqual(tokenizer.tokenize(""""""  ),   []  )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""  ),   ["""un""", """##want""", """##ed""", """runn""", """##ing"""]  )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""  ),   ["""[UNK]""", """runn""", """##ing"""]  )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									self.assertTrue(_is_whitespace(""" """  )  )
									self.assertTrue(_is_whitespace("""\t"""  )  )
									self.assertTrue(_is_whitespace("""\r"""  )  )
									self.assertTrue(_is_whitespace("""\n"""  )  )
									self.assertTrue(_is_whitespace("""\u00A0"""  )  )

									self.assertFalse(_is_whitespace("""A"""  )  )
									self.assertFalse(_is_whitespace("""-"""  )  )






		def 						snake_case__				(							self  )       -> List[str]:






									"""simple docstring"""
									self.assertTrue(_is_control("""\u0005"""  )  )

									self.assertFalse(_is_control("""A"""  )  )
									self.assertFalse(_is_control(""" """  )  )
									self.assertFalse(_is_control("""\t"""  )  )
									self.assertFalse(_is_control("""\r"""  )  )






		def 						snake_case__				(							self  )       -> Optional[int]:






									"""simple docstring"""
									self.assertTrue(_is_punctuation("""-"""  )  )
									self.assertTrue(_is_punctuation("""$"""  )  )
									self.assertTrue(_is_punctuation("""`"""  )  )
									self.assertTrue(_is_punctuation("""."""  )  )

									self.assertFalse(_is_punctuation("""A"""  )  )
									self.assertFalse(_is_punctuation(""" """  )  )






		def 						snake_case__				(							self  )       -> List[str]:






									"""simple docstring"""
									lowercase_	:   Any  				=     self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case__  ) for t in ["""Test""", """\xad""", """test"""]],   [["""[UNK]"""], [], ["""[UNK]"""]]  )

									if self.test_rust_tokenizer:
																lowercase_	:   Optional[Any]  				=     self.get_rust_tokenizer()
																self.assertListEqual(
																    [rust_tokenizer.tokenize(snake_case__  ) for t in ["""Test""", """\xad""", """test"""]],   [["""[UNK]"""], [], ["""[UNK]"""]]  )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																							lowercase_	:   List[str]  				=     self.rust_tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )

																							lowercase_	:   Optional[int]  				=     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																							lowercase_	:   List[str]  				=     tokenizer_r.encode_plus(
																							    snake_case__,   return_attention_mask=snake_case__,   return_token_type_ids=snake_case__,   return_offsets_mapping=snake_case__,   add_special_tokens=snake_case__,   )

																							lowercase_	:   str  				=     tokenizer_r.do_lower_case if hasattr(snake_case__,   """do_lower_case"""  ) else False
																							lowercase_	:   List[Any]  				=     (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), """A"""),
																							        ((1, 2), ""","""),
																							        ((3, 5), """na"""),
																							        ((5, 6), """##ï"""),
																							        ((6, 8), """##ve"""),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), """Allen"""),
																							        ((21, 23), """##NL"""),
																							        ((23, 24), """##P"""),
																							        ((25, 33), """sentence"""),
																							        ((33, 34), """."""),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), """a"""),
																							        ((1, 2), ""","""),
																							        ((3, 8), """naive"""),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), """allen"""),
																							        ((21, 23), """##nl"""),
																							        ((23, 24), """##p"""),
																							        ((25, 33), """sentence"""),
																							        ((33, 34), """."""),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results],   tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]  )  )
																							self.assertEqual([e[0] for e in expected_results],   tokens["""offset_mapping"""]  )






		def 						snake_case__				(							self  )       -> Any:






									"""simple docstring"""
									lowercase_	:   int  				=     ["""的""", """人""", """有"""]
									lowercase_	:   List[Any]  				=     """""".join(snake_case__  )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																							lowercase_	:   Any  				=     True
																							lowercase_	:   Union[str, Any]  				=     self.tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )
																							lowercase_	:   Tuple  				=     self.rust_tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )

																							lowercase_	:   str  				=     tokenizer_p.encode(snake_case__,   add_special_tokens=snake_case__  )
																							lowercase_	:   List[str]  				=     tokenizer_r.encode(snake_case__,   add_special_tokens=snake_case__  )

																							lowercase_	:   Optional[int]  				=     tokenizer_r.convert_ids_to_tokens(snake_case__  )
																							lowercase_	:   Optional[Any]  				=     tokenizer_p.convert_ids_to_tokens(snake_case__  )

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(snake_case__,   snake_case__  )
																							self.assertListEqual(snake_case__,   snake_case__  )

																							lowercase_	:   Any  				=     False
																							lowercase_	:   int  				=     self.rust_tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )
																							lowercase_	:   Optional[Any]  				=     self.tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )

																							lowercase_	:   Tuple  				=     tokenizer_r.encode(snake_case__,   add_special_tokens=snake_case__  )
																							lowercase_	:   Any  				=     tokenizer_p.encode(snake_case__,   add_special_tokens=snake_case__  )

																							lowercase_	:   List[Any]  				=     tokenizer_r.convert_ids_to_tokens(snake_case__  )
																							lowercase_	:   Optional[Any]  				=     tokenizer_p.convert_ids_to_tokens(snake_case__  )

																							# it is expected that only the first Chinese character is not preceded by "##".
																							lowercase_	:   int  				=     [
																							    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__  )
																							]
																							self.assertListEqual(snake_case__,   snake_case__  )
																							self.assertListEqual(snake_case__,   snake_case__  )






		@slow
		def 						snake_case__				(							self  )       -> List[Any]:






									"""simple docstring"""
									lowercase_	:   Any  				=     self.tokenizer_class(self.vocab_file,   self.word_shape_file,   self.word_pronunciation_file  )

									lowercase_	:   Any  				=     tokenizer.encode("""你好""",   add_special_tokens=snake_case__  )
									lowercase_	:   List[str]  				=     tokenizer.encode("""你是谁""",   add_special_tokens=snake_case__  )

									lowercase_	:   str  				=     tokenizer.build_inputs_with_special_tokens(snake_case__  )
									lowercase_	:   str  				=     tokenizer.build_inputs_with_special_tokens(snake_case__,   snake_case__  )

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]






		def 						snake_case__				(							self  )       -> Optional[Any]:






									"""simple docstring"""
									lowercase_	:   int  				=     self.get_tokenizers(do_lower_case=snake_case__  )
									for tokenizer in tokenizers:
																with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
																							lowercase_	:   str  				=     """你好，你是谁"""
																							lowercase_	:   Optional[int]  				=     tokenizer.tokenize(snake_case__  )
																							lowercase_	:   List[Any]  				=     tokenizer.convert_tokens_to_ids(snake_case__  )
																							lowercase_	:   int  				=     tokenizer.convert_tokens_to_shape_ids(snake_case__  )
																							lowercase_	:   Optional[Any]  				=     tokenizer.convert_tokens_to_pronunciation_ids(snake_case__  )
																							lowercase_	:   Tuple  				=     tokenizer.prepare_for_model(
																							    snake_case__,   snake_case__,   snake_case__,   add_special_tokens=snake_case__  )

																							lowercase_	:   Dict  				=     tokenizer.encode_plus(snake_case__,   add_special_tokens=snake_case__  )

																							self.assertEqual(snake_case__,   snake_case__  )






from __future__ import annotations

import numpy as np
from numpy import floataa
from numpy.typing import NDArray





def  __magic_name__  (				lowercase				,       lowercase				,       lowercase				,       lowercase				,       )				-> list[float]:

							"""simple docstring"""



							lowercase_				,		lowercase_	:   List[str]  				=     coefficient_matrix.shape
							lowercase_				,		lowercase_	:   Any  				=     constant_matrix.shape

							if rowsa != colsa:
														lowercase_	:   List[Any]  				=     f"""Coefficient matrix dimensions must be nxn but received {rowsa}x{colsa}"""
														raise ValueError(lowercase				)

							if colsa != 1:
														lowercase_	:   Optional[int]  				=     f"""Constant matrix must be nx1 but received {rowsa}x{colsa}"""
														raise ValueError(lowercase				)

							if rowsa != rowsa:
														lowercase_	:   Tuple  				=     (
														    """Coefficient and constant matrices dimensions must be nxn and nx1 but """
														    f"""received {rowsa}x{colsa} and {rowsa}x{colsa}"""
														)
														raise ValueError(lowercase				)

							if len(lowercase				) != rowsa:
														lowercase_	:   int  				=     (
														    """Number of initial values must be equal to number of rows in coefficient """
														    f"""matrix but received {len(lowercase				)} and {rowsa}"""
														)
														raise ValueError(lowercase				)

							if iterations <= 0:
														raise ValueError("""Iterations must be at least 1"""				)

							lowercase_	:   NDArray[floataa]  				=     np.concatenate(
							    (coefficient_matrix, constant_matrix)				,       axis=1				)

							lowercase_				,		lowercase_	:   Dict  				=     table.shape

							strictly_diagonally_dominant(lowercase				)

							# Iterates the whole matrix for given number of times
							for _ in range(lowercase				):
														lowercase_	:   str  				=     []
														for row in range(lowercase				):
																					lowercase_	:   Dict  				=     0
																					for col in range(lowercase				):
																												if col == row:
																																			lowercase_	:   Optional[Any]  				=     table[row][col]
																												elif col == cols - 1:
																																			lowercase_	:   Tuple  				=     table[row][col]
																												else:
																																			temp += (-1) * table[row][col] * init_val[col]
																					lowercase_	:   List[Any]  				=     (temp + val) / denom
																					new_val.append(lowercase				)
														lowercase_	:   Optional[int]  				=     new_val

							return [float(lowercase				) for i in new_val]





def  __magic_name__  (				lowercase				)				-> bool:

							"""simple docstring"""



							lowercase_				,		lowercase_	:   str  				=     table.shape

							lowercase_	:   str  				=     True

							for i in range(0				,       lowercase				):
														lowercase_	:   Any  				=     0
														for j in range(0				,       cols - 1				):
																					if i == j:
																												continue
																					else:
																												total += table[i][j]

														if table[i][i] <= total:
																					raise ValueError("""Coefficient matrix is not strictly diagonally dominant"""				)

							return is_diagonally_dominant


# Test Cases
if __name__ == "__main__":
							import doctest

							doctest.testmod()